diff --git a/backend/.env.example b/backend/.env.example index 20a107b4..51a1c782 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -129,9 +129,28 @@ MENDER_PAT= # from its association guess before it is rejected; the default is 6 km = 2 x # the 3 km association grid step, since a dark guess is a quantised grid point # rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap. +# +# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default) +# solves once per fixed altitude layer and keeps the lowest rms_delay — six +# process-pool round trips per candidate, and an altitude quantised to a +# ladder whose 2 km spacing puts up to 1 km of error straight into the +# residual the reject gate reads. free makes one pool call to the geolocator's +# multi-start helper, which solves altitude as a sixth unknown. Both modes +# stamp altitude_mode on the solve-history record, so /api/test/mlat-history +# can compare them across a deploy of each. See docs/solverflow.md. +# +# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that +# helper, clamped to the number of layers. 1 (the default) starts at the layer +# nearest the association guess, or at the guess altitude itself when it comes +# from ADS-B; more is a window around it, at one LM run each. Measured on test +# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in +# 13 of them, so the extra runs are off by default and worth turning on only +# where the geometry sends a single start to the wrong side of an ellipse. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 +# SOLVER_ALT_MODE=sweep +# SOLVER_FREE_ALT_STARTS=1 # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/config/constants.py b/backend/config/constants.py index ea788abf..2d1cd19c 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -84,6 +84,19 @@ def as_num(v) -> float: N2_CONFIRM_MIN_EPOCHS = 4 # Floor on samples; span is the real gate N2_TRACK_HISTORY_MAX = 20 # Per-node track samples fed to the fit +# How old a track's newest REAL detection may be before the track stops being +# offered to association (see services/frame_processor.confirmed_track_views). +# A COASTING track is kept alive for N_DELETE=10 frames after its last +# association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s +# of dead reckoning. confirmed_track_views hands association the track's last +# real sample, and association hands the solver that sample as if it were +# current — so an aircraft that has flown out of a node's beam keeps +# contributing a seconds-old delay to n>=3 solves. Measured on the test +# droplet: 230 out of-cone nodes survived into published dark solves in 20 min, +# median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range, +# and they are the nodes the rms trim then throws away. 0 disables the filter. +TRACK_MAX_STALE_S = float(os.getenv("TRACK_MAX_STALE_S", "3.0")) + # A 2-node track needs this many solves before it renders a plane; 1 # disables the gate. One-shot n=2 solves were the dominant ghost source. MN_N2_MIN_SOLVES = int(os.getenv("MN_N2_MIN_SOLVES", "2")) diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..e2b2338d 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,56 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# Measurement epoch alignment (see services/tasks/solver.align_measurement_epochs). +# on/off rather than the off/shadow/active triple its neighbours use: there is +# nothing to shadow — the correction is a closed-form dead-reckoning of each +# delay along its own measured Doppler, so a dry run would produce the same +# number the acting run applies and observe nothing extra. Default "on", +# because leaving it off is the bug: nodes sample at independent phases and the +# solver treats their measurements as simultaneous, so a 250 m/s target charges +# up to ~1 us of delay error per second of skew (measured ~0.3 us rms at 2 s +# skew on the fleet) straight to the 3 us rms gate. The flag exists so the +# alignment can be turned off live without a rollback if it ever misbehaves. +SOLVER_EPOCH_ALIGN = os.getenv("SOLVER_EPOCH_ALIGN", "on").strip().lower() != "off" +# How the n>=3 solve gets its altitude (see services/tasks/solver.py's +# _solve_best_altitude). sweep/free, read here rather than in that module so +# it sits with its sibling mode flags and a test can monkeypatch it without +# reimporting the solver. +# sweep (default) — solve once per fixed altitude layer, keep the lowest +# rms_delay. Six pool round trips per candidate, and an altitude +# quantised to the ladder: layers are 2 km apart, so the pin is +# systematically up to 1 km wrong and that error lands in the +# residual the reject gate reads. +# free — one pool call to retina_geolocator's multi-start helper, which +# solves altitude as a sixth unknown, started from +# SOLVER_FREE_ALT_STARTS of those layers. +# Not off/shadow/active: there is no shadow here, because the two modes +# produce the same shape of result and the history record carries +# altitude_mode either way — running both would double the solver's cost to +# learn what one deploy of each already says. An unrecognised value falls +# back to "sweep", the same degrade-to-inert rule the sibling flags use. +SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower() +if SOLVER_ALT_MODE not in ("sweep", "free"): + SOLVER_ALT_MODE = "sweep" + +# How many start altitudes the free mode hands that helper. Read here beside +# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it +# into [1, len(layers)] against the ladder that module owns. 1 starts at the +# layer nearest the association guess — where the sweep would have pinned; +# more is a window around it. +# +# The default is 1 because three starts did not pay for themselves: over 1019 +# free-mode solves on test, the three starts' rms_delay differed by more than +# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse +# than the best start in 2. That is ~0.2% of solves helped for 3x the solver +# CPU, and the pool — not the altitude ladder — is what this deployment is +# short of (~1.7 attempts/s against a 2.0 s average latency on two workers). +# The knob stays because the reason for several starts is the LM's locality, +# which is a property of the geometry rather than of this fleet: nodes lying +# nearer a bistatic ellipse than these can send a single start to the wrong +# side of it, and finding that out should not need a code change. +SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1"))) + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) @@ -475,6 +525,15 @@ def _adsb_for_seeding() -> dict[str, dict]: # Monotonic counter for dropped frames (useful for monitoring) frames_dropped: int = 0 +# Frames the per-node rate limiter refused before they ever reached +# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate). A different +# event from frames_dropped, which is queue saturation: this one is the +# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at +# 22 fps therefore reports a large number here while dropping nothing. It was +# uncounted, so "how much of a node's evidence does the tracker actually see" +# had no answer at all — the frames_dropped that IS published +# (/api/admin/metrics) says zero throughout. +node_frames_rate_limited: int = 0 frames_processed: int = 0 solver_successes: int = 0 solver_failures: int = 0 @@ -526,6 +585,21 @@ def _adsb_for_seeding() -> dict[str, dict]: # fleet's trigger rate — constraints are then converging slower than the # coverage they follow, which no rebuild counter can show. coverage_rebuild_backlog: int = 0 + +# Confirmed tracks withheld from association because their newest REAL +# detection was older than TRACK_MAX_STALE_S at the frame being processed — +# see services/frame_processor.confirmed_track_views. These are aircraft that +# have left a node's beam and whose track is dead-reckoning toward deletion; +# their last real sample used to reach the solver as a current measurement. +tracks_stale_skipped: int = 0 + +# Solver inputs whose measurements could not be aligned to a common epoch +# because at least one lacked t_s, doppler_hz, or a node config with fc_hz — +# see services/tasks/solver.align_measurement_epochs. Counted only when +# SOLVER_EPOCH_ALIGN is on; a nonzero value against solver_successes says how +# much of the fleet is still emitting untimed measurements. +solver_epoch_align_skipped: int = 0 + solver_queue_drops: int = 0 # Queue items discarded unsolved because they aged past _SOLVER_MAX_QUEUE_AGE_S # waiting for a worker. Was only a DEBUG log, which staging does not emit — @@ -535,14 +609,36 @@ def _adsb_for_seeding() -> dict[str, dict]: solver_stale_drops: int = 0 # Candidates dequeued and skipped because every single-node track they carry -# was already solved within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes (see -# solver.py's _claim_resolve_slot). Association is per-node and rate-limited -# per node, so one aircraft arrives as one candidate per node that can see it; -# this counts the copies that were never worth solving. High against -# solver_successes is normal and is the mechanism working — it is -# solver_stale_drops that means work was lost. +# was already PUBLISHED within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes +# (see solver.py's _resolve_slot_covered). Association is per-node and +# rate-limited per node, so one aircraft arrives as one candidate per node +# that can see it; this counts the copies that were never worth solving. High +# against solver_successes is normal and is the mechanism working — it is +# solver_stale_drops that means work was lost. Read it against +# solver_successes, not against attempts: while the claim was taken on +# ADMISSION rather than on publication, a rejected candidate blacked out every +# later one sharing a track id and this counter ran at ~2.4x attempts. solver_resolve_skips: int = 0 +# The dark-lane share of the counter above, split out because the two lanes +# read completely differently: an ADS-B-anchored duplicate that is skipped +# costs nothing (the transponder keeps the track alive anyway), while a +# skipped dark candidate may be the only chance that aircraft had of reaching +# the map this window. Lane is decided by solver._is_dark_solver_input, the +# same predicate routes.test._record_lane falls back to for a record that +# never got a key — and a skip never gets one. +solver_resolve_skips_dark: int = 0 + +# The last few hundred resolve-slot skips, with the claims that blocked them. +# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and +# writing one record per skip into mlat_solve_history would evict the real +# records at roughly twice their rate (live: ~1 537 skips per 646 dark +# attempts per 30 min). Small and separate, read by +# /api/test/solver-stats' resolve_skips block and dumped by +# /api/test/mlat-history?kind=resolve_skips. ~250 B/entry. +SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500 +solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX) + # Multinode entries removed because a later solve shared a source single-node # track with them AND the spatial/identical-inputs guard in solver.py's # _supersession_match agreed they are the same aircraft — the age-scaled @@ -746,12 +842,14 @@ def _reset_for_tests() -> None: global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes global latest_storage_bytes, simulation_config global frames_dropped, frames_processed, solver_successes, solver_failures + global node_frames_rate_limited global adsb_seed_frames_autotagged, adsb_capture_ts_fallback global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes - global coverage_rebuild_backlog + global coverage_rebuild_backlog, tracks_stale_skipped, solver_epoch_align_skipped global solver_queue_drops, solver_stale_drops, solver_resolve_skips + global solver_resolve_skips_dark global mn_superseded, mn_superseded_blocked, solver_trimmed global solver_consensus_selected, solver_consensus_filtered global solver_consensus_fallback, solver_consensus_shadow @@ -801,6 +899,7 @@ def _reset_for_tests() -> None: track_archive_buffer.clear() mlat_solve_history.clear() mlat_solve_history_known.clear() + solver_resolve_skips_recent.clear() accuracy_samples.clear() mlat_samples.clear() for q in (frame_queue, solver_queue): @@ -832,7 +931,7 @@ def _reset_for_tests() -> None: simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS) with counters_lock: - frames_dropped = frames_processed = 0 + frames_dropped = frames_processed = node_frames_rate_limited = 0 solver_successes = solver_failures = n2_unconfirmed = 0 adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0 known_claims_made = known_claim_contentions = known_claims_bound = 0 @@ -840,8 +939,9 @@ def _reset_for_tests() -> None: known_claims_world_rejects = 0 coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 + tracks_stale_skipped = solver_epoch_align_skipped = 0 solver_stale_drops = 0 - solver_resolve_skips = 0 + solver_resolve_skips = solver_resolve_skips_dark = 0 mn_superseded = mn_superseded_blocked = 0 solver_trimmed = 0 solver_consensus_selected = solver_consensus_filtered = 0 diff --git a/backend/routes/admin.py b/backend/routes/admin.py index 2d0eca6d..4d11fb0b 100644 --- a/backend/routes/admin.py +++ b/backend/routes/admin.py @@ -590,6 +590,8 @@ async def system_metrics(_user=Depends(require_admin)): "solver_queue_drops": state.solver_queue_drops, "solver_stale_drops": state.solver_stale_drops, "solver_resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "solver_epoch_align_skipped": state.solver_epoch_align_skipped, "mn_superseded": state.mn_superseded, "solver_trimmed": state.solver_trimmed, "solver_last_latency_s": round(state.solver_last_latency_s, 3), diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py index 07d83a9c..93af0243 100644 --- a/backend/routes/analytics.py +++ b/backend/routes/analytics.py @@ -125,26 +125,42 @@ async def association_status(): return { "registered_nodes": len(_a.node_geometries), "overlap_zones": len(_a.overlap_zones), + # Node pairs that got no grid because the two nodes are in different + # worlds (node_world above). Read next to overlap_zones: on a fleet + # of 50 synthetic nodes over the same city as 8 receivers it is the + # 400 sim/real pairs whose grids could only ever have paired a + # simulated echo with a real one. Counted per pair considered, so it + # keeps rising as nodes re-register — zero means the fleet is single- + # world (or untagged), not that the gate is off. + "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0), # Confirmed single-node tracks each node last submitted; these are what # pairings are drawn from. "pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())}, # Track-pairing outcomes since boot. gated is everything past the # coarse delay grid; unfitted counts the pairings handed to the solver # worker (which runs the fit and the n=2 gate); deferred counts rounds - # a budget cut short. Those three are the live production surface. + # a budget cut short; superseded counts pairings dropped because a + # better-ranked one claiming the same track implied a velocity theirs + # contradicts; cluster_splits counts position clusters that held two + # tracks of one node and were emitted as one solver input each. All + # five are the live production surface. "track_pairs": { "gated": getattr(_a, "track_pairs_gated", 0), "unfitted": getattr(_a, "track_pairs_unfitted", 0), "deferred": getattr(_a, "track_pairs_deferred", 0), + "superseded": getattr(_a, "track_pairs_superseded", 0), + "cluster_splits": getattr(_a, "cluster_splits", 0), }, # Inline-fit counters — permanently zero in production BY DESIGN # (state.py builds the associator with cv_fit=None; only the offline - # bench's inline mode exercises stage-2 selection). Split out so + # bench's inline mode exercises the chi2 threshold). Split out so # nobody reads a structural zero as "no rejections happening". + # superseded used to live here too, and no longer can: the deferred + # path now has an exclusivity stage of its own, so the counter moves + # in production. "track_pairs_inline_only": { "accepted": getattr(_a, "track_pairs_accepted", 0), "rejected": getattr(_a, "track_pairs_rejected", 0), - "superseded": getattr(_a, "track_pairs_superseded", 0), }, # Top-down claiming (ASSOC_CLAIM_MODE) since boot. rounds/matched/ # conflicts/anchored_inputs are all live in shadow too — _claim_round diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..7874c8ae 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -210,10 +210,17 @@ def _build_dashboard_data() -> bytes: # not the queue size. "stale_drops": state.solver_stale_drops, # Duplicate candidates for an aircraft already solved this - # window (see solver.py's _claim_resolve_slot). Read it + # window (see solver.py's _resolve_slot_covered). Read it # against stale_drops: skips are work correctly not done, # stale drops are work lost. "resolve_skips": state.solver_resolve_skips, + # Confirmed tracks withheld from association because their + # newest real detection had aged past TRACK_MAX_STALE_S, and + # solver inputs the epoch alignment could not correct because a + # measurement carried no sample time (see frame_processor's + # confirmed_track_views and solver's align_measurement_epochs). + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, # Multinode entries replaced because a later solve consumed # the same source tracks under a new key (fragmented re-solve). "mn_superseded": state.mn_superseded, @@ -752,11 +759,40 @@ def _record_lane(rec: dict) -> str: return "adsb" if hexn and is_transponder_hex(hexn) else "dark" +_LANES = ("dark", "known", "adsb") + + +def _cap_per_lane(records: list[dict], limit: int) -> list[dict]: + """Keep the ``limit`` newest records OF EACH LANE, newest first. + + ``records`` must already be newest-first. A single flat ``[:limit]`` + made the cap a race between lanes rather than a retention rule, exactly + as the shared deque did before PR #289 split it: the known lane writes + ~16x the dark lane's volume, so a flat 1 000-record answer to a 30 min + request held only the newest ~6 min of dark records and the rest of the + window read as a quiet period. Capping per lane means known-lane volume + can never evict a dark record from a response. + """ + kept: list[dict] = [] + counts: dict[str, int] = {} + for r in records: + lane = _record_lane(r) + n = counts.get(lane, 0) + if n >= limit: + continue + counts[lane] = n + 1 + kept.append(r) + return kept + + @router.get("/api/test/mlat-history") async def mlat_history( hex: str | None = None, all: int = 0, minutes: float = 30.0, + lane: str = "all", + limit: int = 1000, + kind: str = "solves", ): """Per-solve MLAT history from the last ~30 minutes. @@ -770,22 +806,73 @@ async def mlat_history( merged here, so both lanes answer either query exactly as they did when they shared a deque. + ``?lane=dark|known|adsb`` narrows the answer to one lane (default + ``all``, classified by ``_record_lane``); ``?limit=`` caps the record + list (default 1 000, max 5 000) and is applied PER LANE, so a known-lane + burst can never push dark records out of an ``all`` response — see + _cap_per_lane. + + ``?kind=resolve_skips`` dumps a different store entirely: the solver's + recent resolve-slot refusals (state.solver_resolve_skips_recent), each + with the claims that blocked it. Those are not solve outcomes and + deliberately do not live in the solve-history deques. + ``window_effective_minutes`` is how much of the requested window the stores actually hold — below ``window_minutes`` the answer is truncated. """ + if lane not in ("all", *_LANES): + return Response( + content=orjson.dumps({"error": f"lane must be one of all,{','.join(_LANES)}"}), + media_type="application/json", + status_code=400, + ) + if kind not in ("solves", "resolve_skips"): + return Response( + content=orjson.dumps({"error": "kind must be solves or resolve_skips"}), + media_type="application/json", + status_code=400, + ) minutes = max(0.0, min(minutes, 35.0)) + limit = max(1, min(int(limit), 5000)) cutoff_ms = int((time.time() - minutes * 60.0) * 1000) + + if kind == "resolve_skips": + skips = [ + s + for s in list(state.solver_resolve_skips_recent) + if s["ts_ms"] >= cutoff_ms and (lane == "all" or s["lane"] == lane) + ] + skips.reverse() # newest first + payload = { + "kind": "resolve_skips", + "window_minutes": minutes, + "lane": lane, + "lane_counts": {ln: sum(1 for s in skips if s["lane"] == ln) for ln in _LANES}, + "n_records": len(skips), + "records": skips[:limit], + } + return Response(content=orjson.dumps(payload), media_type="application/json") + merged = _merged_solve_history() effective_minutes = _window_effective_minutes(merged, minutes) records = [r for r in merged if r["ts_ms"] >= cutoff_ms] records.reverse() # newest first + if lane != "all": + records = [r for r in records if _record_lane(r) == lane] + lane_counts = dict.fromkeys(_LANES, 0) + for r in records: + lane_counts[_record_lane(r)] += 1 if all: payload = { "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + # Pre-cap, so a truncated `records` can be read against what the + # window actually held. + "lane_counts": lane_counts, "n_records": len(records), - "records": records[:1000], + "records": _cap_per_lane(records, limit), } return Response(content=orjson.dumps(payload), media_type="application/json") @@ -813,6 +900,8 @@ async def mlat_history( "hex": norm, "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + "lane_counts": lane_counts, "n_solves": len(solves), "solves": solves[:500], "rejects_nearby": { @@ -914,6 +1003,46 @@ def _solver_window_stats(minutes: float) -> dict: reason = outcome[len("rejected_") :] if outcome.startswith("rejected_") else outcome by_reason[reason] = by_reason.get(reason, 0) + 1 + # ── cluster contamination (dark, windowed) ────────────────────────────── + # Of the dark records this window that matched ground truth, how many + # carried a node that could not see the aircraft they were matched to — + # the live version of the offline number Phase 2 exists to move (~60 %). + # Records without the stamp are records nothing could be asked about (no + # GT match, or no registered geometry for any contributing node) and stay + # out of the denominator rather than counting as clean; see + # solver._stamp_foreign_nodes. + judged = [r for r in records if r.get("foreign_node_ids") is not None] + contaminated = [r for r in judged if r.get("contaminated")] + n_judged = len(judged) + contamination = { + "records_with_gt": n_judged, + "contaminated": len(contaminated), + "pct": round(100.0 * len(contaminated) / n_judged, 1) if n_judged else None, + "foreign_nodes_per_record": ( + round(sum(len(r["foreign_node_ids"]) for r in judged) / n_judged, 2) if n_judged else None + ), + } + + # ── resolve-slot skips (windowed, from the skip deque) ────────────────── + # The counter in "counters" below is since-boot; these are the skips that + # happened inside this window, so they can be read against the attempts in + # the same window. attempts_ratio is (all-lane skips / DARK attempts) — + # the shape the acceptance target for the claim-on-publish fix is quoted + # in (live baseline ~1 537 / 646 = 2.4), not a per-lane rate. The dark + # numerator is published beside it for anyone who wants one. + all_skips = list(state.solver_resolve_skips_recent) + skips = [s for s in all_skips if s["ts_ms"] >= cutoff_ms] + resolve_skips = { + "total": len(skips), + "dark": sum(1 for s in skips if s["lane"] == "dark"), + "attempts_ratio": round(len(skips) / attempts, 3) if attempts else None, + # The skip deque is 500 entries against a live rate of ~50/min, so a + # long window IS truncated here even when the solve-history stores + # cover it. Same honesty rule as window_effective_minutes above: read + # it before reading total as a window count. + "window_effective_minutes": _window_effective_minutes(all_skips, minutes), + } + pos_errors.sort() n_err = len(pos_errors) median_err = pos_errors[n_err // 2] if n_err else None @@ -1066,6 +1195,9 @@ def _solver_window_stats(minutes: float) -> dict: "published": {"total": n2 + n3plus, "n2": n2, "n3plus": n3plus}, "rejects": {"total": reject_total, "by_reason": by_reason}, "position_error_km": {"median": median_err, "p90": p90_err, "n": n_err}, + # Both windowed and both DARK-lane, like the funnel above them. + "contamination": contamination, + "resolve_skips": resolve_skips, "ghosts": { # Scoped to dark tracks: precision_pct's denominator is # dark_tracks, and it is None (not 100.0) when there are none. @@ -1192,7 +1324,17 @@ def _solver_window_stats(minutes: float) -> dict: "solver_trimmed": state.solver_trimmed, "stale_drops": state.solver_stale_drops, "resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, + # Dark share of the line above. The windowed version, with the + # blocking claims, is the "resolve_skips" block further up. + "resolve_skips_dark": state.solver_resolve_skips_dark, "queue_drops": state.solver_queue_drops, + # Frames the per-node rate limiter refused before the tracker ever + # saw them (tcp_handler's NODE_FRAME_MIN_INTERVAL_S). Not the + # same event as /api/admin/metrics' frames_dropped, which is + # frame_queue saturation and normally reads zero. + "node_frames_rate_limited": state.node_frames_rate_limited, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, }, diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index e08999bb..1bbe8967 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -69,6 +69,7 @@ # superseded detection path that --mode detection measures as the baseline, so # constructing it unconditionally leaves --mode track unaffected. import retina_analytics.association as _assoc_module # noqa: E402 +from retina_analytics.association import predict_observation # noqa: E402 from retina_analytics.detection_association import DetectionAssociator # noqa: E402 from retina_analytics.manager import NodeAnalyticsManager # noqa: E402 from retina_geolocator.consensus import solve_consensus # noqa: E402 @@ -248,6 +249,133 @@ def _frame_to_detections(frame: dict) -> list[dict]: return dets +# ── Contamination scoring (truth side-channel) ─────────────────────────── +# How far a detection may sit from an aircraft's noiseless (delay, doppler) +# and still be attributed to it. The simulator's own measurement noise is +# gauss(0, 0.1-0.2 us) in delay and gauss(0, 2-4 Hz) in Doppler +# (world.generate_detections_for_node), so these are ~5 sigma: wide enough +# that a real echo is never mistaken for clutter, tight enough that clutter +# — uniform over tens of us — almost never lands on an aircraft. +_TRUTH_DELAY_GATE_US = 1.0 +_TRUTH_DOPPLER_GATE_HZ = 25.0 +# Sentinel for "this measurement matched two different aircraft equally +# well". Neither foreign nor own — excluded from the numerator so an +# ambiguity in the scorer is never reported as a contamination. +_TRUTH_AMBIGUOUS = "?ambiguous" + + +def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, aircraft: list) -> None: + """Record which aircraft produced each detection in one node's frame. + + Truth side-channel, for the CONTAMINATION metric only: it is built from + the frame BEFORE _strip_adsb and never reaches association, so the blind + discipline is intact. It cannot be read off the frame's own ``adsb`` + list either — the simulator appends None there for every aircraft with + has_adsb False, and dark aircraft are exactly the population this metric + exists to score. So each detection is instead matched back to the + aircraft whose noiseless observation it is nearest. + + Keyed on the (delay, doppler) floats themselves because that is the only + handle the metric gets downstream: a solver input's measurement carries + its track's latest delay/doppler verbatim (history[-1] -> the detection + dict -> here), and the simulator rounds both to 2 dp, so the equality is + exact rather than approximate. + """ + delays = frame.get("delay") or [] + if not delays: + return + dopplers = frame.get("doppler") or [] + preds = [] + for ac in aircraft: + d_us, f_hz = predict_observation( + geo, + ac.lat, + ac.lon, + ac.alt_km, + ac.vel_east * 1000.0, + ac.vel_north * 1000.0, + ac.vel_up * 1000.0, + ) + preds.append((d_us, f_hz, ac.object_id)) + for d, f in zip(delays, dopplers): + best = best2 = None + for d_us, f_hz, oid in preds: + dd, df = abs(d - d_us), abs(f - f_hz) + if dd > _TRUTH_DELAY_GATE_US or df > _TRUTH_DOPPLER_GATE_HZ: + continue + # Normalised so the two axes are comparable at their own gates. + cost = (dd / _TRUTH_DELAY_GATE_US) ** 2 + (df / _TRUTH_DOPPLER_GATE_HZ) ** 2 + if best is None or cost < best[0]: + best, best2 = (cost, oid), best + elif best2 is None or cost < best2[0]: + best2 = (cost, oid) + if best is None: + continue # clutter: left absent, which the scorer reads as foreign + oid = best[1] + if best2 is not None and best2[0] < 4.0 * best[0]: + oid = _TRUTH_AMBIGUOUS + key = (node_id, float(d), float(f)) + prev = det_truth.get(key) + # The same (node, delay, doppler) recurring for a different aircraft + # later in the run would silently relabel an earlier measurement, so + # a collision demotes the key rather than overwriting it. + det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS + + +def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> int | None: + """Count the nodes in one solver input that are not looking at its aircraft. + + The input's own aircraft is the plurality of its measurements' true + aircraft — the honest reading of "what is this candidate mostly about", + and the one that does not assume the (possibly contaminated) initial + guess is anywhere near a target. Ties are broken by the nearest ground + truth to the initial guess, which is the criterion the ghost/matched + split already uses. + + A node is foreign when its measurement belongs to a different aircraft, + or to no aircraft at all (clutter that survived the tracker's M-of-N and + the delay grid). Ambiguous attributions are counted in neither. + + Returns the foreign-node count, so the caller can score the same input + again at the publish point (see the PUBLISHED counters on Result: the + candidate-level rate has a denominator the association layer itself moves, + and a change that emits more, cleaner candidates reads as a regression on + it while being an improvement on what actually reaches the map). None + when nothing could be attributed at all. + """ + oids = [ + det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"]))) + for m in s_in.get("measurements") or [] + ] + if not oids: + return None + counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS) + if not counts: + return None + top_n = max(counts.values()) + contenders = sorted(o for o, c in counts.items() if c == top_n) + if len(contenders) > 1: + guess = s_in.get("initial_guess") or {} + contenders.sort( + key=lambda o: min( + ( + _haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) + for a, b, oid, _ in truth + if oid == o + ), + default=float("inf"), + ) + ) + own = contenders[0] + foreign = sum(1 for o in oids if o != own and o != _TRUTH_AMBIGUOUS) + res.inputs_scored += 1 + res.input_nodes += len(oids) + res.foreign_nodes += foreign + if foreign: + res.inputs_contaminated += 1 + return foreign + + def _strip_adsb(frame: dict) -> dict: """Return the frame as a real receiver would see it. @@ -482,6 +610,9 @@ class Result: gate_accepted: int = 0 gate_unfitted: int = 0 gate_superseded: int = 0 + # Position clusters that held two different tracks of one node and were + # split into one solver input each, straight off the associator. + cluster_splits: int = 0 # Deferred mode only: what the *solver-side* n=2 gate did. In production the # associator emits unscored pairings and this gate is the one that runs, so # without these the shipped configuration's selection is invisible. @@ -531,6 +662,30 @@ class Result: keys_real: int = 0 keys_ghost: int = 0 + # ── Candidate contamination (--mode track) ──────────────────────────── + # Scored on every solver input association emits, BEFORE the solve and + # before every downstream gate: the question is what association handed + # the solver, not what survived it. A contaminated input is one whose + # measurements do not all belong to the same aircraft — the failure the + # cluster-merge rework targets, and the one the ghost rate cannot see + # (a two-aircraft merge usually still solves within MATCH_KM of one of + # them, so it counts as matched while carrying 4-5 km of position + # error). See _score_contamination. + inputs_scored: int = 0 + inputs_contaminated: int = 0 + foreign_nodes: int = 0 + input_nodes: int = 0 + # The same score restricted to inputs that cleared every gate and bound + # to a real aircraft — what actually reached the map. Reported next to + # the candidate rate because the two answer different questions: the + # candidate rate's denominator is the number of candidates association + # chooses to emit, so splitting one contaminated cluster into several + # clean ones plus the false pairing it was hiding *raises* it while + # lowering this one. + published_inputs: int = 0 + published_contaminated: int = 0 + published_foreign_nodes: int = 0 + # Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None # when it was off, stonesoup wasn't available, or the recorder had # nothing to score (see stonesoup_metrics.MetricRecorder.compute). Not @@ -634,6 +789,14 @@ def keys_per_object(self): "distinct_keys", "keys_real", "keys_ghost", + "inputs_scored", + "inputs_contaminated", + "foreign_nodes", + "input_nodes", + "published_inputs", + "published_contaminated", + "published_foreign_nodes", + "cluster_splits", ) _EXTEND_FIELDS = ( "errors_km", @@ -685,6 +848,22 @@ def merge(self, other: Result, tag: str) -> None: def ghost_pct(self): return 100.0 * self.ghosts / self.total if self.total else 0.0 + @property + def contaminated_inputs_pct(self): + return 100.0 * self.inputs_contaminated / self.inputs_scored if self.inputs_scored else 0.0 + + @property + def foreign_nodes_per_input(self): + return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0 + + @property + def published_contaminated_pct(self): + return 100.0 * self.published_contaminated / self.published_inputs if self.published_inputs else 0.0 + + @property + def published_foreign_per_solve(self): + return self.published_foreign_nodes / self.published_inputs if self.published_inputs else 0.0 + def build_scene( seed: int, @@ -789,6 +968,7 @@ def run( ss_metric_dt=5.0, ss_hold_s=12.0, smoother_legs=None, + cluster_opts=None, ) -> Result: import random @@ -861,12 +1041,17 @@ def fov_provider(node_id): # emits unscored pairings and the solver worker fits and arbitrates. The # two are different code paths, so they need separate baselines. deferred = mode == "track" and cv_fit_mode == "deferred" + # The cluster-merge knobs are passed only when the caller overrode them, + # so a plain run measures whatever the library currently ships rather than + # freezing today's defaults into the bench. + _cluster_kwargs = {k: v for k, v in (cluster_opts or {}).items() if v is not None} assoc = DetectionAssociator( grid_step_km=3.0, cv_fit=(fit_constant_velocity if (mode == "track" and not deferred) else None), cv_chi2_max=chi2_max, cv_min_span_s=min_span_s, cv_exclusive=exclusive, + **_cluster_kwargs, ) n2_gate = DeferredN2Gate(chi2_max, claim_ttl_s=claim_ttl_s, claim_policy=claim_policy) if deferred else None # One tracker per node, driven by every frame — mirrors @@ -915,6 +1100,10 @@ def fov_provider(node_id): _BENCH_MN_MAX_AGE_MS = 60_000 res = Result() + # (node_id, delay_us, doppler_hz) -> the aircraft that produced that + # detection. Truth side-channel for the contamination metric only, built + # from the un-stripped frame below — see _index_detection_truth. + det_truth: dict = {} _all_keys_seen: set = set() _keys_real: set = set() _keys_ghost: set = set() @@ -960,6 +1149,10 @@ def _geo_key(nid): for nid in due_nodes: next_send[nid] += frame_interval frame = world.generate_detections_for_node(nid, ts_ms) + if mode == "track": + # Before _strip_adsb, and never fed to association: the + # contamination metric's truth channel. + _index_detection_truth(det_truth, assoc.node_geometries[nid], nid, frame, world.aircraft) if fov_analytics is not None: # The truth channel, not the (possibly blind) association # stream below -- a real node's ADS-B calibration reaches @@ -978,7 +1171,7 @@ def _geo_key(nid): # frame is what association pairs against — but only let a node # *trigger* a round on its own cadence. if mode == "track": - assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n) + assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n, ts_ms) else: assoc._pending_frames[nid] = frame if (t - last_assoc.get(nid, -1e9)) < assoc_interval: @@ -1001,6 +1194,13 @@ def _geo_key(nid): res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1 if s_in.get("n_nodes", 0) < 2: continue + _foreign = None + if mode == "track": + # Scored here, ahead of the solve and every gate below: + # this measures what association emitted, which is the + # thing the cluster-merge rework changes. Re-scored at + # the publish point further down, on the same number. + _foreign = _score_contamination(res, s_in, det_truth, truth) try: _t0 = time.perf_counter() out = solve_fn(s_in, node_cfgs) @@ -1104,6 +1304,17 @@ def _geo_key(nid): (_keys_real if d <= MATCH_KM else _keys_ghost).add(_key) if d <= MATCH_KM: res.matched += 1 + if _foreign is not None: + # Same input, scored again now that every gate has + # accepted it and it has bound to a real aircraft: + # this is the population the live audit sampled (45% + # of published dark solves carried a foreign node), + # and unlike the candidate rate its denominator is + # not something association can inflate. + res.published_inputs += 1 + res.published_foreign_nodes += _foreign + if _foreign: + res.published_contaminated += 1 res.errors_km.append(d) res.n_nodes_matched[nn] += 1 # Broken out because the whole dual-site hypothesis is @@ -1153,6 +1364,7 @@ def _geo_key(nid): res.gate_accepted = assoc.track_pairs_accepted res.gate_unfitted = assoc.track_pairs_unfitted res.gate_superseded = assoc.track_pairs_superseded + res.cluster_splits = getattr(assoc, "cluster_splits", 0) res.claims_matched = assoc.claims_matched res.claim_conflicts = assoc.claim_conflicts res.anchored_inputs = assoc.anchored_inputs_emitted @@ -1329,6 +1541,21 @@ def report(label: str, r: Result, truth_max_kt: float | None = None): f" solves faster than any real aircraft ({truth_max_kt:.0f} kt): " f"{over} ({100 * over / len(r.speeds_kt):.0f}%)" ) + if r.inputs_scored: + print( + f" CONTAMINATION: {r.inputs_contaminated}/{r.inputs_scored} solver inputs carry a foreign node" + f" -> {r.contaminated_inputs_pct:5.1f}% " + f"foreign nodes/input {r.foreign_nodes_per_input:.2f}" + f" ({r.foreign_nodes}/{r.input_nodes} nodes)" + ) + if r.published_inputs: + print( + f" CONTAMINATION (published): {r.published_contaminated}/{r.published_inputs} matched solves" + f" -> {r.published_contaminated_pct:5.1f}% " + f"foreign nodes/solve {r.published_foreign_per_solve:.2f}" + ) + if r.cluster_splits: + print(f" cluster splits (same-node track conflict): {r.cluster_splits}") if r.gate_gated: print( f" CV gate: {r.gate_gated} pairings past the delay grid " @@ -1498,6 +1725,32 @@ def main(): help="track mode: disable one-to-one hypothesis selection " "(each pairing then answers only to the chi2 threshold)", ) + # Cluster-merge knobs (track mode). Each defaults to None, meaning "leave + # the library's own default alone", so the bench does not silently pin a + # value the library later changes — and so a sweep leg reads as exactly + # the deviation it is testing. + p.add_argument( + "--merge-dist-km", + type=float, + default=None, + help="track mode: how close two pairings must be to merge into one solver input (association._MERGE_DIST_KM)", + ) + p.add_argument( + "--pair-vel-exclusive", + choices=("on", "off"), + default=None, + help="track mode, deferred only: drop a pairing whose implied velocity " + "contradicts a better-scoring pairing that claims the same track", + ) + p.add_argument( + "--merge-vel-consistent", + choices=("on", "off"), + default=None, + help="track mode: require implied-velocity agreement, not just " + "proximity, before two pairings are merged into one cluster", + ) + p.add_argument("--pair-vel-dv-ms", type=float, default=None, help="velocity-conflict speed threshold (m/s)") + p.add_argument("--pair-vel-dtheta-deg", type=float, default=None, help="velocity-conflict heading threshold (deg)") p.add_argument("--min-aircraft", type=int, default=10, help="matches FLEET_AIRCRAFT lower bound") p.add_argument("--max-aircraft", type=int, default=20) p.add_argument("--metro-traffic-frac", type=float, default=0.85, help="matches FLEET_METRO_TRAFFIC_FRAC") @@ -1568,6 +1821,14 @@ def main(): ) args = p.parse_args() + cluster_opts = { + "merge_dist_km": args.merge_dist_km, + "pair_vel_exclusive": None if args.pair_vel_exclusive is None else args.pair_vel_exclusive == "on", + "merge_vel_consistent": None if args.merge_vel_consistent is None else args.merge_vel_consistent == "on", + "pair_vel_dv_ms": args.pair_vel_dv_ms, + "pair_vel_dtheta_deg": args.pair_vel_dtheta_deg, + } + # --ss-metrics auto/on/off resolution. "on" without stonesoup installed # is a hard error (the user explicitly asked for numbers this image # cannot produce); "auto" degrades quietly except for one notice line so @@ -1622,6 +1883,7 @@ def main(): + f", fov={args.fov}" + f", ss-metrics={'on' if ss_metrics_enabled else 'off'}" + (f", smoother-legs={','.join(lbl for lbl, _, _ in smoother_legs)}" if smoother_legs else "") + + "".join(f", {k.replace('_', '-')}={v}" for k, v in sorted(cluster_opts.items()) if v is not None) ) # chi2 only means anything in track mode; keep one pass otherwise. @@ -1632,6 +1894,7 @@ def main(): solve_fn = _ESTIMATORS[estimator_name] rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], [] n2_rates = [] + contam_rates, foreign_rates, med_errs, pub_contam_rates = [], [], [], [] agg = Result() last = None for k in range(args.repeat): @@ -1666,6 +1929,7 @@ def main(): ss_metric_dt=args.ss_metric_dt, ss_hold_s=args.ss_hold_s, smoother_legs=smoother_legs, + cluster_opts=cluster_opts, ) agg.merge(last, tag=f"s{args.seed + k}") # Track-level is the comparable metric — solve-level and @@ -1678,6 +1942,10 @@ def main(): fakes.append(len(last.ghost_tracks)) if last.speed_err_ms: speed_errs.append(statistics.median(last.speed_err_ms)) + contam_rates.append(last.contaminated_inputs_pct) + foreign_rates.append(last.foreign_nodes_per_input) + pub_contam_rates.append(last.published_contaminated_pct) + med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan")) label = f"assoc_interval={interval:g}s" if chi2_max is not None: label += f" chi2/dof<={chi2_max:g}" @@ -1700,6 +1968,27 @@ def main(): f"({', '.join(f'{x:.0f}%' for x in n2_rates)})" ) print(f" by solve: {', '.join(f'{x:.1f}%' for x in solve_rates)}") + if any(contam_rates): + print( + f" contaminated inputs per seed: " + f"{', '.join(f'{x:.0f}%' for x in contam_rates)}" + f" mean {statistics.mean(contam_rates):.1f}%" + ) + print( + f" foreign nodes/input per seed: " + f"{', '.join(f'{x:.2f}' for x in foreign_rates)}" + f" mean {statistics.mean(foreign_rates):.2f}" + ) + print( + f" published contaminated per seed: " + f"{', '.join(f'{x:.0f}%' for x in pub_contam_rates)}" + f" mean {statistics.mean(pub_contam_rates):.1f}%" + ) + print( + f" median matched error per seed: " + f"{', '.join(f'{x:.2f}' for x in med_errs)} km" + f" real tracks {', '.join(str(x) for x in reals)}" + ) if speed_errs: print( f" median speed error per seed: " diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..d534b74b 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -18,6 +18,7 @@ ARCHIVE_BATCH_MAX, ARCHIVE_FLUSH_INTERVAL_S, N2_TRACK_HISTORY_MAX, + TRACK_MAX_STALE_S, ) from core import state from pipeline.passive_radar import PassiveRadarPipeline @@ -156,6 +157,27 @@ def get_node_configs() -> dict[str, dict]: return configs +def configs_for_solver_input(node_cfgs: dict[str, dict], s_in: dict) -> dict[str, dict]: + """The subset of ``node_cfgs`` a solver input can actually reach. + + The solver runs in a *spawn* process pool, so everything queued with an + input is pickled and shipped to a child on every call — and the fleet is + 58 nodes while a candidate carries 2-8 measurements. Sending the whole + set meant ~50 configs per solve that no code path could look at. + + Nothing downstream needs the rest. The solver builds NodeSetups from the + measurements only; trimming and consensus both narrow that set further + (_filter_s_in_to_nodes) and never widen it; the beam gate iterates the + result's contributing_node_ids, which are measurement node ids by + construction; and cv_epochs is built from the same matched nodes as the + measurements in all three input shapes association emits. The known lane + is unaffected — it fetches its own configs (known_lane.run_known_lane_pass) + rather than reusing what was queued here. + """ + wanted = {m.get("node_id") for m in (s_in.get("measurements") or ())} + return {nid: cfg for nid, cfg in node_cfgs.items() if nid in wanted} + + # ── Per-node pipeline factory ───────────────────────────────────────────────── @@ -254,7 +276,11 @@ def _view_adsb_hex(track, hist) -> str | None: return hexn -def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> list[dict]: +def confirmed_track_views( + tracker, + history_n: int = N2_TRACK_HISTORY_MAX, + now_ts_ms: int | None = None, +) -> list[dict]: """A tracker's confirmed tracks, in the shape submit_tracks takes. TENTATIVE tracks are excluded, the same filter the arc builder applies: they @@ -264,9 +290,26 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis reason arcs keep it — at 22 fps a single missed frame flips ACTIVE → COASTING and the next flips it back. + But COASTING is kept only while its newest REAL detection is fresh. What + travels downstream is ``history[-1]``, and association hands that sample to + the solver as the node's current measurement — so a track coasting toward + its N_DELETE deletion point contributes a delay from wherever the aircraft + was several seconds ago. That is the out-of-cone node the rms trim then + has to discard (see TRACK_MAX_STALE_S). The staleness test reads + ``hist[-1]["timestamp"]`` rather than the track's coast count because + get_recent_detections returns only ASSOCIATED samples — mark_missed appends + None to ``history["measurements"]`` and the reverse scan skips those — so + that timestamp IS the last real detection's, exactly the honest signal, + while n_missed only counts frames the node happened to process. Compared + against *now_ts_ms*, the frame timestamp being processed, never wall clock: + the fleet replays and backfills, and a filter keyed on wall clock would + silently empty every view in those runs. Skipped when the caller supplies + no frame time, or when TRACK_MAX_STALE_S is 0. + Shared with scripts/association_bench.py (which carried a near-verbatim copy) so the bench feeds association exactly what production does. """ + max_stale_ms = TRACK_MAX_STALE_S * 1000.0 if now_ts_ms is not None else 0.0 views = [] for tr in tracker.tracks: if tr.state_status == TrackState.TENTATIVE: @@ -274,6 +317,9 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis hist = tr.get_recent_detections(history_n) if len(hist) < 2: continue + if max_stale_ms > 0 and (now_ts_ms - hist[-1]["timestamp"]) > max_stale_ms: + state.bump_counter("tracks_stale_skipped") + continue views.append( { "track_id": tr.id or f"tmp-{id(tr)}", @@ -292,8 +338,8 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis return views -def _node_track_views(pipeline: PassiveRadarPipeline) -> list[dict]: - return confirmed_track_views(pipeline.tracker) +def _node_track_views(pipeline: PassiveRadarPipeline, now_ts_ms: int | None = None) -> list[dict]: + return confirmed_track_views(pipeline.tracker, now_ts_ms=now_ts_ms) def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarPipeline): @@ -397,7 +443,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP # Track-level association. The detection-level path it replaced now lives # in retina_analytics.detection_association, reachable only from the # offline bench, which keeps it as the A/B baseline. - _track_views = _node_track_views(pipeline) + _track_views = _node_track_views(pipeline, _ts_ms_assoc or None) # Feed the per-node distinct-track counters — total_tracks / # geolocated_tracks were exported (and read by the admin API) but never # written anywhere. @@ -427,7 +473,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP if s_in["n_nodes"] < 2: continue try: - state.solver_queue.put_nowait((s_in, node_cfgs, time.time())) + state.solver_queue.put_nowait((s_in, configs_for_solver_input(node_cfgs, s_in), time.time())) except Exception: state.bump_counter("solver_queue_drops") if state.solver_queue_drops % 100 == 1: diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py index 6ffe29ad..8d335bbd 100644 --- a/backend/services/tasks/known_lane.py +++ b/backend/services/tasks/known_lane.py @@ -245,6 +245,12 @@ def _build_solver_input(hexn: str, claims: dict[str, dict]) -> dict | None: "delay_us": float(c["delay_us"]), "doppler_hz": float(c["doppler_hz"]), "snr": _num(c.get("snr")), + # This lane does NOT have one epoch: _CLAIM_SPREAD_S admits + # claims up to 5 s apart, which at 300 m/s is ~1.5 km of target + # motion charged straight to the residual this lane exists to + # measure. Carrying each claim's own capture time lets + # _attempt reuse the regular lane's epoch alignment. + "t_s": int(c["ts_ms"]) / 1000.0, } for nid, c in sorted(claims.items()) ], @@ -362,6 +368,15 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non record's displacement_km and the accuracy error are the same number. """ state.bump_counter("known_lane_attempts") + # Same correction, same flag, same helper as the regular lane — see + # solver.align_measurement_epochs. Applied here rather than in + # _build_solver_input because the alignment needs the node configs, and + # because the accuracy classification below compares the solve against an + # initial guess already dead-reckoned to the newest claim's epoch, which is + # exactly the t0 the helper aligns onto. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN: + s_in, epoch_meta = solver_mod.align_measurement_epochs(s_in, node_cfgs) try: # Single solve at the pinned ADS-B altitude — no layer sweep. The # sweep exists to DISCOVER an unknown altitude; here identity already @@ -379,7 +394,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non "known_no_converge", s_in, result if isinstance(result, dict) else None, - extra={"known_lane": True, "label": "no_converge", "published": False}, + extra={"known_lane": True, "label": "no_converge", "published": False, **epoch_meta}, ) return @@ -419,7 +434,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non raw_lat=raw_lat, raw_lon=raw_lon, displacement_km=err_km, - extra={"known_lane": True, "label": label, "published": published}, + extra={"known_lane": True, "label": label, "published": published, **epoch_meta}, ) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..0acd94b6 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -11,6 +11,8 @@ from collections import deque from concurrent.futures.process import BrokenProcessPool +from retina_analytics.association import _point_in_beam + from config.constants import ( ARC_ONLY_ANOMALY_ALLOWLIST, ASSOC_GRID_STEP_KM, @@ -100,6 +102,24 @@ def _pool_call(fn, *args): return fn(*args) +def _pool_solve_multistart(s_in, node_cfgs, alt_starts_km): + """solve_multinode_multistart via the process pool (inline when none). + + Defined here rather than beside _pool_solve_multinode at the foot of this + module for the reason _pool_select_consensus is: it is a default argument + value, resolved when the ``def`` executes, so it has to be bound before + _solve_best_altitude's signature is reached. + + A module-level function taking only picklable arguments, because the pool + is a *spawn* pool — a child imports retina_geolocator and nothing of the + backend, so what crosses is this function's qualified name plus the input + dicts. + """ + from retina_geolocator.multinode_solver import solve_multinode_multistart + + return _pool_call(solve_multinode_multistart, s_in, node_cfgs, alt_starts_km, True) + + # Altitude layers (km) tried when n_nodes ≥ 3. For an overdetermined system # (3+ delay equations, 2 unknowns after altitude pinning) only the correct # altitude layer yields rms_delay ≈ 0; wrong layers give rms > 0, so picking @@ -279,6 +299,91 @@ def _dark_displacement_cap_km() -> float: # It lived here while the frame path had its own, looser, unstated one. +# ── Measurement epoch alignment ────────────────────────────────────────────── +# The solver's residual model evaluates every measurement against ONE target +# state: the measurement set is assumed simultaneous. It is not. Each node +# samples on its own free-running cadence (~0.74-1 Hz on the fleet), so the +# delays in one solver input were captured at times spread over up to a frame +# interval, and association hands over each track's newest sample regardless of +# when that was. A 250 m/s target moves ~250 m per second of skew, which shows +# up as up to ~1 us of bistatic delay error per second — charged in full to the +# 3 us rms_delay gate, where it is indistinguishable from a contaminated node +# and drives the trim to throw away legitimately in-cone nodes. +# +# The correction is closed-form and needs nothing the measurement does not +# already carry. Writing d_tx / d_rx for the TX->target and target->RX ranges, +# the bistatic delay is (d_tx + d_rx - baseline)/c and the bistatic Doppler is +# (fc/c)(v_tx + v_rx), where v_tx / v_rx are the target's velocity components +# along the unit vectors pointing FROM the target TOWARD the TX and the RX +# (retina_geolocator.multinode_solver._residual_function; the simulator's +# _bistatic_delay / _bistatic_doppler in retina_simulation.world use the +# identical convention). Moving toward a site shortens that leg, so +# d(d_tx)/dt = -v_tx and d(d_rx)/dt = -v_rx, and therefore +# +# d(delay_us)/dt = -(v_tx + v_rx) / C_KM_US +# = -doppler_hz * (C_KM_S / fc_hz) / C_KM_US +# = -doppler_hz * 1e6 / fc_hz +# +# i.e. positive Doppler is a closing target and its delay is DECREASING. The +# unit test test_epoch_alignment.py checks the sign against a target flown +# through the simulator's own geometry helpers at two times, rather than +# against this derivation. +_DELAY_RATE_HZ_TO_US_PER_S = 1e6 + + +def align_measurement_epochs(s_in: dict, node_cfgs: dict) -> tuple[dict, dict]: + """Dead-reckon every measurement's delay onto the newest one's epoch. + + Pure: returns a new solver input (shallow copy, fresh measurement dicts) + and a metadata dict for the history record; *s_in* is never mutated, so a + caller can drop the result and keep the untouched input. + + Alignment is all-or-nothing per input. A partially aligned set is worse + than an unaligned one — the residual model has no way to know which + measurements share an epoch, so mixing corrected and uncorrected delays + just moves the error onto a different node. Any measurement missing t_s + or doppler_hz, or whose node has no config to read fc_hz from, therefore + skips the whole input and counts solver_epoch_align_skipped. + + Returns (s_in, meta) where meta carries epoch_aligned and, when it ran, + epoch_skew_s — the widest gap the correction closed. + """ + meas = s_in.get("measurements") or [] + if len(meas) < 2: + return s_in, {"epoch_aligned": False} + + rates = [] + for m in meas: + t_s = m.get("t_s") + doppler = m.get("doppler_hz") + cfg = node_cfgs.get(m.get("node_id")) or {} + # Same fallback chain the geolocator uses when it builds its NodeSetup, + # so a node whose config spells the carrier "FC" aligns on exactly the + # frequency the solve will predict against. + fc_hz = cfg.get("fc_hz", cfg.get("FC")) + if t_s is None or doppler is None or not fc_hz: + state.bump_counter("solver_epoch_align_skipped") + return s_in, {"epoch_aligned": False} + rates.append((float(t_s), -float(doppler) * _DELAY_RATE_HZ_TO_US_PER_S / float(fc_hz))) + + # The newest sample, not the input's timestamp_ms: t0 has to be a time some + # measurement was actually taken, or every delay is extrapolated and the + # freshest node — the one that needed no correction — acquires an error. + t0 = max(t for t, _ in rates) + skew_s = t0 - min(t for t, _ in rates) + + aligned = dict(s_in) + aligned["measurements"] = [ + {**m, "delay_us": float(m["delay_us"]) + rate * (t0 - t_s)} for m, (t_s, rate) in zip(meas, rates) + ] + if "timestamp_ms" in aligned: + # The set now describes t0, so everything downstream that ages this + # solve (multinode expiry, the dead-reckoning gates, the history + # record's measurement_ts_ms) should date it from t0 too. + aligned["timestamp_ms"] = int(round(t0 * 1000.0)) + return aligned, {"epoch_aligned": True, "epoch_skew_s": round(skew_s, 3)} + + def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[float], metric: str) -> dict | None: """Try each altitude layer; return the result with lowest value of `metric`. @@ -320,17 +425,82 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa return best_result -def _solve_best_altitude(s_in: dict, node_cfgs: dict, solve_fn) -> dict | None: - """Altitude sweep for n≥3: pick by minimum rms_delay. +# Fewest measurements the free mode is used at. Below this altitude is not +# observable and retina_geolocator pins it anyway; the sweep is left in place +# so the n=2 path keeps its documented behaviour exactly. +_FREE_ALT_MIN_NODES = 3 + + +def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]: + """The start altitudes the free mode hands the multi-start helper. + + state.SOLVER_FREE_ALT_STARTS of them, clamped into [1, len(layers)] — read + per call, like the mode flag, so a test and a config reload both see what + they set. One start is the layer nearest ``ig_alt_km``, which is the + altitude spliced into ``layers`` when the input carries a non-layer one of + its own (ADS-B), exactly as the sweep treats it. Several are a window + centred on that layer, clamped to the ends of the ladder so the count never + shrinks there (the top and bottom layers are where a wrong start is least + recoverable, not most). + + Freeing z removes the ladder's quantisation but not the LM's locality, and + the extra starts are what would stop a solve settling on the wrong side of + a bistatic ellipse. On this fleet's geometry they had almost nothing to + stop: over 1019 free-mode solves on test, three starts' rms_delay differed + by more than 0.1 µs in 13 of them, and the nearest-layer start was more + than 0.5 µs worse than the best in 2 — so the default is one start and the + other two are bought explicitly, by a deployment whose geometry shows it + needs them. See core/state.py for the numbers and the trade. + """ + if not layers: + return [] + n = max(1, min(int(state.SOLVER_FREE_ALT_STARTS), len(layers))) + alt = float(ig_alt_km) if ig_alt_km is not None else 7.0 + nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt)) + lo = max(0, min(nearest - (n - 1) // 2, len(layers) - n)) + return layers[lo : lo + n] + - If the initial_guess already carries an ADS-B altitude (not one of the fixed - grid layers), include it in the sweep so the correct exact altitude is tried. +def _solve_best_altitude( + s_in: dict, + node_cfgs: dict, + solve_fn, + multistart_fn=_pool_solve_multistart, +) -> dict | None: + """Altitude for n≥3, by whichever rule state.SOLVER_ALT_MODE names. + + sweep (default): solve once per layer, pick by minimum rms_delay. If the + initial_guess already carries an ADS-B altitude (not one of the fixed grid + layers), include it in the sweep so the correct exact altitude is tried. + + free: one call to the multi-start helper, which solves altitude as a sixth + unknown from _free_alt_starts. The sweep cannot do better than half its + 2 km layer spacing, and on noise-free replay of this fleet's geometry that + quantisation alone left rms_delay at a 1.76 µs median against the 3.0 µs + reject gate — spending most of the gate's budget on an altitude the + measurements themselves determine, and provoking _trim_and_resolve to drop + nodes that were never the problem. Costs one pool round trip per + candidate instead of six. + + The mode is read per call rather than captured at import, so a test (and a + live config reload) sees the value it set. Read here and not inside + _process_solver_item because _trim_and_resolve re-enters through this same + function: a trim must re-solve under the mode its first solve used, or the + residuals it is comparing are not the same quantity. """ ig_alt = s_in.get("initial_guess", {}).get("alt_km") if ig_alt is not None and ig_alt not in _SOLVER_ALT_LAYERS_KM: layers = sorted(set(_SOLVER_ALT_LAYERS_KM + [round(float(ig_alt), 3)])) else: layers = _SOLVER_ALT_LAYERS_KM + n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])}) + if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES: + # No fall back to the sweep when this returns None: a helper that got + # no solve out of its starts is reporting the same thing the sweep + # reports when every layer fails, and sweeping anyway would cost the + # six round trips this mode exists to avoid on exactly the candidates + # that are least likely to repay them. + return multistart_fn(s_in, node_cfgs, _free_alt_starts(ig_alt, layers)) return _sweep_altitudes(s_in, node_cfgs, solve_fn, layers, "rms_delay") @@ -386,6 +556,7 @@ def _trim_and_resolve( node_cfgs: dict, solve_fn, result: dict, + multistart_fn=_pool_solve_multistart, ) -> tuple[dict, dict, dict | None]: """Drop the worst-residual node(s) and re-solve, down to _TRIM_MIN_NODES. @@ -395,6 +566,11 @@ def _trim_and_resolve( so re-solving on the survivors after dropping the offending node recovers a solve the blanket gate would otherwise discard outright. + Re-solves through _solve_best_altitude, so it inherits whichever altitude + mode is in force — the loop compares this round's rms against the previous + round's, and mixing a swept altitude with a free one would make that + comparison meaningless. + Returns (final_result, final_s_in, trim_meta). trim_meta is None only when no round ever produced a successful re-solve — i.e. no trimming was actually performed — never when trimming ran but rms stayed high (that @@ -439,7 +615,7 @@ def _trim_and_resolve( s_next = _filter_s_in_to_nodes(s_in, survivors) try: - new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn) + new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn, multistart_fn) except Exception: logging.exception("Solver trim re-solve failed") break @@ -984,6 +1160,34 @@ def _supersession_match( # Sized against the map, not the association cadence: multinode_tracks expire # at 60 s, so refreshing an aircraft every 12 s leaves four solves' worth of # margin. 0 disables the suppression entirely. +# +# The claim is recorded ON PUBLICATION, not on admission, and from the +# POST-TRIM survivors. Claiming on admission made a candidate that never +# reached the map suppress every later candidate sharing any of its track ids +# for the full window — including other aircraft's, since tracker track ids +# are shared across the association candidates of different aircraft (74 of +# 178 ids in a 6 min live window appeared in solves of more than one +# ground-truth aircraft; the same finding that forced _supersession_match's +# spatial guard). A rejected candidate, or a contaminated superset that the +# gates sank, therefore blacked out the clean subsets behind it for 12 s and +# nothing was refreshed at all. Live that cost ~1 537 skips per 646 dark +# attempts per 30 min — more candidates suppressed than solved, by a factor +# of two. The rule this suppression is FOR is "an aircraft already on the map +# at this width does not need re-solving yet", and only a publication puts an +# aircraft on the map. +# +# Two consequences, both accepted deliberately: +# * the check no longer claims under the same lock, so two workers can now +# both solve duplicates of one aircraft that arrived together. The pair +# costs one extra solve and is resolved downstream by keying and +# supersession, which already handle exactly this; the alternative is the +# starvation above. +# * trimmed nodes' track ids are NOT claimed (_filter_s_in_to_nodes rebuilds +# track_ids from the surviving track_ids_by_node, so result's +# source_track_ids are the survivors). A node dropped for a bad residual +# was probably another aircraft's — claiming its track would suppress that +# aircraft's own candidate on the strength of a measurement this solve +# threw away. _SOLVER_RESOLVE_INTERVAL_S = float(os.getenv("SOLVER_RESOLVE_INTERVAL_S", "12")) _RECENT_SOLVES: dict[str, tuple[float, int]] = {} # track_id → (solved_at, n_nodes) _RECENT_SOLVES_LOCK = threading.Lock() @@ -1001,39 +1205,96 @@ def _sweep_recent_solves(now_s: float) -> None: del _RECENT_SOLVES[tid] -def _claim_resolve_slot(s_in, now_s: float) -> bool: - """False when this candidate re-solves tracks another candidate just took. +def _resolve_slot_covered(s_in, now_s: float) -> tuple[bool, list[dict]]: + """Is every track this candidate carries already ON THE MAP at this width? + + Pure: it reads the claims and mutates nothing, so a candidate that is + admitted here and then rejected by the gate stack leaves no trace. The + claim is made afterwards by _record_resolve_slot, from the publish path + only — see the block comment above for why, and for what the loss of + atomic test-and-claim costs. - Records the claim as a side effect, under one lock with the test, so two - workers cannot both admit the same aircraft's duplicates. An input with no - track provenance (detection-level, or an anchored input carrying none) is - always admitted — there is nothing to match it against. + Returns (covered, blocking). ``blocking`` is the claims that covered it, + for the skip record; it is empty whenever ``covered`` is False. An input + with no track provenance (detection-level, or an anchored input carrying + none) is never covered — there is nothing to match it against. """ if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not isinstance(s_in, dict): - return True + return False, [] track_ids = s_in.get("track_ids") if not track_ids: - return True + return False, [] n_nodes = int(s_in.get("n_nodes") or 0) cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + blocking: list[dict] = [] with _RECENT_SOLVES_LOCK: - covered = True for tid in track_ids: held = _RECENT_SOLVES.get(tid) if held is None or held[0] <= cutoff or held[1] < n_nodes: - covered = False - break - if covered: - return False + return False, [] + blocking.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) + return True, blocking + + +def _record_resolve_slot(track_ids, n_nodes: int, now_s: float) -> None: + """Record that ``track_ids`` are covered by a PUBLISHED solve at n_nodes. + + Called from the publish path alone, with the post-trim survivors + (``result["source_track_ids"]``). Nothing else may call it: a claim is a + statement that this aircraft is on the map, and a rejected solve puts + nothing there. + """ + if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not track_ids: + return + n_nodes = int(n_nodes or 0) + cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + with _RECENT_SOLVES_LOCK: for tid in track_ids: held = _RECENT_SOLVES.get(tid) - # Keep the widest claim of the window: a narrow candidate admitted - # after a wide one must not lower the bar the next copy is tested - # against. + # Keep the widest claim of the window: a narrow publish after a + # wide one must not lower the bar the next copy is tested against. held_nodes = held[1] if held is not None and held[0] > cutoff else 0 _RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes)) _sweep_recent_solves(now_s) - return True + + +def _record_resolve_skip(s_in, now_s: float, blocking: list[dict]) -> None: + """Count and remember one resolve-slot refusal. + + The counter alone could not answer the question the suppression rule + raises — *whose* claim blocked this, and was it even the same aircraft. + Live on the test droplet the rule refuses ~1 537 candidates per 646 dark + attempts per 30 min, and nothing recorded which claim did it, so a skip + that suppressed a genuinely different aircraft (tracker track ids are + shared across candidates — see _supersession_match) was indistinguishable + from one that suppressed a duplicate. The deque carries the blocking + claims and the candidate's own guess position so the two can be told apart + after the fact. + + Deliberately NOT a solve-history record: skips outrun real dark records + roughly two to one, and writing them into that deque would evict the + solves the same investigation needs (see state.solver_resolve_skips_recent). + """ + s = s_in if isinstance(s_in, dict) else {} + track_ids = list(s.get("track_ids") or []) + dark = _is_dark_solver_input(s) + state.bump_counter("solver_resolve_skips") + if dark: + state.bump_counter("solver_resolve_skips_dark") + ig = s.get("initial_guess") or {} + state.solver_resolve_skips_recent.append( + { + "ts_ms": int(now_s * 1000), + # No key is minted for a candidate that never solves, so lane is + # the same fallback routes.test._record_lane uses for a reject. + "lane": "dark" if dark else "adsb", + "track_ids": track_ids, + "n_nodes": int(s.get("n_nodes") or 0), + "blocking": blocking, + "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None, + "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None, + } + ) # Which single-node track pair currently owns a published n=2 track, and how @@ -1384,6 +1645,59 @@ def _is_dark_solver_input(s_in) -> bool: return not (hx and is_transponder_hex(hx)) +def _stamp_foreign_nodes(rec: dict) -> None: + """Stamp which of a dark record's own nodes could not see the aircraft. + + Cluster contamination is the dark lane's largest known defect — a + candidate assembled by format_track_pairs_for_solver can carry a node + whose track belongs to a *different* aircraft, and the solver then fits a + geometry no single aircraft ever occupied. Offline the audit measured it + at ~60 % of dark candidates; this makes the same number live. + + The test is the associator's own visibility predicate applied whole + (retina_analytics.association._point_in_beam against the registered + NodeGeometry), which is the same gate known-lane claiming uses — claiming + and the dark lane must mean the same thing by "this node can see there", + and a second bespoke rule here would let the two disagree. Two + consequences worth knowing: it is a ground-projected bearing/footprint + test with no altitude term, and under FOV_MODE=active it is the learned + FOV rather than the theoretical wedge. Both are exactly what the rest of + the pipeline believes about coverage, which is the point. + + Position is the matched ground-truth point already stamped on the record + (gt_lat/gt_lon at the solve epoch), so this costs no extra trail lookup — + only one cone test per contributing node. Nodes trimmed out by + _trim_and_resolve are included: a node dropped for a bad residual is + precisely the contamination this measures, and leaving it out would hide + every case trimming already rescued. + + A node with no registered geometry is not judged either way. When that + leaves nothing judgeable the record is left unstamped rather than stamped + clean, so contamination_pct never counts an abstention as innocence. + """ + lat, lon = rec.get("gt_lat"), rec.get("gt_lon") + if lat is None or lon is None: + return + node_ids = list(rec.get("contributing_node_ids") or []) + node_ids += [nid for nid in (rec.get("trimmed_node_ids") or []) if nid not in node_ids] + if not node_ids: + return + geometries = state.node_associator.node_geometries + judged = 0 + foreign: list[str] = [] + for nid in node_ids: + geo = geometries.get(nid) + if geo is None: + continue + judged += 1 + if not _point_in_beam(lat, lon, geo): + foreign.append(nid) + if not judged: + return + rec["foreign_node_ids"] = foreign + rec["contaminated"] = bool(foreign) + + def _record_dark_accuracy_sample(rec: dict) -> None: """Offer one published DARK solve to the rolling accuracy store. @@ -1460,7 +1774,14 @@ def _record_solve_history( ``extra`` merges caller-supplied fields (trim metadata, beam-rejection diagnostics) into the record. Applied before the GT stamp so it can - never clobber gt_hex/gt_error_km/gt_lat/gt_lon. + never clobber gt_hex/gt_error_km/gt_lat/gt_lon — and so the trimmed node + ids it carries are in hand for the contamination stamp below. + + ``foreign_node_ids``/``contaminated`` are stamped on DARK records that + matched ground truth: which of this candidate's own nodes could not see + the aircraft it was matched to (see _stamp_foreign_nodes). Absent on + every other record, which is what /api/test/solver-stats' contamination + block counts as "not judged" rather than as clean. """ r = result if isinstance(result, dict) else {} s = s_in if isinstance(s_in, dict) else {} @@ -1574,6 +1895,11 @@ def _record_solve_history( rec["vel_err_ms"] = round(math.hypot(ve - gt_ve, vn - gt_vn), 1) else: rec["vel_err_ms"] = None + # Live cluster-contamination metric, dark lane only and only where ground + # truth actually matched — without a truth position there is nothing to + # ask "could this node see it?" about. See _stamp_foreign_nodes. + if _dark and rec.get("gt_hex"): + _stamp_foreign_nodes(rec) if rec["outcome"] == "published" and _dark and rec.get("gt_error_km") is not None: _record_dark_accuracy_sample(rec) # Route by lane: the known lane's per-hex-per-pass volume would otherwise @@ -1693,7 +2019,12 @@ def fov_gate_verdict(fov, n_nodes: int, brg: float, dist_km: float, range_rule_p return range_rule_pass or fov_pass -def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus) -> dict | None: +def _process_solver_item( + item: tuple, + solve_fn, + select_fn=_pool_select_consensus, + multistart_fn=_pool_solve_multistart, +) -> dict | None: """Process a single solver queue entry. Returns the solver result (or None). Extracted from the worker loop so the success/failure/latency bookkeeping @@ -1704,6 +2035,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus initial_guess and _CONSENSUS_MODE != "off" — n=2 (mirror-disambiguation is the displacement/beam gates' job, not consensus's) and detection-level inputs (no initial_guess to pin an altitude with) never call it. + + multistart_fn is the free-altitude solve (_pool_solve_multistart by + default; tests substitute a stub), reached only when + state.SOLVER_ALT_MODE is "free" — see _solve_best_altitude. """ s_in, node_cfgs = item[0], item[1] enqueued_at: float | None = item[2] if len(item) > 2 else None @@ -1724,10 +2059,19 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # here rather than at enqueue: the frame path must not carry solver state, # and a copy that queued before its twin was solved can only be recognised # once it reaches a worker. - if not _claim_resolve_slot(s_in, time.time()): - state.bump_counter("solver_resolve_skips") + _now_s = time.time() + _covered, _blocking = _resolve_slot_covered(s_in, _now_s) + if _covered: + _record_resolve_skip(s_in, _now_s, _blocking) return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 + # Before anything reads a delay: the nodes did not sample simultaneously, + # and every gate below (rms_delay first among them) assumes they did. Runs + # ahead of consensus and the altitude sweep so both judge the same aligned + # numbers the published solve is fitted to. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN and isinstance(s_in, dict): + s_in, epoch_meta = align_measurement_epochs(s_in, node_cfgs) consensus_meta: dict | None = None try: if "initial_guess" not in s_in: @@ -1736,7 +2080,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus if _CONSENSUS_MODE != "off": s_in, consensus_meta = _consensus_select(s_in, node_cfgs, select_fn) n_nodes = s_in.get("n_nodes", n_nodes) - result = _solve_best_altitude(s_in, node_cfgs, solve_fn) + result = _solve_best_altitude(s_in, node_cfgs, solve_fn, multistart_fn) else: result = _solve_best_altitude_n2(s_in, node_cfgs, solve_fn) except Exception: @@ -1762,7 +2106,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus and (result.get("rms_delay") or 0) > _SOLVER_RMS_DELAY_MAX_US and result.get("per_node_delay_res_us") ): - result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result) + result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result, multistart_fn) n_nodes = result.get("n_nodes", n_nodes) # Built once and threaded through every history record below @@ -1771,6 +2115,24 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus _extra: dict | None = dict(trim_meta) if trim_meta else {} if consensus_meta is not None: _extra["consensus_meta"] = consensus_meta + # Always stamped, aligned or not: "this solve was not aligned" is the + # fact /api/test/mlat-history needs to separate a residual the + # correction could not have helped from one it was applied to. + _extra.update(epoch_meta) + # How this solve got its altitude, and — in free mode — what each + # start altitude fitted to. Stamped on every record, published or + # rejected, and in BOTH modes (the sweep's solves report + # altitude_mode "pinned"), because the only way to judge SOLVER_ALT_MODE + # live is to compare the two lanes' rms_delay and gt_error_km over the + # same history buffer. The per-start list is what says whether the + # three starts were worth keeping or one would have done. + if result.get("altitude_mode"): + _extra["altitude_mode"] = result["altitude_mode"] + if result.get("rms_by_start") is not None: + _extra["alt_starts_km"] = result.get("alt_starts_km") + _extra["alt_start_rms_us"] = [None if v is None else round(float(v), 3) for v in result["rms_by_start"]] + if result.get("z_saturated"): + _extra["z_saturated"] = True _extra = _extra or None rms_delay = result.get("rms_delay", 0) or 0 @@ -2291,6 +2653,12 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus archive_record = dict(result) archive_record["solve_ts_ms"] = int(time.time() * 1000) state.track_archive_buffer.append(archive_record) + # The re-solve claim, taken here and nowhere else: this aircraft is now + # on the map at this width, which is the only thing that makes a + # duplicate not worth solving. Survivors only — source_track_ids is + # rebuilt from the post-trim node set. Outside _MN_TRACKS_LOCK on + # purpose, so _RECENT_SOLVES_LOCK is never nested inside it. + _record_resolve_slot(result.get("source_track_ids"), result.get("n_nodes"), time.time()) _record_solve_history( "published", s_in, diff --git a/backend/services/tcp_handler.py b/backend/services/tcp_handler.py index bbe82197..6b21be77 100644 --- a/backend/services/tcp_handler.py +++ b/backend/services/tcp_handler.py @@ -529,6 +529,12 @@ def _enqueue_detection(msg: dict, node_id: str | None): if node_id: last = _per_node_last_enqueue.get(node_id, 0.0) if (now_m - last) < _NODE_MIN_INTERVAL_S: + # Counted, not silent: this is the only place a node's detections + # are discarded on purpose, and until now nothing said how many. + # state.frames_dropped is the queue-saturation counter and reads + # zero throughout, so "the tracker sees every frame this node + # sent" looked true from every published metric. + state.bump_counter("node_frames_rate_limited") return # position already updated; skip expensive queue work _per_node_last_enqueue[node_id] = now_m diff --git a/backend/tests/test_adsb_seed_backend.py b/backend/tests/test_adsb_seed_backend.py index cba3bb36..77b3fc86 100644 --- a/backend/tests/test_adsb_seed_backend.py +++ b/backend/tests/test_adsb_seed_backend.py @@ -605,3 +605,32 @@ def test_associator_gets_the_state_world_resolver(self): must consult the same resolver claiming and the auto-tag filter use, or one consumer accepts what another rejects.""" assert state.node_associator.node_world_provider is state.node_world + + def test_a_sim_and_a_real_node_over_one_footprint_get_no_overlap_zone(self): + """The same resolver, one level down: bottom-up pairing must not build + a grid across worlds either. Registering a synthetic node and a + hardware node on overlapping coverage used to leave a zone whose only + possible pairing was a simulated echo against a real one — which is how + real node ids reached the synthetic fleet's dark solves.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("hw-9001", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones == {} + assert _a._neighbors.get("synth-GVL-9001", set()) == set() + assert _a.assoc_world_skipped_pairs == 1 + finally: + state._reset_for_tests() + + def test_two_synthetic_nodes_over_one_footprint_still_pair(self): + """The gate is the world difference, not the registration.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("synth-GVL-9002", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones + assert _a.assoc_world_skipped_pairs == 0 + finally: + state._reset_for_tests() diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py index 22a821e0..2b912d03 100644 --- a/backend/tests/test_analytics_routes.py +++ b/backend/tests/test_analytics_routes.py @@ -129,6 +129,47 @@ def test_status_returns_expected_fields(self, client): assert "overlap_zones" in body assert "overlaps" in body + def test_track_pairs_block_reports_the_live_counters(self, client): + """superseded and cluster_splits belong to the LIVE block, not the + inline-only one. + + They used to be structurally zero in production because the only + exclusivity stage ran on a chi2 nothing computes with cv_fit=None. + The deferred path now prunes on implied-velocity conflict and splits + clusters that hold two tracks of one node, so both counters move on a + live fleet and reading them as "inline only" would be wrong. + """ + _a = state.node_associator + _a.track_pairs_superseded += 5 + _a.cluster_splits += 3 + try: + body = client.get("/api/radar/association/status").json() + assert body["track_pairs"].keys() == { + "gated", + "unfitted", + "deferred", + "superseded", + "cluster_splits", + } + assert body["track_pairs"]["superseded"] == 5 + assert body["track_pairs"]["cluster_splits"] == 3 + assert body["track_pairs_inline_only"].keys() == {"accepted", "rejected"} + finally: + _a.track_pairs_superseded -= 5 + _a.cluster_splits -= 3 + + def test_status_reports_world_skipped_pairs(self, client): + """The world gate on overlap zones is otherwise invisible: a fleet + whose sim/real pairs are being refused looks exactly like a fleet whose + pairs never overlapped, and only this counter separates them.""" + _a = state.node_associator + _a.assoc_world_skipped_pairs += 7 + try: + body = client.get("/api/radar/association/status").json() + assert body["assoc_world_skipped_pairs"] == 7 + finally: + state._reset_for_tests() + def test_status_includes_claiming_block(self, client): """Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default in tests, so this pins the shape rather than any particular mode.""" diff --git a/backend/tests/test_epoch_alignment.py b/backend/tests/test_epoch_alignment.py new file mode 100644 index 00000000..a805dc22 --- /dev/null +++ b/backend/tests/test_epoch_alignment.py @@ -0,0 +1,274 @@ +"""Measurement epoch alignment (SOLVER_EPOCH_ALIGN) — solver.align_measurement_epochs. + +The solver's residual model evaluates every measurement against ONE target +state, so a solver input is implicitly a claim that its measurements were +simultaneous. Nodes sample on independent free-running cadences, so the claim +is false by up to a frame interval, and the resulting delay error is charged to +the rms_delay gate. These tests pin the correction, and — more importantly — +pin its SIGN against the simulator's own geometry rather than against the +derivation the helper's comment gives, since a sign error there would silently +double the very error the correction exists to remove. +""" + +import pytest +from retina_simulation.world import _bistatic_delay, _bistatic_doppler + +from core import state +from services.tasks import solver as solver_mod +from services.tasks.solver import align_measurement_epochs + +_FC_HZ = 183e6 + +# One node's ENU geometry, km. Only fc_hz is read by the helper; the rest is +# here because the simulator's delay/Doppler helpers need a real bistatic +# triangle to produce numbers whose sign means anything. +_TX_ENU = (-20.0, 5.0, 0.05) +_RX_ENU = (0.0, 0.0, 0.3) + +_NODE_CFGS = { + "node-a": {"fc_hz": _FC_HZ}, + "node-b": {"fc_hz": _FC_HZ}, + "node-c": {"FC": _FC_HZ}, # the alternate spelling the geolocator accepts +} + + +def _s_in(measurements, **over): + base = { + "initial_guess": {"lat": 34.85, "lon": -82.4, "alt_km": 9.0}, + "measurements": measurements, + "n_nodes": len({m["node_id"] for m in measurements}), + "timestamp_ms": 1_700_000_000_000, + } + base.update(over) + return base + + +def _m(node_id, delay_us, doppler_hz, t_s, snr=15.0): + return { + "node_id": node_id, + "delay_us": delay_us, + "doppler_hz": doppler_hz, + "snr": snr, + "t_s": t_s, + } + + +@pytest.fixture(autouse=True) +def _zero_counter(): + state.solver_epoch_align_skipped = 0 + yield + + +class TestPureHelper: + def test_newest_measurement_is_the_epoch_and_is_untouched(self): + """t0 is the newest SAMPLE time, not the input's timestamp_ms: the + freshest node needed no correction and must not acquire one.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-c"]["delay_us"] == 60.0 + assert meta["epoch_aligned"] is True + assert meta["epoch_skew_s"] == pytest.approx(2.0) + + def test_each_delay_moves_by_its_own_doppler_rate(self): + """d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz, applied over that + measurement's own gap to t0.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-a"]["delay_us"] == pytest.approx(40.0 + (-100.0 * 1e6 / _FC_HZ) * 2.0) + assert by_id["node-b"]["delay_us"] == pytest.approx(50.0 + (80.0 * 1e6 / _FC_HZ) * 0.5) + + def test_zero_doppler_measurement_is_unchanged(self): + """A tangential target's bistatic range is stationary, so no amount of + skew moves its delay — the rate is the only thing that can.""" + s_in = _s_in([_m("node-a", 40.0, 0.0, 1000.0), _m("node-b", 50.0, 20.0, 1004.0)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["measurements"][0]["delay_us"] == 40.0 + + def test_input_is_not_mutated(self): + """Pure: a caller must be able to drop the result and keep the + original, which is exactly what the flag-off path does.""" + meas = [_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.0)] + s_in = _s_in(meas) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert s_in["measurements"][0]["delay_us"] == 40.0 + assert s_in["measurements"] is not out["measurements"] + assert s_in["timestamp_ms"] == 1_700_000_000_000 + + def test_timestamp_ms_is_restamped_to_the_epoch(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.25)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["timestamp_ms"] == 1_002_250 + + def test_fc_spelled_FC_is_accepted(self): + """Same fallback chain the geolocator uses to build its NodeSetup, so + a node aligns on exactly the carrier its solve predicts against.""" + s_in = _s_in([_m("node-c", 40.0, 100.0, 1000.0), _m("node-b", 50.0, 0.0, 1001.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + assert out["measurements"][0]["delay_us"] == pytest.approx(40.0 - 100.0 * 1e6 / _FC_HZ) + + def test_single_measurement_input_is_a_no_op(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 0 + + +class TestSkipPath: + @pytest.mark.parametrize( + "broken", + [ + {"t_s": None}, + {"doppler_hz": None}, + ], + ) + def test_missing_field_skips_the_whole_input(self, broken): + """All-or-nothing: a partially aligned set has no marker saying which + measurements share an epoch, so it just relocates the error.""" + good = _m("node-a", 40.0, 100.0, 1000.0) + bad = {**_m("node-b", 50.0, -80.0, 1002.0), **broken} + s_in = _s_in([good, bad]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 1 + + def test_missing_t_s_key_entirely_skips(self): + """The pre-upgrade measurement shape: no t_s key at all.""" + untimed = {"node_id": "node-b", "delay_us": 50.0, "doppler_hz": -80.0, "snr": 9.0} + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), untimed]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + def test_unknown_node_config_skips(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-zzz", 50.0, -80.0, 1002.0)]) + out, meta = align_measurement_epochs(s_in, {"node-a": {"fc_hz": _FC_HZ}}) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + +class TestSignAgainstSimulatorGeometry: + """The sign check, run against the simulator's own delay/Doppler model. + + A target is flown in a straight line and sampled at two times using + _bistatic_delay / _bistatic_doppler. The older sample plus the correction + must land on the newer sample's true delay — which is a statement about the + sign of the Doppler-to-delay-rate conversion that no amount of algebra in a + comment can substitute for. + """ + + _POS0 = (10.0, 15.0, 9.0) # km ENU + _DT_S = 2.0 + + @staticmethod + def _truth(vel_kms, dt_s): + pos0 = TestSignAgainstSimulatorGeometry._POS0 + pos1 = tuple(pos0[i] + vel_kms[i] * dt_s for i in range(3)) + return ( + _bistatic_delay(pos0, _TX_ENU, _RX_ENU), + _bistatic_delay(pos1, _TX_ENU, _RX_ENU), + _bistatic_doppler(pos0, vel_kms, _TX_ENU, _RX_ENU, _FC_HZ), + ) + + @pytest.mark.parametrize( + "vel_kms", + [ + (-0.20, -0.15, 0.0), # inbound: bistatic range shrinking + (0.20, 0.15, 0.0), # outbound: bistatic range growing + (0.05, -0.24, 0.01), # mostly crossing, with a climb + ], + ) + def test_alignment_moves_the_stale_delay_toward_the_truth(self, vel_kms): + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + + # node-a sampled _DT_S seconds ago; node-b is the newest sample and + # therefore defines t0. node-b's own numbers are irrelevant to the + # assertion — it is only here to set the epoch. + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + aligned = out["measurements"][0]["delay_us"] + + err_before = abs(delay0 - delay1) + err_after = abs(aligned - delay1) + # A wrong sign would double err_before rather than shrink it, so the + # margin here is the sign test. Over 2 s of straight-line flight the + # first-order term dominates; the residual is the trajectory's + # curvature in bistatic range, not a modelling disagreement. + assert err_after < err_before * 0.2 + assert err_before > 0.05 # the case would prove nothing otherwise + + def test_correction_and_truth_share_a_sign(self): + """Stated directly, so a failure says 'the sign is wrong' rather than + 'the error did not shrink enough'.""" + vel_kms = (-0.20, -0.15, 0.0) + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + correction = out["measurements"][0]["delay_us"] - delay0 + assert correction * (delay1 - delay0) > 0 + + +class TestProcessSolverItemWiring: + """The flag, and that the aligned numbers are what the solve actually sees.""" + + @staticmethod + def _item(): + s_in = _s_in( + [ + _m("node-a", 40.0, 300.0, 1000.0), + _m("node-b", 50.0, 0.0, 1004.0), + ] + ) + return (s_in, _NODE_CFGS, None) + + def test_flag_off_leaves_the_input_untouched(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", False) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"] == [40.0, 50.0] + + def test_flag_on_hands_the_solver_aligned_delays(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", True) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"][0] == pytest.approx(40.0 + (-300.0 * 1e6 / _FC_HZ) * 4.0) + assert seen["delays"][1] == 50.0 diff --git a/backend/tests/test_frame_processor.py b/backend/tests/test_frame_processor.py index c5d31e92..56a1409a 100644 --- a/backend/tests/test_frame_processor.py +++ b/backend/tests/test_frame_processor.py @@ -6,15 +6,19 @@ import queue import time +import types import pytest +from retina_tracker.track import TrackState from config.constants import GT_DISPLAY_STALE_S from core import state from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline +from services import frame_processor from services.frame_processor import ( append_track_history, build_combined_aircraft_json, + confirmed_track_views, dedup_aircraft, flush_all_archive_buffers, get_node_configs, @@ -277,6 +281,95 @@ def test_claiming_anchored_inputs_reach_the_solver_queue(self, monkeypatch): # ── Multinode result conversion ────────────────────────────────────────────── +class TestConfirmedTrackViewsStaleness: + """TRACK_MAX_STALE_S: a coasting track stops being offered to association + once its newest REAL detection has aged out. + + The freshness signal is the newest entry get_recent_detections returns, + which by construction is an ASSOCIATED sample (mark_missed appends None to + history["measurements"] and the reverse scan skips those) — so these fakes + hand back only real detections, exactly as the tracker does, and the coast + is expressed as a gap between that newest sample and the frame time. + """ + + @staticmethod + def _track(newest_ts_ms: int, status=TrackState.COASTING, track_id="trk-stale"): + hist = [ + {"timestamp": newest_ts_ms - 1000, "delay": 40.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + {"timestamp": newest_ts_ms, "delay": 41.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + ] + return types.SimpleNamespace( + id=track_id, + state_status=status, + adsb_hex=None, + get_recent_detections=lambda n: hist[-n:], + ) + + def _tracker(self, *tracks): + return types.SimpleNamespace(tracks=list(tracks)) + + def test_five_second_old_coasting_track_is_excluded_at_three(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + assert state.tracks_stale_skipped == 1 + + def test_same_track_is_included_at_ten(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 10.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["trk-stale"] + assert state.tracks_stale_skipped == 0 + + def test_zero_disables_the_filter(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 0.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 600_000)) + assert len(confirmed_track_views(tracker, now_ts_ms=now_ms)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_no_frame_time_disables_the_filter(self, monkeypatch): + """The bench and the ADS-B-seeding tests call without a frame time; + wall clock is not a substitute, so those callers stay unfiltered.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + tracker = self._tracker(self._track(0)) + assert len(confirmed_track_views(tracker)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_fresh_track_survives_beside_a_stale_one(self, monkeypatch): + """Staleness is per track, not per tracker — the node keeps + contributing whatever it can still actually see.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker( + self._track(now_ms - 5000, track_id="gone"), + self._track(now_ms - 500, track_id="here"), + ) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["here"] + assert state.tracks_stale_skipped == 1 + + def test_tentative_is_still_excluded_regardless_of_freshness(self, monkeypatch): + """The TENTATIVE filter is unchanged and independent: a brand-new + TENTATIVE track's newest detection is as fresh as it gets, and it must + still not reach association.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms, status=TrackState.TENTATIVE)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + # Skipped as TENTATIVE, not as stale — the counter must stay clean so + # it only ever means "an aircraft left this node's cone". + assert state.tracks_stale_skipped == 0 + + class TestMultinodeToAircraft: def test_basic_conversion(self): r = { diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index e03d9da4..a406d1b4 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -777,3 +777,292 @@ def test_known_lane_records_are_not_sampled_by_this_path(self): extra={"known_lane": True, "label": "truth_match", "published": True}, ) assert not state.accuracy_samples + + +def _register_geo(node_id, beam_azimuth_deg, rx_lat=LAT, rx_lon=LON, max_range_km=50.0): + """Register one node geometry with the associator, aimed as given. + + The contamination stamp asks the associator's own visibility predicate, + so a test node has to exist there rather than in a config dict. + """ + from retina_analytics.association import NodeGeometry + + geo = NodeGeometry( + node_id=node_id, + rx_lat=rx_lat, + rx_lon=rx_lon, + rx_alt_km=0.0, + tx_lat=rx_lat + 0.5, + tx_lon=rx_lon + 0.5, + tx_alt_km=0.3, + beam_azimuth_deg=beam_azimuth_deg, + beam_width_deg=41.0, + max_range_km=max_range_km, + ) + state.node_associator.node_geometries[node_id] = geo + return geo + + +class TestForeignNodeStamp: + """A dark record matched to ground truth says which of its own nodes + could not have seen that aircraft. + + Cluster contamination — a solver candidate assembled from tracks of two + different aircraft — is the dark lane's largest known defect, and until + now it was measurable only offline. The verdict is the associator's own + visibility predicate, the same one known-lane claiming gates on. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _run(self, contributing=("n_in", "n_out"), **extra): + return solver_mod._process_solver_item( + (dict(_CONFIRMED_N2), {}, time.time()), + _solve_fn(contributing_node_ids=list(contributing), **extra), + ) + + def test_a_node_aimed_away_is_named_foreign(self): + # Ground truth sits due north of both nodes; n_in is aimed at it and + # n_out at the opposite bearing. + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert rec["foreign_node_ids"] == ["n_out"] + assert rec["contaminated"] is True + + def test_all_nodes_in_cone_is_not_contaminated(self): + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=10.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["foreign_node_ids"] == [] + assert rec["contaminated"] is False + + def test_a_node_out_of_range_is_foreign(self): + """Range, not only bearing: the predicate applies whole.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=0.0, max_range_km=1.0) + self._run() + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_out"] + + def test_trimmed_nodes_are_judged_too(self): + """A node dropped by _trim_and_resolve is exactly the contamination + this measures — excluding it would hide every case trimming already + rescued.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_trimmed", beam_azimuth_deg=180.0) + solver_mod._record_solve_history( + "published", + dict(_CONFIRMED_N2), + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2, "contributing_node_ids": ["n_in"]}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + extra={"trimmed_node_ids": ["n_trimmed"], "trim_rounds": 1}, + ) + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_trimmed"] + + def test_no_ground_truth_means_no_stamp(self): + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert "foreign_node_ids" not in rec + assert "contaminated" not in rec + + def test_unregistered_nodes_are_not_stamped_clean(self): + """Nothing judgeable is an abstention, not innocence.""" + _put_gt(lat=LAT + 0.05, lon=LON) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert "foreign_node_ids" not in rec + + def test_an_adsb_record_is_not_stamped(self): + """Dark lane only — the tagged lane's identity is not in doubt.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + s_in = dict(_CONFIRMED_N2, adsb_hex="abc123") + solver_mod._process_solver_item( + (s_in, {}, time.time()), + _solve_fn(contributing_node_ids=["n_in", "n_out"]), + ) + assert "foreign_node_ids" not in state.mlat_solve_history[0] + + +class TestLaneFilterAndPerLaneCap: + """?lane= and ?limit= on /api/test/mlat-history. + + The flat records[:1000] cap made the response a race between lanes: the + known lane writes ~16x the dark lane's volume, so a 30 min request held + only the newest ~6 min of dark records and the rest of the window read as + a quiet period. The cap is now per lane. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _dark(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + ) + + def _known(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "known_truth_match", + {"n_nodes": 2, "adsb_hex": "abc123", "initial_guess": {"lat": LAT, "lon": LON}}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2}, + extra={"known_lane": True, "label": "truth_match", "published": False}, + ) + + def _adsb(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3, "adsb_hex": "abc123"}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-adsb-abc123", + raw_lat=LAT, + raw_lon=LON, + ) + + def test_default_lane_is_all_and_counts_every_lane(self): + self._dark() + self._known() + self._adsb() + data = self._client().get("/api/test/mlat-history?all=1").json() + assert data["lane"] == "all" + assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1} + assert data["n_records"] == 3 + + def test_lane_dark_returns_only_dark_records(self): + self._dark(2) + self._known(3) + self._adsb(1) + data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json() + assert data["n_records"] == 2 + assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0} + assert all(r["solve_key"] == "mn-dark-1" for r in data["records"]) + + def test_lane_known_returns_only_known_records(self): + self._dark(2) + self._known(3) + data = self._client().get("/api/test/mlat-history?all=1&lane=known").json() + assert data["n_records"] == 3 + assert all(r["known_lane"] for r in data["records"]) + + def test_unknown_lane_is_rejected(self): + assert self._client().get("/api/test/mlat-history?all=1&lane=bogus").status_code == 400 + + def test_known_volume_cannot_evict_dark_records_from_the_response(self): + """The bug the per-lane cap fixes, at 1/500 scale.""" + self._dark(2) + self._known(20) + data = self._client().get("/api/test/mlat-history?all=1&limit=2").json() + # 2 dark + 2 known survive the cap; the flat cap would have returned + # the 2 newest records overall, both known. + lanes = [("known" if r.get("known_lane") else "dark") for r in data["records"]] + assert sorted(lanes) == ["dark", "dark", "known", "known"] + # n_records / lane_counts stay pre-cap so truncation is legible. + assert data["n_records"] == 22 + assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0} + + def test_limit_is_clamped_to_the_maximum(self): + self._dark(3) + data = self._client().get("/api/test/mlat-history?all=1&limit=99999").json() + assert len(data["records"]) == 3 + + def test_hex_lookup_reports_the_lane_block_too(self): + self._dark() + rec = state.mlat_solve_history[0] + data = self._client().get(f"/api/test/mlat-history?hex={rec['solver_hex']}").json() + assert data["lane"] == "all" + assert data["lane_counts"]["dark"] == 1 + + +class TestResolveSkipDump: + """?kind=resolve_skips dumps the solver's skip deque. + + A skip is not a solve outcome and must not be written into the + solve-history deques: on the live fleet skips outrun dark records roughly + two to one and would evict exactly the records an investigation needs. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in): + now = time.time() + s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in) + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s), now, blocking) + + def test_skip_records_the_blocking_claim(self): + self._skip() + data = self._client().get("/api/test/mlat-history?kind=resolve_skips").json() + assert data["kind"] == "resolve_skips" + assert data["n_records"] == 1 + rec = data["records"][0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 3 + assert {b["track_id"] for b in rec["blocking"]} == {"a1", "b1"} + assert all(b["held_n"] == 3 for b in rec["blocking"]) + + def test_skips_do_not_land_in_the_solve_history(self): + self._skip() + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + + def test_lane_filter_applies_to_skips(self): + self._skip(track_ids=("a1", "b1")) + self._skip(track_ids=("a2", "b2"), adsb_hex="abc123") + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=dark").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=adsb").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips").json()["lane_counts"] == { + "dark": 1, + "known": 0, + "adsb": 1, + } + + def test_unknown_kind_is_rejected(self): + assert self._client().get("/api/test/mlat-history?kind=bogus").status_code == 400 diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py new file mode 100644 index 00000000..2c9eec02 --- /dev/null +++ b/backend/tests/test_solver_alt_mode.py @@ -0,0 +1,344 @@ +"""SOLVER_ALT_MODE: how the n>=3 solve gets its altitude. + +sweep (the default) calls the LM once per fixed altitude layer and keeps the +lowest rms_delay — six process-pool round trips, and an altitude quantised to +a ladder 2 km wide, which puts up to 1 km of error into the residual the +reject gate reads. free makes ONE call to the geolocator's multi-start +helper, which solves altitude as a sixth unknown from SOLVER_FREE_ALT_STARTS +start layers — one by default, the layer nearest the association guess. + +These tests are about the routing, not the physics: the geolocator's own +suite (tests/test_free_altitude.py there) measures what the free solve +actually fits. What matters here is that the default is byte-identical to +the sweep, that free spends one call and not six, that trimming re-solves +under the same mode, and that both modes leave enough on the history record +to be compared live. +""" + +import time + +import pytest + +from core import state +from services import frame_processor +from services.tasks import solver as solver_mod + +LAT, LON = 35.0, -82.0 + + +def _s_in(node_ids, alt_km=9.0, **overrides): + s_in = { + "initial_guess": {"lat": LAT, "lon": LON, "alt_km": alt_km}, + "measurements": [{"node_id": nid, "delay_us": 10.0, "doppler_hz": 1.0, "snr": 15.0} for nid in node_ids], + "n_nodes": len(node_ids), + "timestamp_ms": int(time.time() * 1000), + } + s_in.update(overrides) + return s_in + + +def _stub_result(node_ids, rms_delay=0.5, **overrides): + result = { + "success": True, + "lat": LAT, + "lon": LON, + "alt_m": 9000.0, + "timestamp_ms": int(time.time() * 1000), + "vel_east": 0.0, + "vel_north": 0.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "n_nodes": len(node_ids), + "n_measurements": len(node_ids), + "contributing_node_ids": list(node_ids), + } + result.update(overrides) + return result + + +class _Recorder: + """A solve_fn / multistart_fn that records every call it is given.""" + + def __init__(self, result_for): + self.calls: list[tuple] = [] + self._result_for = result_for + + def __call__(self, s_in, node_cfgs, *rest): + self.calls.append((s_in, node_cfgs, rest)) + nodes = tuple(m["node_id"] for m in s_in["measurements"]) + return self._result_for(nodes, s_in, *rest) + + +class _AltModeBase: + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + +class TestFreeAltStarts: + """The starts handed to the multi-start helper: SOLVER_FREE_ALT_STARTS of + them, one by default.""" + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [9.0]), + (7.0, [7.0]), + (8.2, [9.0]), + (1.5, [1.5]), + # Off the ends of the ladder: still the nearest layer, not nothing. + (0.4, [1.5]), + (40.0, [11.0]), + ], + ) + def test_one_start_at_the_nearest_layer_by_default(self, alt_km, expected): + assert state.SOLVER_FREE_ALT_STARTS == 1 + assert solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) == expected + + def test_an_adsb_altitude_in_the_ladder_is_the_start(self): + """_solve_best_altitude splices a non-layer altitude (ADS-B) into the + layers, and the starts are taken over that spliced list — so the single + default start is that exact altitude, which is what the sweep would + have pinned too.""" + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [8.4] + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [7.0, 9.0, 11.0]), + (7.0, [5.0, 7.0, 9.0]), + (8.2, [7.0, 9.0, 11.0]), + # Clamped at the ends: the ladder's first and last layers still get + # three starts, not one or two. + (1.5, [1.5, 3.0, 5.0]), + (0.4, [1.5, 3.0, 5.0]), + (11.0, [7.0, 9.0, 11.0]), + (40.0, [7.0, 9.0, 11.0]), + ], + ) + def test_three_starts_are_the_window_around_the_nearest_layer(self, alt_km, expected, monkeypatch): + """The pre-default behaviour, still reachable by configuration.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) + starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) + assert starts == expected + assert len(starts) == 3 + + def test_three_starts_window_the_spliced_adsb_altitude(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0] + + @pytest.mark.parametrize("configured", [0, -3]) + def test_fewer_than_one_start_still_starts_somewhere(self, configured, monkeypatch): + """A count below one would leave the LM no start at all, so it clamps + rather than raises: a mis-set env degrades to a working solve.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", configured) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == [9.0] + + def test_more_starts_than_layers_is_every_layer(self, monkeypatch): + """The other clamp: a count past the end of the ladder would slice + short of it, quietly dropping starts that were asked for.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 99) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == solver_mod._SOLVER_ALT_LAYERS_KM + + def test_no_layers_gives_no_starts(self): + assert solver_mod._free_alt_starts(9.0, []) == [] + + +class TestSweepIsTheDefault(_AltModeBase): + def test_sweep_calls_the_lm_once_per_layer_and_never_the_multistart(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("multistart called in sweep mode")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert [c[0]["initial_guess"]["alt_km"] for c in solve.calls] == solver_mod._SOLVER_ALT_LAYERS_KM + assert multistart.calls == [] + assert state.SOLVER_ALT_MODE == "sweep" + + +class TestFreeMode(_AltModeBase): + def setup_method(self): + super().setup_method() + self._saved_mode = state.SOLVER_ALT_MODE + state.SOLVER_ALT_MODE = "free" + + def teardown_method(self): + state.SOLVER_ALT_MODE = self._saved_mode + super().teardown_method() + + def test_one_multistart_call_with_one_start(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode")) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4])) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert solve.calls == [] + assert len(multistart.calls) == 1 + (_, _, rest) = multistart.calls[0] + assert rest == ([9.0],) + + def test_an_adsb_guess_altitude_is_the_start(self): + """A non-layer initial_guess altitude is spliced into the ladder and + becomes the start itself — the free-mode analogue of the sweep's extra + layer, and the one exact altitude the candidate has.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(nodes, alt_km=8.437), {}, lambda s, c: None, multistart) + + assert multistart.calls[0][2] == ([8.437],) + + def test_the_start_count_is_configurable(self, monkeypatch): + """SOLVER_FREE_ALT_STARTS buys back the neighbour window for a geometry + whose single start lands on the wrong side of an ellipse.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, lambda s, c: None, multistart) + + assert len(multistart.calls) == 1 + assert multistart.calls[0][2] == ([7.0, 9.0, 11.0],) + + def test_n2_keeps_the_sweep(self): + """Altitude is unobservable at n=2 — the free path is not entered even + with the mode on.""" + nodes = ["n1", "n2"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken at n=2")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert multistart.calls == [] + + def test_a_failed_multistart_is_a_failed_solve(self): + """No silent fall back to the sweep: three starts producing nothing is + the same verdict as every layer producing nothing.""" + solve = _Recorder(lambda n, s, *r: pytest.fail("swept after a failed multistart")) + multistart = _Recorder(lambda n, s, *r: None) + + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) is None + + def test_history_carries_the_mode_and_the_per_start_residuals(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result( + n, altitude_mode="free", rms_by_start=[1.2345, None, 0.4321], alt_starts_km=[5.0, 7.0, 9.0] + ) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["altitude_mode"] == "free" + assert rec["alt_starts_km"] == [5.0, 7.0, 9.0] + assert rec["alt_start_rms_us"] == [1.234, None, 0.432] + + def test_a_single_start_still_records_its_residual(self): + """The comparison channel does not depend on there being several + starts: one start records a one-element list, not a bare number or + nothing at all.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4321], alt_starts_km=[9.0]) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["alt_starts_km"] == [9.0] + assert rec["alt_start_rms_us"] == [0.432] + + def test_z_saturation_reaches_the_history(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True)) + solver_mod._process_solver_item((_s_in(nodes), {}, time.time()), lambda s, c: None, multistart_fn=multistart) + assert state.mlat_solve_history[-1]["z_saturated"] is True + + def test_trimming_re_solves_through_the_multistart(self): + """A trim round must use the mode its first solve used, or the rms it + compares against the previous round is a different quantity.""" + full = ["n1", "n2", "n3", "n4", "bad"] + trimmed = ["n1", "n2", "n3", "n4"] + + def _result(nodes, s_in, *rest): + if "bad" in nodes: + return _stub_result( + nodes, + rms_delay=8.0, + altitude_mode="free", + per_node_delay_res_us={n: (12.0 if n == "bad" else 0.5) for n in nodes}, + ) + return _stub_result( + nodes, + rms_delay=0.8, + altitude_mode="free", + per_node_delay_res_us={n: 0.3 for n in nodes}, + ) + + multistart = _Recorder(_result) + result = solver_mod._process_solver_item( + (_s_in(full), {}, time.time()), + lambda s, c: pytest.fail("sweep ran during a free-mode trim"), + multistart_fn=multistart, + ) + + assert result is not None and result["n_nodes"] == 4 + assert sorted(m["node_id"] for m in multistart.calls[-1][0]["measurements"]) == trimmed + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["trimmed_node_ids"] == ["bad"] + assert rec["altitude_mode"] == "free" + + def test_an_unrecognised_mode_would_sweep(self): + """The flag degrades to the inert mode, like its siblings — asserted on + the resolution rule rather than by re-importing core.state.""" + state.SOLVER_ALT_MODE = "definitely-not-a-mode" + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken for a bad mode")) + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + + +class TestConfigsForSolverInput: + """Only the configs a candidate can reach are queued with it. + + The pool is a spawn pool, so whatever is queued is pickled and shipped on + every solve — 58 fleet configs against a candidate's 2-8 measurements. + """ + + _FLEET = {f"n{i}": {"rx_lat": 35.0 + i, "rx_lon": -82.0} for i in range(8)} + + def test_restricted_to_the_measurement_nodes(self): + s_in = _s_in(["n1", "n3", "n5"]) + cfgs = frame_processor.configs_for_solver_input(self._FLEET, s_in) + assert sorted(cfgs) == ["n1", "n3", "n5"] + assert cfgs["n3"] is self._FLEET["n3"] + + def test_unknown_measurement_nodes_are_simply_absent(self): + """A measurement from a node with no config is the case + solve_multinode already handles by skipping it — not an error here.""" + cfgs = frame_processor.configs_for_solver_input(self._FLEET, _s_in(["n1", "ghost"])) + assert sorted(cfgs) == ["n1"] + + def test_no_measurements_gives_nothing(self): + assert frame_processor.configs_for_solver_input(self._FLEET, {"measurements": []}) == {} + assert frame_processor.configs_for_solver_input(self._FLEET, {}) == {} diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..5e9aa45e 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -257,6 +257,10 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.tracks_stale_skipped = 13 + state.solver_epoch_align_skipped = 14 + state.solver_resolve_skips_dark = 9 + state.node_frames_rate_limited = 13 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -265,7 +269,11 @@ def test_consensus_and_counters_reflect_state(self): "solver_trimmed": 3, "stale_drops": 4, "resolve_skips": 12, + "tracks_stale_skipped": 13, + "epoch_align_skipped": 14, + "resolve_skips_dark": 9, "queue_drops": 6, + "node_frames_rate_limited": 13, "worker_errors": 0, "vel_untrusted_published": 11, } @@ -788,3 +796,118 @@ def get(self, *a, **kw): state.multinode_tracks["mn-dark-1"] = {"lat": 35.0, "lon": -82.0} state.adsb_aircraft["real1"] = _MutatingFix({"lat": 35.009, "lon": -82.0, "last_seen_ms": now_ms}) assert _solver_window_stats(10.0)["ghosts"]["ghost_tracks"] == 0 + + +def _skip_rec(lane="dark", age_s=0.0, track_ids=("a1",), n_nodes=3): + return { + "ts_ms": int((time.time() - age_s) * 1000), + "lane": lane, + "track_ids": list(track_ids), + "n_nodes": n_nodes, + "blocking": [{"track_id": track_ids[0], "held_ts": time.time() - age_s, "held_n": n_nodes}], + "guess_lat": None, + "guess_lon": None, + } + + +class TestResolveSkipBlock: + """Resolve-slot skips are windowed from their own deque, not from the + since-boot counter, so they can be read against the attempts in the same + window — the ratio the claim-on-publish fix is judged on.""" + + def setup_method(self): + state._reset_for_tests() + + def test_totals_split_by_lane(self): + for _ in range(3): + state.solver_resolve_skips_recent.append(_skip_rec("dark")) + state.solver_resolve_skips_recent.append(_skip_rec("adsb")) + out = _solver_window_stats(10.0)["resolve_skips"] + assert out["total"] == 4 + assert out["dark"] == 3 + + def test_window_excludes_old_skips(self): + state.solver_resolve_skips_recent.append(_skip_rec(age_s=20 * 60)) + state.solver_resolve_skips_recent.append(_skip_rec(age_s=1)) + assert _solver_window_stats(10.0)["resolve_skips"]["total"] == 1 + + def test_attempts_ratio_is_skips_over_dark_attempts(self): + for _ in range(4): + state.solver_resolve_skips_recent.append(_skip_rec()) + state.mlat_solve_history.append(_rec("published")) + state.mlat_solve_history.append(_rec("rejected_beam")) + out = _solver_window_stats(10.0) + assert out["attempts"] == 2 + assert out["resolve_skips"]["attempts_ratio"] == 2.0 + + def test_attempts_ratio_is_none_without_attempts(self): + state.solver_resolve_skips_recent.append(_skip_rec()) + assert _solver_window_stats(10.0)["resolve_skips"]["attempts_ratio"] is None + + def test_window_effective_minutes_exposes_a_truncated_deque(self): + """The deque is 500 entries against ~50 skips/min live, so a long + window IS truncated here even when the solve stores cover it.""" + state.solver_resolve_skips_recent.append(_skip_rec(age_s=6 * 60)) + out = _solver_window_stats(30.0)["resolve_skips"] + assert 5.9 <= out["window_effective_minutes"] <= 6.1 + + def test_a_skip_is_not_an_attempt_or_a_reject(self): + """Skips must not leak into the funnel — they never reached a solve.""" + for _ in range(5): + state.solver_resolve_skips_recent.append(_skip_rec()) + out = _solver_window_stats(10.0) + assert out["attempts"] == 0 + assert out["rejects"]["total"] == 0 + + +def _gt_rec(foreign=(), **kw): + """A dark record carrying the contamination stamp.""" + rec = _rec("published", **kw) + rec["gt_hex"] = "abc123" + rec["foreign_node_ids"] = list(foreign) + rec["contaminated"] = bool(foreign) + return rec + + +class TestContaminationBlock: + """Live cluster contamination: of the dark records that matched ground + truth, how many carried a node that could not see the aircraft.""" + + def setup_method(self): + state._reset_for_tests() + + def test_pct_and_mean_over_judged_records(self): + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_gt_rec(foreign=["n1", "n2"])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 4 + assert out["contaminated"] == 2 + assert out["pct"] == 50.0 + assert out["foreign_nodes_per_record"] == 0.75 + + def test_unstamped_records_are_out_of_the_denominator(self): + """No GT match, or no judgeable node geometry, is an abstention — not + a clean record.""" + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_rec("published")) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 1 + assert out["pct"] == 100.0 + + def test_empty_window_abstains_rather_than_reporting_zero(self): + out = _solver_window_stats(10.0)["contamination"] + assert out == { + "records_with_gt": 0, + "contaminated": 0, + "pct": None, + "foreign_nodes_per_record": None, + } + + def test_known_lane_records_are_not_counted(self): + """Dark lane only, like every other block in the funnel.""" + rec = _gt_rec(foreign=["n1"]) + rec["known_lane"] = True + state.mlat_solve_history_known.append(rec) + assert _solver_window_stats(10.0)["contamination"]["records_with_gt"] == 0 diff --git a/backend/tests/test_solver_trimming.py b/backend/tests/test_solver_trimming.py index 93504a5a..0f529848 100644 --- a/backend/tests/test_solver_trimming.py +++ b/backend/tests/test_solver_trimming.py @@ -827,3 +827,76 @@ def solve_fn(_s_in, _cfgs): assert state.fov_shadow_agree == 1 assert state.fov_shadow_would_pass == 0 assert state.fov_shadow_would_reject == 0 + + +class TestTrimmedTracksAreNotClaimed(_TrimmingTestBase): + """A trimmed node's tracks must not take a re-solve claim. + + The claim says "this aircraft is on the map at this width". A node + dropped for a bad residual contributed nothing to the published position + and its track was probably a different aircraft's — claiming it would + suppress that aircraft's own candidate on the strength of a measurement + this solve threw away. + """ + + _FULL = ["n1", "n2", "n3", "n4", "bad"] + _TRIM = ["n1", "n2", "n3", "n4"] + + def test_the_dropped_nodes_track_is_left_unclaimed(self): + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + result = self._run(s_in, _stub_solve_fn(table)) + assert result is not None and result["success"] + assert result["source_track_ids"] == ["t1", "t2", "t3", "t4"] + assert set(solver_mod._RECENT_SOLVES) == {"t1", "t2", "t3", "t4"} + + def test_a_candidate_built_on_the_dropped_track_still_runs(self): + """The other half of the same claim: whoever "tbad" really belongs to + keeps its slot.""" + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + self._run(s_in, _stub_solve_fn(table)) + neighbour = {"n_nodes": 2, "track_ids": ["tbad", "tother"]} + assert solver_mod._resolve_slot_covered(neighbour, time.time())[0] is False diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py index 6f5ea7b3..6e3a2035 100644 --- a/backend/tests/test_solver_worker.py +++ b/backend/tests/test_solver_worker.py @@ -38,6 +38,7 @@ def _reset_state(): state.n2_unconfirmed = 0 state.solver_stale_drops = 0 state.solver_resolve_skips = 0 + state.solver_resolve_skips_dark = 0 state.multinode_tracks.clear() state.task_last_success.clear() @@ -334,56 +335,153 @@ class TestResolveSuppression: emits its own candidate for it inside one association window. Solving all of them starves aircraft that have no solve at all — the queue ages out behind work whose result is superseded the moment it lands. + + The claim that suppresses a duplicate is taken on PUBLICATION + (_record_resolve_slot), not on admission: the rule is "this aircraft is + already on the map at this width", and only a publish puts it there. + _resolve_slot_covered is the pure test run before the solve. """ def _s_in(self, track_ids, n_nodes=2): return dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids)) - def test_a_second_copy_of_the_same_tracks_is_skipped(self): + def _covered(self, track_ids, n_nodes=2, now=None): + return solver_mod._resolve_slot_covered(self._s_in(track_ids, n_nodes), now or time.time())[0] + + def _publish(self, track_ids, n_nodes=2, now=None): + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now or time.time()) + + def test_a_second_copy_of_a_published_candidate_is_skipped(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is False + assert self._covered(["a1", "b1"], now=now) is False + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is True - def test_a_candidate_carrying_an_unsolved_track_runs(self): + def test_the_check_alone_claims_nothing(self): + """The whole point of the split: a candidate that is admitted and then + rejected by the gate stack must leave no trace.""" + now = time.time() + assert self._covered(["a1", "b1"], now=now) is False + assert self._covered(["a1", "b1"], now=now) is False + + def test_a_candidate_carrying_an_unpublished_track_runs(self): """An aircraft entering coverage must never be suppressed.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b2"], now=now) is False def test_a_wider_view_of_the_same_tracks_runs(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True + self._publish(["a1", "b1"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=5, now=now) is False def test_a_narrower_view_after_a_wider_one_is_skipped(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True - def test_a_narrow_admission_does_not_lower_the_bar(self): + def test_a_narrow_publish_does_not_lower_the_bar(self): """The window holds the widest claim, not the most recent one.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=3), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + self._publish(["a1", "b2"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=3, now=now) is True def test_claims_expire(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) later = now + solver_mod._SOLVER_RESOLVE_INTERVAL_S + 1.0 - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), later) is True + assert self._covered(["a1", "b1"], now=later) is False def test_an_input_without_track_provenance_always_runs(self): """Detection-level inputs carry no track ids — nothing to match on.""" now = time.time() - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False + solver_mod._record_resolve_slot(None, 2, now) + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False def test_zero_interval_disables_suppression(self, monkeypatch): monkeypatch.setattr(solver_mod, "_SOLVER_RESOLVE_INTERVAL_S", 0.0) now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is False + + def test_the_check_names_every_blocking_claim(self): + now = time.time() + self._publish(["a1", "b1"], n_nodes=4, now=now) + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"], n_nodes=3), now) + assert covered is True + assert {b["track_id"]: b["held_n"] for b in blocking} == {"a1": 4, "b1": 4} + + def test_an_admitted_candidate_reports_no_blockers(self): + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"]), time.time()) + assert (covered, blocking) == (False, []) + + def _solve_fn(self, calls, rms_delay=0.5, lat=37.5, lon=-122.1): + def fn(s_in, cfgs): + calls.append(s_in) + return { + "success": True, + "lat": lat, + "lon": lon, + "alt_m": 9000.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "timestamp_ms": int(time.time() * 1000), + "contributing_node_ids": ["n1", "n2"], + "n_nodes": s_in.get("n_nodes", 2), + } + + return fn + + def test_a_rejected_candidate_does_not_block_an_identical_twin(self, monkeypatch): + """The bug this split exists to fix. A candidate the gate stack sank + put nothing on the map, so the next copy of the same aircraft is its + first real chance — and used to be blacked out for the full 12 s.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + s_in = self._s_in(["a1", "b1"]) + + # rms_delay past the gate: solves, then rejected, publishes nothing. + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert state.solver_fail_rms_delay == 1 + assert not state.multinode_tracks + + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2, "the twin must not be suppressed by a reject" + assert state.multinode_tracks + assert state.solver_resolve_skips == 0 + + def test_a_subset_for_another_aircraft_survives_a_rejected_superset(self, monkeypatch): + """Tracker track ids are shared across the candidates of DIFFERENT + aircraft, so a contaminated superset that the gates sank used to take + every clean subset behind it down with it — including its neighbour's + only candidate.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + superset = self._s_in(["a1", "b1", "c1"], n_nodes=3) + solver_mod._process_solver_item((superset, {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert not state.multinode_tracks + + # The neighbour: fewer nodes, sharing one contaminated track id. + subset = self._s_in(["a1", "b1"], n_nodes=2) + solver_mod._process_solver_item((subset, {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2 + assert state.multinode_tracks + + def test_only_the_published_width_is_claimed(self, monkeypatch): + """A publish claims at the width it published, so a later narrower + copy is suppressed and a wider one still runs.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + solver_mod._process_solver_item((self._s_in(["a1", "b1"], n_nodes=3), {}, time.time()), self._solve_fn(calls)) + assert state.multinode_tracks + now = time.time() + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True + assert self._covered(["a1", "b1"], n_nodes=4, now=now) is False def test_a_skipped_item_never_reaches_the_solver(self, monkeypatch): _reset_state() @@ -407,6 +505,8 @@ def solve_fn(s_in, cfgs): solver_mod._process_solver_item((s_in, {}, time.time()), solve_fn) assert len(solve_calls) == 1 assert state.solver_successes == 1 + # The first item PUBLISHED, which is what makes the second redundant. + assert state.multinode_tracks assert solver_mod._process_solver_item((dict(s_in), {}, time.time()), solve_fn) is None assert len(solve_calls) == 1, "the duplicate must not be solved" @@ -415,6 +515,50 @@ def solve_fn(s_in, cfgs): assert state.solver_failures == 0 assert state.solver_stale_drops == 0 + def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self): + """The counter alone cannot say WHOSE claim suppressed a candidate, + and tracker track ids are shared between different aircraft — so a + skip that suppressed a duplicate and one that suppressed a neighbour + looked identical. The deque carries the blocking claims.""" + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0}) + solver_mod._record_resolve_slot(["a1", "b1"], 4, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s_in), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s_in), now, blocking) + + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 1 + assert len(state.solver_resolve_skips_recent) == 1 + rec = state.solver_resolve_skips_recent[0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 4 + assert rec["guess_lat"] == 35.0 + assert {b["track_id"]: b["held_n"] for b in rec["blocking"]} == {"a1": 4, "b1": 4} + + def test_a_tagged_candidate_is_counted_but_not_as_dark(self): + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123") + solver_mod._record_resolve_skip(s_in, now, []) + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 0 + assert state.solver_resolve_skips_recent[0]["lane"] == "adsb" + + def test_skips_never_enter_the_solve_history(self): + """One skip per solve-history record would evict the solves the same + investigation needs — live, skips outrun dark records two to one.""" + _reset_state() + state.mlat_solve_history.clear() + s_in = self._s_in(["a1", "b1"]) + solver_mod._record_resolve_skip(s_in, time.time(), []) + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + class TestSolveBestAltitude: """Altitude-sweep helpers: n_nodes >= 3 uses a layer sweep, n_nodes = 2 uses initial_guess directly.""" diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..db3bdedd 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -11,10 +11,12 @@ beyond what publication needs, see [`pipeline.md`](pipeline.md) (its own §3 is stale on the known lane and pool fallback — this doc is the current source for those two topics). -File:line references are repo-relative to `backend/`, except the `libs/*` -paths, which are already fully qualified (those are separate submodule repos -vendored under `libs/`). All references were checked against `main` at -`0a1d30f`. +References name a **file and a symbol**, never a line number: paths are +repo-relative to `backend/`, except the `libs/*` ones, which are already fully +qualified (those are separate submodule repos vendored under `libs/`). Line +numbers were what this document used to carry, and they were stale within two +weeks of being written — every one of them had drifted by the time anyone +followed it. A symbol survives an edit above it, so grep for the name. ## Legend @@ -74,13 +76,15 @@ lane rides the solver loop's idle cycles rather than owning workers of its own. Everything that reaches a solve passes through one gate stack (`_process_solver_item`) before publication. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py:358-359` | -| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py:365-366` | -| `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | -| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | -| `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | +| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py` | +| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py` | +| `FRAME_WORKERS` | 4 (compose sets 6) | `core/state.py` (`FRAME_WORKERS`), `docker-compose.yml` | +| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py` (`_N_SOLVER_WORKERS`, `_make_solver_pool`) | +| `KNOWN_LANE_MODE` default | `binding` | `core/state.py` (`KNOWN_LANE_MODE`) | +| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py` (`SOLVER_ALT_MODE`) | +| `SOLVER_FREE_ALT_STARTS` default | 1 | `core/state.py` (`SOLVER_FREE_ALT_STARTS`) | --- @@ -89,11 +93,11 @@ own. Everything that reaches a solve passes through one gate stack ```mermaid flowchart TD subgraph producers["Five producers"] - p1["TCP (primary)
tcp_handler.py:326"] - p2["blah2 bridge
blah2_bridge.py:289"] - p3["v1 node HTTP API
node_stream.py:250"] - p4["Legacy HTTP radar routes
routes/radar.py:151,202"] - p5["Startup priming
node_pipeline.py:139"] + p1["TCP (primary)
tcp_handler._enqueue_detection"] + p2["blah2 bridge
blah2_bridge.blah2_bridge_task"] + p3["v1 node HTTP API
node_stream._file_frame"] + p4["Legacy HTTP radar routes
radar.ingest_detections(_bulk)"] + p5["Startup priming
node_pipeline.prime_pipeline"] end p1 --> gA{"Gate A: timestamp present?"} @@ -131,25 +135,25 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -The ordering inside `process_one_frame` (`services/frame_processor.py:294`) is +The ordering inside `process_one_frame` (`services/frame_processor.py`) is load-bearing, not incidental: claiming (2.3) runs **before** ADS-B seeding (2.4) so a node-supplied `adsb` field is still distinguishable from a claim, and both run **before** the tracker (2.5) so that, in `binding` mode, a claimed detection never reaches the dark-lane tracker or association at all -— see the ordering comment at `services/frame_processor.py:327-337`. +— see the ordering comment at the head of `process_one_frame`'s claiming step. Frame-level gates (A/B/C on TCP, plus the connected-node check on the v1 API) sit ahead of everything else; nothing downstream sees a frame that failed one of them. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| Gate A: timestamp required | — | `tcp_handler.py:513-516` | -| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node | `tcp_handler.py:495,527-532` | -| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler.py:536-552` | -| `process_one_frame` entry | — | `services/frame_processor.py:294` | -| Ordering rationale (claim → seed → tracker) | — | `services/frame_processor.py:327-337` | -| Gate 2.10: `n_nodes < 2` skip | — | `services/frame_processor.py:409-426` | -| blah2 poll interval | 1.0 s | `config/constants.py:266` | +| Gate A: timestamp required | — | `tcp_handler._enqueue_detection` | +| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node, counted as `node_frames_rate_limited` | `tcp_handler` (`_NODE_MIN_INTERVAL_S`, `_enqueue_detection`) | +| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler._enqueue_detection` | +| `process_one_frame` entry | — | `services/frame_processor.py` | +| Ordering rationale (claim → seed → tracker) | — | `frame_processor.process_one_frame` | +| Gate 2.10: `n_nodes < 2` skip | — | `frame_processor.process_one_frame` | +| blah2 poll interval | 1.0 s | `config/constants.py` (`BLAH2_POLL_INTERVAL_S`) | --- @@ -205,17 +209,17 @@ also reject — a differential property test in `test_known_claiming.py` failure increments the same `known_claims_visibility_rejects` counter as a gate failure: same event, same meaning, just caught cheaper. -**Mode semantics** (`KNOWN_LANE_MODE`, read once at `core/state.py:72-74`, +**Mode semantics** (`KNOWN_LANE_MODE`, read once in `core/state.py`, default `binding`; an unrecognized value falls back to `shadow`, not to the default — a typo should degrade to the inert mode, not the acting one): | Mode | Claiming | Frame the dark lane sees | Known-lane solver | Publication | |---|---|---|---|---| -| `off` | never runs | untouched | returns 0 immediately (`known_lane.py:391-392`); worker never even calls it (`solver.py:1961`) | none | +| `off` | never runs | untouched | returns 0 immediately (`known_lane.run_known_lane_pass`); worker never even calls it (`solver._run_solver_worker`) | none | | `shadow` | runs, records claims + residuals + counters | untouched | runs: solves, classifies, records accuracy samples | never | -| `binding` | runs | `strip_claimed_detections` removes claimed indices (`frame_processor.py:347`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | +| `binding` | runs | `strip_claimed_detections` removes claimed indices (called from `frame_processor.process_one_frame`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | -`strip_claimed_detections` (`services/known_claiming.py:343`) returns a copy +`strip_claimed_detections` (`services/known_claiming.py`) returns a copy with claimed indices removed from `delay`/`doppler`/`snr`/`adsb`; the original frame still feeds the archive and ADS-B extraction (steps 2.11-2.12) unchanged. @@ -224,7 +228,7 @@ unchanged. ```mermaid flowchart TD - arm["Solver worker loop arms known_lane
at thread start (solver.py:1961)"] + arm["Solver worker loop arms known_lane
at thread start (solver._run_solver_worker)"] arm --> drain["After every queue-drain iteration,
call maybe_run_pass"] drain --> gm{"mode == off?"} gm -->|"yes"| ret1["return"]:::inert @@ -261,7 +265,7 @@ flowchart TD gpub -->|"no"| noop["accuracy sample only,
no feed entry"]:::inert ``` -The docstring at `services/tasks/known_lane.py:19-27` calls this the "free +The module docstring of `services/tasks/known_lane.py` calls this the "free solve invariant": the ADS-B fix seeds the initial guess and pins altitude, nothing else — no regularization pulls the solve toward the truth position, so the residual (`err_km`) is a genuine measurement of radar accuracy, not a @@ -269,21 +273,21 @@ circular check. One more intentional-by-omission detail: known-lane measurements carry `snr = 0.0` (claim records have no `snr` key), which the LM's SNR weighting maps to a uniform weight of 1.0. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `ADSB_SEED_MAX_DR_AGE_S`, `association.py:106`) | -| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `ADSB_SEED_*`, `association.py:98,99`) | -| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py:79` | -| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py:205` | -| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py:89`, applied in `known_claiming.py:_dark_global_projections` | -| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py:73,77` | -| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py:80` | -| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py:85,86` | -| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py:274-275` | -| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py:105` | -| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py:91,99` | -| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` | -| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` | +| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `association.ADSB_SEED_MAX_DR_AGE_S`) | +| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `association.ADSB_SEED_DELAY_GATE_US` / `_DOPPLER_GATE_HZ`) | +| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py` | +| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py` | +| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py`, applied in `known_claiming._dark_global_projections` | +| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py` | +| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py` | +| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py` | +| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py` | +| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py` | +| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py` | +| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py` | +| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py` | --- @@ -291,7 +295,7 @@ LM's SNR weighting maps to a uniform weight of 1.0. ```mermaid flowchart TD - frame["pipeline.process_frame
passive_radar.py:672"] + frame["PassiveRadarPipeline.process_frame
pipeline/passive_radar.py"] frame --> tracker["retina_tracker
Kalman + GNN"] tracker --> geo["_run_geolocation per track
with new data"] @@ -345,7 +349,7 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -`compute_overlap_zone` (`libs/retina-analytics/.../association.py:578`) +`compute_overlap_zone` (`libs/retina-analytics/.../association.py`) underlies both the confirmed-track association round and the overlap-grid cache: it fast-prunes non-overlapping node pairs by receiver separation, grids the shared coverage at `ASSOC_GRID_STEP_KM` on six altitude layers that @@ -353,22 +357,22 @@ must match the solver's `_SOLVER_ALT_LAYERS_KM`, and requires each grid column to fall in **both** beams (`_point_in_beam`, FOV-aware only when `FOV_MODE=active`). -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py:194` | -| Single-node min detections | 3 | `passive_radar.py:356-361` | -| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py:59` | -| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py:220` | -| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py:22` | -| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py:23` | -| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py:31`, `association.py:1043` | -| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.py:883` | -| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.py:884` | -| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py:166` | -| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py:57-58` | -| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.py:2160` | -| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py:21` | -| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py:111` | +| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py` (applied as `_GEO_INTERVAL_S` in `_run_geolocation`) | +| Single-node min detections | 3 | `pipeline/passive_radar.py` (`_geolocate_track_event`, `min_det`) | +| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py` | +| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py` | +| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py` | +| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py` | +| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py`, `association.py` | +| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.compute_overlap_zone` (default arg) | +| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.compute_overlap_zone` (default arg) | +| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py` | +| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py` | +| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.InterNodeAssociator.format_track_pairs_for_solver` (local) | +| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py` | +| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py` | --- @@ -376,17 +380,34 @@ column to fall in **both** beams (`_point_in_beam`, FOV-aware only when The centerpiece: every candidate from either lane, once dequeued from `solver_queue`, runs through `_process_solver_item` -(`services/tasks/solver.py:1344`) as a strict, ordered chain. A failure at +(`services/tasks/solver.py`) as a strict, ordered chain. A failure at any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a named record to solve history. +**6.2 claims on publication, not on admission.** The suppression rule is +"this aircraft is already on the map at this width, at every track it is +built from" — so `_resolve_slot_covered` only *reads* the claims, and +`_record_resolve_slot` takes them from the publish path, with the +**post-trim survivors** (`result["source_track_ids"]`, rebuilt from the +surviving `track_ids_by_node`). Claiming on admission instead meant a +candidate that never reached the map still blacked out every later candidate +sharing any of its track ids for the full 12 s — including *other aircraft's*, +since tracker track ids are shared across the association candidates of +different aircraft (the same finding behind `_supersession_match`'s spatial +guard; see Caveats). Live that ran at ~1 537 skips per 646 dark attempts per +30 min: more candidates suppressed than solved, by a factor of two. The price +of the split is that the check no longer claims under the same lock, so two +workers can now both solve duplicates that arrived together; that costs one +extra solve and is resolved downstream by keying and supersession, which +handle exactly this case already. + ```mermaid flowchart TD deq["Dequeue (s_in, node_cfgs, enqueued_at)"] deq --> g61{"6.1 Staleness
age_s > _SOLVER_MAX_QUEUE_AGE_S 45.0s?"} g61 -->|"yes"| f61["solver_stale_drops
(no history record)"]:::inert - g61 -->|"no"| g62{"6.2 Re-solve suppression
_claim_resolve_slot False?"} - g62 -->|"yes"| f62["solver_resolve_skips"]:::inert + g61 -->|"no"| g62{"6.2 Re-solve suppression
_resolve_slot_covered (pure)?"} + g62 -->|"yes"| f62["solver_resolve_skips (+_dark)
+ skip record with blockers"]:::inert g62 -->|"no"| g63["6.3 Solve dispatch:
no guess -> bare solve_fn;
n>=3 -> consensus? then
_solve_best_altitude (sweep);
n=2 -> _solve_best_altitude_n2
(single altitude)"] g63 -->|"exception"| f63["solver_failures +
solver_fail_exception,
result=None"]:::inert g63 --> g64{"6.4 Trim & resolve (recovery):
guess AND n>=4 AND
rms_delay > 3.0us?"} @@ -430,14 +451,14 @@ flowchart TD ``` `SOLVER_CONSENSUS_MODE` is `off` in production (see the mode-flag table in -[`architecture.md:94-110`](architecture.md#feature-gates)), so in practice +[`architecture.md`](architecture.md#feature-gates)), so in practice this sub-branch never reaches `active` outside staging. ### The LM itself -`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py:518`, +`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py`, invoked through the process pool via `_pool_solve_multinode` -(`services/tasks/solver.py:1915`). +(`services/tasks/solver.py`). ```mermaid flowchart TD @@ -453,25 +474,78 @@ flowchart TD m6 -->|"no"| m7["vz_saturated if vz on bound;
rms recomputed unweighted;
cov_en_km2 from s^2(J^T J)^-1"] m7 --> alt{"n_nodes >= 3?"} - alt -->|"yes"| sweep["_solve_best_altitude wrapper:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + alt -->|"yes"| mode{"SOLVER_ALT_MODE"} + mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
SOLVER_FREE_ALT_STARTS start
layers (1 by default), z solved"] alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"] classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -| Constant | Value | File:line | +#### `SOLVER_ALT_MODE` — how the n>=3 solve gets its altitude + +`solve_multinode` pins altitude from `initial_guess.alt_km`, so the fix is only +as good as the altitude the caller found for it. `sweep`, the default, searches +the six fixed layers of `_SOLVER_ALT_LAYERS_KM` — 2 km apart, so the pin is +systematically up to 1 km wrong. On noise-free replay of this fleet's geometry +that quantisation alone left `rms_delay` at a 1.76 us median against the 3.0 us +gate at 6.5, while a solve at the true altitude reaches 0. Most of the gate's +budget is spent on the ladder, and the residual left over gets blamed on nodes: +trimming (6.4) drops measurements that were never the problem. + +`free` instead calls `solve_multinode_multistart`, which runs the LM with +altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded +0.05–20 km, the `vz` bound unchanged) from `SOLVER_FREE_ALT_STARTS` start +layers, keeping the lowest `rms_delay`. It is also cheaper: **one** process-pool +round trip per candidate instead of six, each of which pickles the node configs +the input needs. + +`SOLVER_FREE_ALT_STARTS` defaults to **1** — the layer nearest the association +guess, or the guess altitude itself when that came from ADS-B and was spliced +into the ladder (the same splice the sweep does). Freeing z removes the +ladder's quantisation but not the LM's locality, and extra starts are what +would stop a solve settling on the wrong side of a bistatic ellipse; on this +fleet's geometry they had almost nothing to stop. Over a 20-minute window of +1019 free-mode solves on test, the three starts' `rms_delay` differed by more +than 0.1 us in **13** of them, and the nearest-layer start was more than 0.5 us +worse than the best start in **2** — ~0.2% of solves helped, at three times the +solver CPU, while the pool is the binding constraint (~1.7 attempts/s against a +2.0 s average latency on two workers). Set it above 1 for a geometry where that +locality does bite; `_free_alt_starts` clamps it into `[1, len(layers)]` and +values above 1 give the same neighbour window as before, so `3` restores the +original behaviour exactly. + +At n=2 the mode is inert — four residuals cannot support six unknowns, so the +geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged. +Trimming re-solves through `_solve_best_altitude`, so a trim round inherits +whichever mode its first solve used. + +Both modes stamp `altitude_mode` (`"free"` / `"pinned"`) on every +`mlat_solve_history` record, published or rejected; `free` adds `alt_starts_km`, +`alt_start_rms_us` (each start's residual) and `z_saturated` (the altitude +analogue of `vz_saturated` — z stopped on a bound rather than converging, so +`alt_m` is the bound and not a fit). That is the comparison channel: deploy one +mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off +`/api/test/mlat-history`. + +| Mode | Pool calls per n>=3 candidate | Altitude | +|---|---|---| +| `sweep` (default) | 6 (one per layer) | quantised to the nearest layer | +| `free` | 1 (`SOLVER_FREE_ALT_STARTS` starts inside it, 1 by default) | solved, 0.05–20 km | + +| Constant | Value | Defined in | |---|---|---| -| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` | -| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py:744` | -| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py:160-162` | -| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py:132` | -| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py:173` | -| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py:205` | -| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py:56` | -| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py:807` | -| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py:154` | -| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py:51,52` | -| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py:57,63` | +| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` | +| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`, `_resolve_slot_covered`, `_record_resolve_slot`) | +| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` | +| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) | +| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` | +| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py` | +| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py` | +| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py` | +| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py` | +| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py` | +| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py` | --- @@ -507,7 +581,8 @@ flowchart TD popped --> store["state.multinode_tracks[key] = result"] blocked --> store store --> archive["track-archive buffer append"] - archive --> histpub["_record_solve_history: published"] + archive --> claimslot["_record_resolve_slot:
claim the POST-TRIM survivors
for _SOLVER_RESOLVE_INTERVAL_S"] + claimslot --> histpub["_record_solve_history: published"] histpub --> feed["build_combined_aircraft_json
(1 Hz flush)"] feed --> gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"} @@ -525,26 +600,61 @@ flowchart TD | Value | Set at | Meaning | |---|---|---| -| `multinode_solve` | `aircraft_feed.py:132` | published multi-node solve | -| `solver_adsb_seed` | `track_gates.py:330` | single-node LM with fresh ADS-B fix | -| `solver_single_node` | `track_gates.py:330` | single-node LM, no ADS-B | -| `single_node_ellipse_arc` | `track_gates.py:378` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | -| `adsb_single_node` | `aircraft_feed.py:_claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | -| `known_lane_truth_match` / `known_lane_ghost` | `known_lane.py:260` | accuracy-sample-only, not a feed entry | - -| Constant | Value | File:line | +| `multinode_solve` | `aircraft_feed.multinode_to_aircraft` | published multi-node solve | +| `solver_adsb_seed` | `track_gates.track_entry` | single-node LM with fresh ADS-B fix | +| `solver_single_node` | `track_gates.track_entry` | single-node LM, no ADS-B | +| `single_node_ellipse_arc` | `track_gates.track_entry` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | +| `adsb_single_node` | `aircraft_feed._claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | +| `known_lane_truth_match` / `known_lane_ghost` | `known_lane._record_accuracy` | accuracy-sample-only, not a feed entry | + +| Constant | Value | Defined in | |---|---|---| | `_MN_ASSOC_MAX_DIST_KM` / `_MN_ASSOC_MAX_AGE_S` (identity step 2/3) | 6.0 km / 60.0 s | `services/tasks/solver.py` | | `_MN_ASSOC_DRIFT_KM_PER_S` / `_MN_ASSOC_MAX_DIST_CAP_KM` (step 3 only — the gate grows with the matched entry's age) | 0.13 km/s / 12.0 km | `services/tasks/solver.py` | | Supersession gate (`_supersession_match`) — the same age-scaled `_mn_assoc_gate_km` and `_MN_ASSOC_MAX_AGE_S` as step 3, applied to the solve's RAW position | 6.0 + 0.13·dt km, cap 12.0 / 60.0 s | `services/tasks/solver.py` | -| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` | -| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` | -| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` | -| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` | -| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` | -| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` | -| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py:167` | -| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py:206,213` | +| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py` | +| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py` | +| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py` | +| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py` | +| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py` | +| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py` (`_DEDUP_PROXIMITY_KM`, `_DEDUP_ALT_GATE_FT`) | +| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py` | +| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py` | + +--- + +## 7. Reading the pipeline from outside + +Three endpoints answer questions about the two lanes, and each has a shape +worth knowing before it is trusted. + +**`/api/test/mlat-history`** dumps solve records. Both lanes write their own +deque (`state.mlat_solve_history`, `state.mlat_solve_history_known`) and every +reader merges them. `?lane=dark|known|adsb|all` narrows the answer; +`?limit=` (default 1 000, max 5 000) is applied **per lane**, so a known-lane +burst can never push dark records out of the response — the flat cap that +preceded it left a 30 min request holding only the newest ~6 min of dark +records, which reads exactly like a quiet dark lane. `lane_counts` is +reported pre-cap so a truncated `records` list is legible. +`?kind=resolve_skips` dumps a different store entirely — see below. + +**`/api/test/solver-stats`** is the Solver Report panel's source. Its funnel, +error percentiles, ghosts, fragmentation, `contamination` and `resolve_skips` +are all the DARK lane; `lane_split` gives the per-lane record counts and +`known_lane` that lane's own numbers. + +| Block | Says | Watch for | +|---|---|---| +| `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence | +| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts. It read ~2.4 while 6.2 claimed on admission; with the claim on publication it should sit at or below 0.5. The deque holds 500 entries against a live rate of tens per minute, so read `window_effective_minutes` before reading `total` as a window count | +| `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — | +| `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero | + +A skip is deliberately **not** a solve-history record: skips outrun dark +records roughly two to one on the live fleet, so writing them into +`mlat_solve_history` would evict exactly the solves an investigation needs. +They are also not counted as attempts or rejects — a skipped candidate never +reached a solve. --- @@ -566,15 +676,15 @@ flowchart TD (`fragmentation`) and `superseded_keys` / `superseded_blocked` on each published `mlat_solve_history` record are how this is watched. - **Node-trust residuals are measure-only.** `node_bias.py` computes them but - nothing in the solver consumes them yet (`node_bias.py:33-40` docstring). + nothing in the solver consumes them yet (`node_bias.py` module docstring). - **`docs/pipeline.md` §3 is stale.** It predates the known lane and the process-pool inline fallback; this doc supersedes it for both topics. - **The bottom-up doppler gate is inert.** `doppler_gate_hz` in the dark lane's coarse pairing step is defined but the grid gate is delay-only in - practice (`libs/retina-analytics/.../association.py:884`). + practice (`association.compute_overlap_zone`'s `doppler_gate_hz`). - **Production runs with every mode flag off** except `KNOWN_LANE_MODE`, which is `binding` everywhere by code default and is set in no environment's `.env`. The in-repo statement of what each environment sets is - [`architecture.md:94-110`](architecture.md#feature-gates); the actual + [`architecture.md`](architecture.md#feature-gates); the actual values live in the gitignored `backend/.env` on each host, not in this repo. diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..c58b662d 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit c58b662d764c8ef51cd003f98b21c3b69640d346 diff --git a/libs/retina-geolocator b/libs/retina-geolocator index 2da39822..6979943a 160000 --- a/libs/retina-geolocator +++ b/libs/retina-geolocator @@ -1 +1 @@ -Subproject commit 2da3982220374f05dd622633576108e57bd9e34f +Subproject commit 6979943aa7ae3d05e61a9cb355490eafcc8b45b3