From cce9b9933df1a129d286b3fe17e64ffe80b15cec Mon Sep 17 00:00:00 2001 From: Babissimo Date: Thu, 10 Sep 2026 20:51:29 +0100 Subject: [PATCH] Stop the diagnostic dumps handing back the receivers they hide Every receiver coordinate the API publishes is displaced (services/public_location.py), but the unauthenticated /api/test/* payloads carried quantities measured from the TRUE receiver to a point the same payload gives the position of: a beam-gate range and bearing, a bistatic range against a transmitter published untranslated, an FOV read-out at the true bearing off a curve /api/radar/analytics also publishes, a per-track bistatic delay. Each is a locus the receiver sits on, and enough of them intersect well inside the 0.5-1 km fuzz, so the displacement was decorative on any node appearing in one of these dumps. services/public_geometry.py puts the rule in one place: a per-node constant is the envelope and may be published, a per-record value that varies with the true geometry is a measurement and is withheld. The pass is structural over containers rather than a per-route field list, so a field is withheld at any nesting depth and a new record shape needs no list widened. It matches on the leaf key alone, which is its one real limit and is stated in the module. Applied to each payload at its serialisation edge rather than per record, so a key added to one of these payloads later cannot ship raw by being one the author forgot to wrap. /api/test/mlat-verification is the exception: both readers of its store are unauthenticated, so _publish_mlat_verification strips once on write rather than on every request, and both writers now go through it. /api/test/node/{id}/verification is node-scoped, so it also withholds solver_lat and solver_lon: every track entry is measured from that one node's receiver, and the aircraft feed publishes the same solve position displaced under the same hex, so the true frame beside it differences to the offset. A multinode payload keeps them, since no single receiver is behind that position. node_detection_range's hand-rolled furthest_detections drop becomes a call to the same module, so that rule has one owner rather than two. That summary's keys come from the retina_analytics submodule, so a test pins its overlap with the withheld set: an upgrade that adds a colliding name fails there instead of quietly dropping the field from the payload. Closes 123zgec25me. Leaves 123zgec25a0 (position_error_km, which needs a decision about what the route is for) and 123zgec25bc (the private-node opt-out) open, both already tracked. Co-Authored-By: Claude Opus 5 --- backend/routes/test.py | 26 +- backend/services/public_geometry.py | 99 ++++++++ backend/services/tasks/analytics_refresh.py | 21 +- backend/tests/test_public_geometry.py | 260 ++++++++++++++++++++ 4 files changed, 393 insertions(+), 13 deletions(-) create mode 100644 backend/services/public_geometry.py create mode 100644 backend/tests/test_public_geometry.py diff --git a/backend/routes/test.py b/backend/routes/test.py index 9d1202cb..62f48396 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -18,6 +18,7 @@ from services.frame_processor import resolve_ground_truth_hex from services.geo import haversine_km from services.id_utils import is_transponder_hex, normalize_hex_key +from services.public_geometry import without_receiver_geometry from services.public_location import fuzz_enabled, public_latlon, translate_polygon from services.tasks import solver as solver_mod @@ -731,11 +732,18 @@ def _mlat_verification_summary() -> dict: @router.get("/api/test/node/{node_id}/verification") async def node_verification(node_id: str): - """Return pre-computed solver-vs-ADS-B verification stats for one node.""" - return Response( - content=state.latest_node_verification_bytes.get(node_id, b"{}"), - media_type="application/json", - ) + """Return pre-computed solver-vs-ADS-B verification stats for one node. + + Unauthenticated, and everything in a track entry is measured from this one + node's true receiver, so the entries are served node-scoped: the per-track + delays and the node's own solve position go, the errors beside them stay. + Stripped at the route rather than in the store, because it is this + route's one-node addressing that makes the solve position + receiver-relative; the store holds the computation's own output. + """ + raw = state.latest_node_verification_bytes.get(node_id, b"{}") + payload = without_receiver_geometry(orjson.loads(raw), node_scoped=True) + return Response(content=orjson.dumps(payload), media_type="application/json") @router.get("/api/test/mlat-verification") @@ -904,7 +912,7 @@ async def mlat_history( "n_records": len(skips), "records": skips[:limit], } - return Response(content=orjson.dumps(payload), media_type="application/json") + return Response(content=orjson.dumps(without_receiver_geometry(payload)), media_type="application/json") merged = _merged_solve_history() effective_minutes = _window_effective_minutes(merged, minutes) @@ -927,7 +935,7 @@ async def mlat_history( "n_records": len(records), "records": _cap_per_lane(records, limit), } - return Response(content=orjson.dumps(payload), media_type="application/json") + return Response(content=orjson.dumps(without_receiver_geometry(payload)), media_type="application/json") norm = (hex or "").strip().lower() if not norm: @@ -966,7 +974,7 @@ async def mlat_history( "records": rejects_nearby[:200], }, } - return Response(content=orjson.dumps(payload), media_type="application/json") + return Response(content=orjson.dumps(without_receiver_geometry(payload)), media_type="application/json") # ── Solver Report (full funnel/error/ghost/consensus picture) ───────────────── @@ -1722,7 +1730,7 @@ async def node_detection_range(node_id: str): status_code=404, ) - summary = {k: v for k, v in area.summary().items() if k != "furthest_detections"} + summary = without_receiver_geometry(area.summary()) rx = summary.get("rx") or {} pub_lat, pub_lon = public_latlon(rx.get("lat"), rx.get("lon"), node_id) summary["rx"] = {**rx, "lat": pub_lat, "lon": pub_lon} diff --git a/backend/services/public_geometry.py b/backend/services/public_geometry.py new file mode 100644 index 00000000..67009e10 --- /dev/null +++ b/backend/services/public_geometry.py @@ -0,0 +1,99 @@ +"""Receiver-relative measurements withheld from unauthenticated payloads. + +Every published receiver coordinate is displaced by services/public_location.py. +A quantity measured from a node's TRUE receiver to a point the same payload +gives the position of hands that displacement straight back, and enough of +them intersect well inside it. The line, applied field by field below: a +per-node constant is the envelope and may be published; a per-record value +that varies with the true geometry is a measurement and is withheld. So a beam +entry keeps `max_range_km`, `max_bistatic_range_km` and `half_width_deg`, +which /api/radar/analytics already carries per node, and `rule`, which names +which of them a refused solve breached and so refines an exclusion rather than +bounding the receiver. + +The pass is structural over containers, so a field named below is withheld at +any nesting depth. It matches on the leaf key alone, though, not on shape or +position: the same quantity published under a name this module has not seen +stays published until that name is added. +""" + +from __future__ import annotations + +from typing import Any + +# Withheld wherever they appear. +_RECEIVER_RELATIVE = frozenset( + { + # Beam-gate margins (solve records, beam_failures[]): a range and a + # bearing from the true receiver to an aircraft the same record + # locates. bistatic_km joins them because the transmitter is published + # untranslated, so the differential range fixes the receiver on a + # hyperbola through two known foci. + "range_km", + "bearing_off_deg", + "bistatic_km", + # Learned-FOV read-outs at the true bearing. The curve they are read + # off is itself published, as `empirical_polygon` on + # /api/radar/analytics, so a value off it inverts to the bearing it was + # read at; "closed" is the same channel at three-value resolution. + "fov_limit_km", + "fov_state", + # FOV shadow-mode verdicts. Containment tests at the true bearing and + # range, stamped for the nodes that PASSED as well as the ones that + # failed, so each bounds the receiver to a region where a failure would + # only exclude one. + "fov_verdict", + "today_pass", + # Cluster contamination. Membership is _point_in_beam against the true + # geometry and the record publishes that point beside the verdict, so + # it is another labelled sample of the published beam. + # + # /api/test/solver-stats does not route through this pass, and must + # not: its contamination.contaminated is a windowed count sharing the + # name rather than this field, and the pass would delete it. + "foreign_node_ids", + "contaminated", + # Real aircraft fixes each carrying their distance from the true + # receiver: a ranging circle per detection, and three intersect. + "furthest_detections", + # Per-node verification tracks. measured_delay_us is the bistatic range + # from the true receiver to the truth position beside it, and + # delay_match_us its residual against the range predicted from that + # position, so the predicted one follows from the pair. + "measured_delay_us", + "delay_match_us", + # The angle at the truth position between the published transmitter and + # the true receiver of the worst-geometry contributing node, which + # names the direction from a known point towards a receiver. + "max_bistatic_angle_deg", + } +) + +# Withheld additionally from a payload scoped to ONE node, where a track +# position is that node's own single-node solve. The aircraft feed publishes +# those same two fields displaced with the icon (services/track_gates.py), so +# the true frame beside them is the displacement by one subtraction. A +# multinode payload keeps them: no single receiver is behind that position. +_NODE_SCOPED = _RECEIVER_RELATIVE | {"solver_lat", "solver_lon"} + + +def _stripped(value: Any, fields: frozenset[str]) -> Any: + if isinstance(value, dict): + return {k: _stripped(v, fields) for k, v in value.items() if k not in fields} + if isinstance(value, list | tuple): + return [_stripped(v, fields) for v in value] + return value + + +def without_receiver_geometry(value: Any, *, node_scoped: bool = False) -> Any: + """`value` with the receiver-relative measurements taken out, at any depth. + + `node_scoped` for a payload that answers for one node, where a track + position is that node's own. + + Copies rather than editing in place, so the caller's own structure keeps + the geometry: the per-node verification store and the rolling MLAT sample + buffer hold what the route may not publish. Tuples come back as lists, + which is what they would have serialised as anyway. + """ + return _stripped(value, _NODE_SCOPED if node_scoped else _RECEIVER_RELATIVE) diff --git a/backend/services/tasks/analytics_refresh.py b/backend/services/tasks/analytics_refresh.py index 8ef3ba37..c4d73948 100644 --- a/backend/services/tasks/analytics_refresh.py +++ b/backend/services/tasks/analytics_refresh.py @@ -26,6 +26,7 @@ from services.id_utils import multinode_hex_from_key from services.node_config import position_status from services.node_sites import log_colocation_audit +from services.public_geometry import without_receiver_geometry from services.public_location import ( fuzz_enabled, location_uncertainty_km, @@ -1280,6 +1281,19 @@ def _refresh_mlat_accuracy_stats() -> None: ) +def _publish_mlat_verification(result: dict) -> None: + """Serialise a verification result onto the store the public route serves. + + Both writers go through here so a third cannot reach those bytes without + the receiver-geometry pass. The store's only readers are unauthenticated + (GET /api/test/mlat-verification, and the dashboard summary beside it), so + nothing downstream wants the withheld fields back. + """ + state.latest_mlat_verification_bytes = orjson.dumps( + without_receiver_geometry(result), option=orjson.OPT_SERIALIZE_NUMPY + ) + + def _refresh_mlat_verification(): """Compare multinode solve results to ground-truth trails pushed by the fleet orchestrator. @@ -1406,7 +1420,7 @@ def _refresh_mlat_verification(): # /api/test/mlat-accuracy silently serves numbers frozen at the moment # the truth feed stopped, with nothing marking them stale. _refresh_mlat_accuracy_stats() - state.latest_mlat_verification_bytes = orjson.dumps( + _publish_mlat_verification( { "computed_at": round(now, 1), "skip_reason": "no_truth_candidates", @@ -1428,8 +1442,7 @@ def _refresh_mlat_verification(): "nearest_truth": {"mean_km": None, "median_km": None, "p95_km": None}, "tracks": [], }, - }, - option=orjson.OPT_SERIALIZE_NUMPY, + } ) return @@ -1752,7 +1765,7 @@ def _min_truth_dist_km(kv: tuple) -> float: "tracks": sorted(unmatched, key=lambda x: x.get("nearest_truth_km") or 999)[:50], }, } - state.latest_mlat_verification_bytes = orjson.dumps(result, option=orjson.OPT_SERIALIZE_NUMPY) + _publish_mlat_verification(result) def _ensure_custody_data(): diff --git a/backend/tests/test_public_geometry.py b/backend/tests/test_public_geometry.py new file mode 100644 index 00000000..0ac3bdb6 --- /dev/null +++ b/backend/tests/test_public_geometry.py @@ -0,0 +1,260 @@ +"""services/public_geometry.py, and the routes that publish through it. + +The module's own docstring carries the rule and the per-field reasoning; these +assert it, and that every unauthenticated surface actually applies it. +""" + +import time + +import orjson +import pytest +from fastapi.testclient import TestClient + +from core import state +from services.public_geometry import _RECEIVER_RELATIVE, without_receiver_geometry +from services.tasks.analytics_refresh import _publish_mlat_verification + + +class TestWhatIsWithheld: + def test_a_beam_margin_goes_and_its_envelope_stays(self): + """The margin varies with the true geometry; the envelope is the + per-node constant /api/radar/analytics already publishes.""" + entry = { + "node_id": "ret1a2b3c4d", + "range_km": 6.2, + "bearing_off_deg": 73.9, + "bistatic_km": 41.0, + "max_range_km": 59.9, + "half_width_deg": 21.0, + "rule": "bearing", + } + assert without_receiver_geometry(entry) == { + "node_id": "ret1a2b3c4d", + "max_range_km": 59.9, + "half_width_deg": 21.0, + "rule": "bearing", + } + + def test_the_learned_fov_read_outs_go(self): + out = without_receiver_geometry({"fov_limit_km": 40.0, "fov_state": "closed", "keep": 1}) + assert out == {"keep": 1} + + def test_the_shadow_verdicts_go(self): + out = without_receiver_geometry({"fov_verdict": [{"today_pass": True}], "keep": 1}) + assert out == {"keep": 1} + + def test_the_contamination_verdict_goes(self): + out = without_receiver_geometry({"foreign_node_ids": ["ret1a2b3c4d"], "contaminated": True, "keep": 1}) + assert out == {"keep": 1} + + def test_the_per_node_delays_go(self): + out = without_receiver_geometry({"measured_delay_us": 120.0, "delay_match_us": 0.4, "keep": 1}) + assert out == {"keep": 1} + + def test_the_furthest_detections_go(self): + out = without_receiver_geometry({"furthest_detections": [{"distance_km": 71.2}], "keep": 1}) + assert out == {"keep": 1} + + +class TestHowItWalks: + def test_a_withheld_field_goes_at_any_depth(self): + payload = {"a": [{"b": {"c": [{"range_km": 1.0, "keep": 2}]}}]} + assert without_receiver_geometry(payload) == {"a": [{"b": {"c": [{"keep": 2}]}}]} + + def test_a_tuple_comes_back_as_a_list(self): + """Which is what orjson would have serialised it as anyway.""" + assert without_receiver_geometry({"a": ({"range_km": 1.0, "keep": 2},)}) == {"a": [{"keep": 2}]} + + def test_the_input_is_not_edited(self): + payload = {"range_km": 1.0, "keep": 2} + without_receiver_geometry(payload) + assert payload == {"range_km": 1.0, "keep": 2} + + def test_a_payload_with_nothing_withheld_is_unchanged(self): + payload = {"max_range_km": 59.9, "tracks": [{"hex": "abc123"}]} + assert without_receiver_geometry(payload) == payload + + +class TestNodeScoped: + def test_the_nodes_own_solve_goes_when_scoped(self): + out = without_receiver_geometry({"solver_lat": 34.9, "solver_lon": -82.4, "keep": 1}, node_scoped=True) + assert out == {"keep": 1} + + def test_it_stays_when_not_scoped(self): + """A multinode position has no single receiver behind it.""" + payload = {"solver_lat": 34.9, "solver_lon": -82.4, "keep": 1} + assert without_receiver_geometry(payload) == payload + + def test_scoping_still_withholds_everything_else(self): + out = without_receiver_geometry({"range_km": 1.0, "solver_lat": 34.9, "keep": 1}, node_scoped=True) + assert out == {"keep": 1} + + +# ── The surfaces ────────────────────────────────────────────────────────────── +# Route-level, because the helper being correct proves nothing about whether a +# route calls it: every one of these fails if its call site is dropped. + +_NODE_ID = "retdeadbeef" +_HEX = "mn0123456789" + +# The receiver-relative fields as the solver actually spells them on a record +# (services/tasks/solver.py) and on a verification track (analytics_refresh.py). +_WITHHELD_ON_A_SOLVE = { + "range_km": 6.2, + "bearing_off_deg": 73.9, + "bistatic_km": 41.0, + "fov_limit_km": 40.0, + "fov_state": "closed", + "foreign_node_ids": ["ret1a2b3c4d"], + "contaminated": True, +} + + +def _client() -> TestClient: + from main import app + + return TestClient(app) + + +def _solve_record(**over) -> dict: + rec = { + "ts_ms": int(time.time() * 1000), + "solve_key": "mn-dark-0123456789", + "solver_hex": _HEX, + "outcome": "published", + "raw_lat": 34.85, + "raw_lon": -82.39, + "n_nodes": 2, + "beam_failures": [{"node_id": "ret1a2b3c4d", "max_range_km": 59.9, **_WITHHELD_ON_A_SOLVE}], + **_WITHHELD_ON_A_SOLVE, + } + rec.update(over) + return rec + + +def _names(value) -> set[str]: + """Every key appearing anywhere in a decoded payload.""" + if isinstance(value, dict): + return set(value) | {k for v in value.values() for k in _names(v)} + if isinstance(value, list): + return {k for v in value for k in _names(v)} + return set() + + +@pytest.fixture +def clean_state(): + solves, known = list(state.mlat_solve_history), list(state.mlat_solve_history_known) + skips = list(state.solver_resolve_skips_recent) + node_bytes = dict(state.latest_node_verification_bytes) + mlat_bytes = state.latest_mlat_verification_bytes + state.mlat_solve_history.clear() + state.mlat_solve_history_known.clear() + state.solver_resolve_skips_recent.clear() + state.latest_node_verification_bytes.clear() + yield + state.mlat_solve_history.clear() + state.mlat_solve_history.extend(solves) + state.mlat_solve_history_known.clear() + state.mlat_solve_history_known.extend(known) + state.solver_resolve_skips_recent.clear() + state.solver_resolve_skips_recent.extend(skips) + state.latest_node_verification_bytes.clear() + state.latest_node_verification_bytes.update(node_bytes) + state.latest_mlat_verification_bytes = mlat_bytes + + +@pytest.mark.usefixtures("clean_state") +class TestTheRoutesApplyIt: + def test_the_whole_window_dump_withholds(self): + state.mlat_solve_history.append(_solve_record()) + data = _client().get("/api/test/mlat-history?all=1").json() + assert data["n_records"] == 1 + assert _names(data) & set(_WITHHELD_ON_A_SOLVE) == set() + + def test_one_markers_solves_withhold(self): + state.mlat_solve_history.append(_solve_record()) + data = _client().get(f"/api/test/mlat-history?hex={_HEX}").json() + assert data["n_solves"] == 1 + assert _names(data) & set(_WITHHELD_ON_A_SOLVE) == set() + + def test_the_nearby_rejects_withhold(self): + """A second payload key on the same response, and the one the flat + per-record pass reached only because someone remembered it.""" + state.mlat_solve_history.append(_solve_record()) + state.mlat_solve_history.append(_solve_record(outcome="rejected_rms", solver_hex="mn9999999999")) + data = _client().get(f"/api/test/mlat-history?hex={_HEX}").json() + assert data["rejects_nearby"]["n"] == 1 + assert _names(data) & set(_WITHHELD_ON_A_SOLVE) == set() + + def test_the_resolve_skips_dump_withholds(self): + state.solver_resolve_skips_recent.append( + {"ts_ms": int(time.time() * 1000), "lane": "dark", **_WITHHELD_ON_A_SOLVE} + ) + data = _client().get("/api/test/mlat-history?kind=resolve_skips").json() + assert data["n_records"] == 1 + assert _names(data) & set(_WITHHELD_ON_A_SOLVE) == set() + + def test_a_nodes_verification_withholds_its_own_solve_too(self): + """Node-scoped: the track position IS that node's single-node solve.""" + state.latest_node_verification_bytes[_NODE_ID] = orjson.dumps( + { + "node_id": _NODE_ID, + "n_matched": 1, + "tracks": [ + { + "hex": "a1b2c3", + "measured_delay_us": 120.0, + "delay_match_us": 0.4, + "solver_lat": 34.9, + "solver_lon": -82.4, + "truth_lat": 34.91, + "truth_lon": -82.41, + "position_error_km": 1.2, + } + ], + } + ) + data = _client().get(f"/api/test/node/{_NODE_ID}/verification").json() + track = data["tracks"][0] + assert set(track) == {"hex", "truth_lat", "truth_lon", "position_error_km"} + + def test_an_unknown_node_still_answers_empty(self): + assert _client().get("/api/test/node/retnosuchxx/verification").json() == {} + + def test_the_multinode_verification_withholds_on_write(self): + """Stripped onto the store rather than at the route, because both of + that store's readers are unauthenticated.""" + _publish_mlat_verification( + { + "n_matched": 1, + "tracks": [{"truth_hex": "a1b2c3", "max_bistatic_angle_deg": 151.3, "solver_lat": 34.9}], + } + ) + data = orjson.loads(state.latest_mlat_verification_bytes) + # solver_lat stays: a multinode position, not one node's receiver. + assert data["tracks"][0] == {"truth_hex": "a1b2c3", "solver_lat": 34.9} + + +class TestTheDetectionAreaSummary: + """node_detection_range runs the pass over DetectionAreaState.summary(), + whose keys come from the retina_analytics submodule rather than this repo. + The pass matches on leaf key name, so a field added there under one of the + withheld names would vanish from the public payload with nothing saying so. + This pins the overlap, so an upgrade that collides fails here instead. + """ + + def _area(self): + from retina_analytics.detection_area import DetectionAreaState + + area = DetectionAreaState(node_id="ret1a2b3c4d", rx_lat=34.9, rx_lon=-82.4) + area.update(120.0, 30.0) + area.record_verified_detection(35.4, -82.9, "a1b2c3") + return area + + def test_only_furthest_detections_collides(self): + assert set(self._area().summary()) & _RECEIVER_RELATIVE == {"furthest_detections"} + + def test_so_the_pass_drops_that_and_nothing_else(self): + summary = self._area().summary() + assert summary["furthest_detections"], "the fixture must exercise the field being dropped" + assert without_receiver_geometry(summary) == {k: v for k, v in summary.items() if k != "furthest_detections"}