From adbc7136c8a6ebb2af721100ce8a0487163e82ac Mon Sep 17 00:00:00 2001 From: jehanazad Date: Thu, 10 Sep 2026 19:51:21 +0000 Subject: [PATCH] Seed the world with live ADS-B traffic from adsb.retina.fm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real aircraft over the fleet's metro now join the simulated world and are echoed by the synthetic nodes: a new LiveAdsbClient polls an adsb.lol-shaped feed (/v2/point/{lat}/{lon}/{radius_nm}; adsb.retina.fm serves it), and SimulationWorld.ingest_live_aircraft turns each row into a `live-` aircraft flying its real position, altitude, ground speed, track and vertical rate, extrapolated from the fix's capture time and dead-reckoned between polls. The feed owns these aircraft: no waypoints, no separation slowing, no lifetime — they leave when the feed stops reporting them. Two independent knobs: - frac_live_dark casts a share of the LIVE aircraft as dark — mirrored without their transponder (has_adsb off, adsb_hex None), so the node frames carry no tag for them, the ADS-B push omits them, and the ground truth keys them by object id, exactly like a synthetic dark spawn. The cast is a stable per-hex hash, so moving the knob re-partitions the aircraft already in the air (raising it only adds dark aircraft) instead of re-rolling them, and set_frac_live_dark applies it at the next config poll rather than the next fleet turnover. - min/max_aircraft and frac_dark keep governing the SYNTHETIC spawns layered on top: spawning counts synthetic aircraft only, so the feed's population never starves synthetic spawns or vice versa. Orchestrator: --live-adsb-url / --live-adsb-interval start _seed_live_adsb over the --metro/--metros areas; the backend's frac_live_dark and live_adsb_enabled ride the existing /api/simulation/config poll (pausing clears the live aircraft). Ground-truth payloads carry source=live|sim; the ADS-B push now carries the callsign. --real-adsb is ignored while seeding runs — both would push the same hexes into one cache under two world tags. Co-Authored-By: Claude Fable 5.1 --- retina_simulation/live_adsb.py | 147 ++++++++++++++ retina_simulation/orchestrator.py | 147 +++++++++++++- retina_simulation/world.py | 218 ++++++++++++++++++++- tests/test_dual_fraction.py | 7 + tests/test_live_adsb.py | 310 ++++++++++++++++++++++++++++++ 5 files changed, 813 insertions(+), 16 deletions(-) create mode 100644 retina_simulation/live_adsb.py create mode 100644 tests/test_live_adsb.py diff --git a/retina_simulation/live_adsb.py b/retina_simulation/live_adsb.py new file mode 100644 index 0000000..3e716f0 --- /dev/null +++ b/retina_simulation/live_adsb.py @@ -0,0 +1,147 @@ +"""Live ADS-B client — pulls real aircraft from adsb.retina.fm to seed the world. + +The endpoint speaks the adsb.lol ``/v2/point/{lat}/{lon}/{radius_nm}`` shape +(tar1090 aircraft objects under ``ac``), so the parsing here is the same as +the backend's adsb.lol client; it is a separate, dependency-free copy because +this package must not import the backend, and because the two feeds serve +different purposes: the backend polls adsb.lol for *external truth* to score +nodes against, while this client feeds ``SimulationWorld.ingest_live_aircraft`` +so that the synthetic nodes fly real trajectories. + +Rows come back normalised to what the world needs, with ``captured_at`` as an +absolute wall-clock time (``seen_pos`` resolved against the fetch), so a +dead-reckoned position can be extrapolated to "now" on ingest. +""" + +import gzip +import json +import logging +import time +import urllib.error +import urllib.request + +log = logging.getLogger(__name__) + +DEFAULT_BASE_URL = "https://adsb.retina.fm" +_TIMEOUT_S = 8 +_USER_AGENT = "retina-simulation/1.0 (+https://github.com/offworldlabs/retina-simulation)" +# One last-good result per area may stand in for a failed fetch this long. +# Longer than the world's own staleness window (LIVE_STALE_S) is pointless: +# the world would drop the aircraft anyway before the cache stopped serving. +_CACHE_MAX_AGE_S = 90.0 + + +def _num(v, default=0.0) -> float: + """Coerce a tar1090 numeric field; sentinels like ``"ground"`` → default.""" + if isinstance(v, bool): + return default + if isinstance(v, (int, float)): + return float(v) + return default + + +def parse_point_response(data: dict, fetched_at: float) -> list[dict]: + """Normalise one ``/v2/point`` payload into world-ready rows. + + Rows without a position, and aircraft reported on the ground (tar1090's + ``alt_baro: "ground"``), are dropped: a parked airliner is not a radar + target the fleet should be echoing at zero altitude. + """ + rows = [] + for ac in data.get("ac", []) or []: + if not isinstance(ac, dict): + continue + hex_code = str(ac.get("hex") or "").strip().lower() + lat = ac.get("lat") + lon = ac.get("lon") + if not hex_code or not isinstance(lat, (int, float)) or not isinstance(lon, (int, float)): + continue + alt_baro = ac.get("alt_baro") + if alt_baro == "ground" or not isinstance(alt_baro, (int, float)): + continue + seen_pos = ac.get("seen_pos") + captured_at = fetched_at - seen_pos if isinstance(seen_pos, (int, float)) else fetched_at + rows.append( + { + "hex": hex_code, + "flight": (ac.get("flight") or "").strip(), + "lat": float(lat), + "lon": float(lon), + "alt_baro": float(alt_baro), # ft + "gs": _num(ac.get("gs")), # knots + "track": _num(ac.get("track")), # deg + "baro_rate": _num(ac.get("baro_rate")), # ft/min + "captured_at": captured_at, # epoch s + } + ) + return rows + + +class LiveAdsbClient: + """Polls adsb.retina.fm (or any adsb.lol-shaped server) for one or more areas.""" + + def __init__(self, areas: list[dict], base_url: str = DEFAULT_BASE_URL): + """ + Args: + areas: dicts with name, lat, lon and optional radius_nm (default 80). + base_url: server root; ``/v2/point/...`` is appended. + """ + self.base_url = base_url.rstrip("/") + self.areas = [a for a in areas if isinstance(a, dict) and "lat" in a and "lon" in a] + self._cache: dict[str, list[dict]] = {} + self._cache_ts: dict[str, float] = {} + self.last_status: dict[str, bool] = {} + self.last_error: str | None = None + + def _url(self, area: dict) -> str: + return f"{self.base_url}/v2/point/{area['lat']}/{area['lon']}/{area.get('radius_nm', 80)}" + + def _get(self, url: str) -> dict: + req = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip", + "User-Agent": _USER_AGENT, + }, + ) + with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp: # noqa: S310 — https URL built from config + raw = resp.read() + if (resp.headers or {}).get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return json.loads(raw) + + def _last_good(self, name: str) -> list[dict]: + ts = self._cache_ts.get(name) + if ts is None or time.monotonic() - ts > _CACHE_MAX_AGE_S: + self._cache.pop(name, None) + self._cache_ts.pop(name, None) + return [] + return self._cache.get(name, []) + + def fetch_area(self, area: dict) -> list[dict]: + name = str(area.get("name") or f"{area['lat']},{area['lon']}") + try: + data = self._get(self._url(area)) + rows = parse_point_response(data, time.time()) + except (urllib.error.URLError, OSError, ValueError) as e: + self.last_status[name] = False + self.last_error = str(e) + return self._last_good(name) + self._cache[name] = rows + self._cache_ts[name] = time.monotonic() + self.last_status[name] = True + self.last_error = None + return rows + + def fetch_all(self) -> list[dict]: + """Every configured area, deduplicated by hex (first area wins).""" + seen: set[str] = set() + out: list[dict] = [] + for area in self.areas: + for row in self.fetch_area(area): + if row["hex"] in seen: + continue + seen.add(row["hex"]) + out.append(row) + return out diff --git a/retina_simulation/orchestrator.py b/retina_simulation/orchestrator.py index 5af3af3..26b88bc 100644 --- a/retina_simulation/orchestrator.py +++ b/retina_simulation/orchestrator.py @@ -39,6 +39,7 @@ fleet_summary, generate_fleet, ) +from retina_simulation.live_adsb import LiveAdsbClient from retina_simulation.tower_resolver import apply_tower_assignments, resolve_towers from retina_simulation.world import ( MetroCell, @@ -266,6 +267,10 @@ def __init__( self.metro_traffic_frac = min(1.0, max(0.0, metro_traffic_frac)) self.connections: dict[str, NodeConnection] = {} self.world: SimulationWorld | None = None + # Runtime switch for the live ADS-B seed task (_seed_live_adsb): + # the task only runs when main_async wires it, but the backend's + # simulation config can pause it without a restart. + self.live_adsb_enabled = True self._running = False self._stats = { "total_frames": 0, @@ -651,6 +656,7 @@ def get_stats(self) -> dict: "frames_per_sec": round(self._stats["total_frames"] / max(elapsed, 1), 1), "detections_per_sec": round(self._stats["total_detections"] / max(elapsed, 1), 1), "aircraft_count": len(self.world.aircraft) if self.world else 0, + "live_aircraft_count": len(self.world.live_aircraft) if self.world else 0, "ground_truth_snapshots": len(self.ground_truth), } @@ -696,6 +702,10 @@ def build_ground_truth_payload(aircraft_summaries: list[dict]) -> list[dict]: "adsb_silent": ac.get("adsb_silent", False), "adsb_callsign": ac.get("adsb_callsign") or None, "anomaly_event": ac.get("anomaly_event") or None, + # "live" for an aircraft mirrored from the ADS-B feed, "sim" + # for the world's own spawns — so the server can count and + # colour the two populations apart. + "source": ac.get("source") or "sim", } ) return payload_aircraft @@ -745,7 +755,7 @@ def build_adsb_push_payload(aircraft_summaries: list[dict]) -> list[dict]: payload_aircraft.append( { "hex": hex_code, - "flight": "", + "flight": ac.get("adsb_callsign") or "", "lat": round(ac["lat"], 5), "lon": round(ac["lon"], 5), "alt_baro": round(ac["alt_km"] * 1000 / 0.3048), @@ -890,6 +900,70 @@ async def _push_real_adsb( log.debug("Real ADSB push failed: %s", e) +async def _seed_live_adsb( + orchestrator: FleetOrchestrator, + feed_url: str, + areas: list[dict], + interval_s: float = 5.0, +): + """Pull real aircraft from the live feed into the world every interval_s. + + This is what makes the synthetic nodes fly real traffic: every aircraft + the feed reports over the metro becomes a world aircraft the nodes echo + (delay/Doppler from its real position and velocity), cast as ADS-B or + dark by frac_live_dark. Unlike the opt-in adsb.lol relay + (_push_real_adsb), which only decorates the map with a second world's + positions, these aircraft ARE in the simulated world — so their ADS-B + tags reach the server through the ordinary node frames and 1 Hz push, + untagged, as simulated-world traffic. + + The backend's live_adsb_enabled flag (via _poll_simulation_config) + pauses the pull and clears the live aircraft; re-enabling resumes on the + next poll. A failed fetch leaves the world alone: aircraft coast on + their last velocity and expire only after the world's staleness window. + """ + client = LiveAdsbClient(areas, base_url=feed_url) + loop = asyncio.get_event_loop() + log.info( + "Live ADS-B seeding started (feed=%s, %d area(s), interval=%.0fs)", + feed_url, + len(client.areas), + interval_s, + ) + last_live = -1 + was_enabled = True + while orchestrator._running: + await asyncio.sleep(interval_s) + if orchestrator.world is None: + continue + if not orchestrator.live_adsb_enabled: + if was_enabled: + n = orchestrator.world.clear_live_aircraft() + log.info("Live ADS-B seeding paused by simulation config (%d live aircraft removed)", n) + was_enabled = False + continue + was_enabled = True + try: + rows = await loop.run_in_executor(None, client.fetch_all) + except Exception as e: # defensive: the client swallows its own errors + log.debug("Live ADS-B fetch failed: %s", e) + continue + if not any(client.last_status.values()): + log.warning("Live ADS-B feed unreachable (%s) — live aircraft coasting", client.last_error) + continue + stats = orchestrator.world.ingest_live_aircraft(rows) + if stats["live"] != last_live: + n_dark = sum(1 for ac in orchestrator.world.live_aircraft.values() if not ac.has_adsb) + log.info( + "Live ADS-B: %d aircraft in world (%d dark, frac_live_dark=%.2f; +%d new this poll)", + stats["live"], + n_dark, + orchestrator.world.frac_live_dark, + stats["created"], + ) + last_live = stats["live"] + + async def _poll_simulation_config( orchestrator: FleetOrchestrator, base_url: str, @@ -955,15 +1029,28 @@ def _fetch(): orchestrator.world.min_aircraft = int(cfg["min_aircraft"]) if "max_aircraft" in cfg: orchestrator.world.max_aircraft = int(cfg["max_aircraft"]) + # Live feed knobs. Fallbacks match the world/orchestrator + # defaults (0.0 dark, seeding on) so an older backend that + # ships neither key changes nothing. set_frac_live_dark + # re-casts the aircraft already in the air, so the slider + # takes effect at the next poll rather than the next fleet + # turnover. + n_live_dark = orchestrator.world.set_frac_live_dark(float(cfg.get("frac_live_dark", 0.0))) + orchestrator.live_adsb_enabled = bool(cfg.get("live_adsb_enabled", True)) last_updated_at = updated_at log.info( - "Simulation config updated: anomalous=%.2f drone=%.2f dark=%.2f adsb_outage=%.2f aircraft=%d–%d", + "Simulation config updated: anomalous=%.2f drone=%.2f dark=%.2f adsb_outage=%.2f " + "aircraft=%d–%d live_adsb=%s live_dark=%.2f (%d/%d live aircraft dark)", orchestrator.world.frac_anomalous, orchestrator.world.frac_drone, orchestrator.world.frac_dark, orchestrator.world.frac_adsb_outage, orchestrator.world.min_aircraft, orchestrator.world.max_aircraft, + "on" if orchestrator.live_adsb_enabled else "off", + orchestrator.world.frac_live_dark, + n_live_dark, + len(orchestrator.world.live_aircraft), ) # Scene-change detection. Absent scene stamp (stale volume, @@ -1296,18 +1383,43 @@ def _near_any_metro(node): # time metro config alone switched this on — the "realistic mix" arrived # with ghost planes attached. adsb_metros = getattr(args, "metros", "") or getattr(args, "metro", "") or "" - if getattr(args, "real_adsb", False) and args.validation_url and adsb_metros: - metro_areas = _parse_metro_areas(adsb_metros) - if metro_areas: + # Live ADS-B seeding: real aircraft over the metro join the simulated + # world and are echoed by the synthetic nodes (see _seed_live_adsb). + # Needs an area to query, so it is scoped like the relay below. + live_url = (getattr(args, "live_adsb_url", "") or "").strip() + live_seeding = False + if live_url and adsb_metros: + live_areas = _parse_metro_areas(adsb_metros) + if live_areas: + live_seeding = True tasks.append( - _push_real_adsb( + _seed_live_adsb( orchestrator, - args.validation_url, - areas=metro_areas, - interval_s=10.0, + live_url, + areas=live_areas, + interval_s=max(1.0, float(getattr(args, "live_adsb_interval", 5.0) or 5.0)), ) ) - elif adsb_metros and args.validation_url: + elif live_url: + log.warning("Live ADS-B seeding needs --metro/--metros to know where to look — disabled") + if getattr(args, "real_adsb", False) and args.validation_url and adsb_metros: + if live_seeding: + # Both would push the same real hexes, one tagged source=real and + # one as simulated-world traffic, and the server keys its cache + # by hex — the two writers would take turns owning each entry. + log.warning("--real-adsb ignored: live ADS-B seeding already carries real traffic in the simulated world") + else: + metro_areas = _parse_metro_areas(adsb_metros) + if metro_areas: + tasks.append( + _push_real_adsb( + orchestrator, + args.validation_url, + areas=metro_areas, + interval_s=10.0, + ) + ) + elif adsb_metros and args.validation_url and not live_seeding: log.info("Real ADS-B relay disabled (pass --real-adsb to inject adsb.lol traffic)") if args.validate and args.validation_url: @@ -1424,6 +1536,21 @@ def main(): "config alone, and the decoy transponders it added produced ghost " "planes on the map.", ) + parser.add_argument( + "--live-adsb-url", + type=str, + default="", + help="Base URL of an adsb.lol-shaped live feed (e.g. https://adsb.retina.fm). " + "Real aircraft over the --metro/--metros areas then join the simulated " + "world and are echoed by the synthetic nodes; the backend's " + "frac_live_dark casts a share of them as dark. Empty = off.", + ) + parser.add_argument( + "--live-adsb-interval", + type=float, + default=5.0, + help="Seconds between live feed polls (min 1)", + ) parser.add_argument( "--no-hub-radial", action="store_true", diff --git a/retina_simulation/world.py b/retina_simulation/world.py index 1bab0e3..db1d0de 100644 --- a/retina_simulation/world.py +++ b/retina_simulation/world.py @@ -24,6 +24,7 @@ import json import math import random +import time from dataclasses import asdict, dataclass, field C_KM_US = 0.299792458 # speed of light km/μs @@ -119,6 +120,10 @@ def waypoints_for_metro(metro: str | None) -> list[tuple[float, float]]: # sooner: verification should not have to wait for a whole fleet turnover. _ADSB_OUTAGE_RUNTIME_START_S = (5.0, 60.0) +# Unit conversions for the feed's tar1090 fields. +_KNOTS_TO_KM_S = 1.852 / 3600.0 +_FT_TO_KM = 0.0003048 + @dataclass class SimulatedAircraft: @@ -172,6 +177,18 @@ class SimulatedAircraft: # World clock as of the last step(), so adsb_silent can be a plain # property: the aircraft has no back-reference to its world. sim_now_s: float = 0.0 + # Provenance. "sim" = spawned by the world's own planner; "live" = a real + # aircraft mirrored from an ADS-B feed (SimulationWorld.ingest_live_aircraft). + # A live aircraft flies its real track: no waypoints, no separation + # slowing, no lifetime — it leaves when the feed stops reporting it. + source: str = "sim" + # The real transponder hex of a live aircraft, kept even while the world + # casts it as dark (adsb_hex is None then), so the cast can flip back + # without a re-ingest and the feed can find the aircraft again by key. + live_hex: str | None = None + # World clock at the last feed update; a live aircraft not refreshed for + # SimulationWorld.live_stale_s is dropped. + live_seen_s: float = 0.0 @property def adsb_silent(self) -> bool: @@ -401,6 +418,20 @@ def __init__( # default so nothing changes for anyone who does not set it. self.frac_adsb_outage: float = 0.0 # remaining fraction = commercial aircraft with ADS-B + # + # Live-derived traffic (ingest_live_aircraft): the share of feed + # aircraft the world casts as DARK — mirrored without their + # transponder, so the synthetic nodes echo a real trajectory the + # server holds no ADS-B tag for. Orthogonal to frac_dark, which only + # rolls the world's own synthetic spawns, and to min/max_aircraft, + # which only count them: the feed decides how many live aircraft + # there are. The cast is a stable per-hex hash (live_role_is_dark), + # so moving the knob re-partitions the aircraft already in the air + # instead of re-rolling them. + self.frac_live_dark: float = 0.0 + # Feed aircraft not refreshed for this long (world seconds) are dropped. + self.live_stale_s: float = 60.0 + self.live_aircraft: dict[str, SimulatedAircraft] = {} # Hub-radial flight planning: when metro_cells is non-empty, this # fraction of spawns is routed through a metro cell; the rest are # en-route traffic on the inter-metro waypoint net. Empty cells → @@ -750,16 +781,25 @@ def step(self, dt: float, mode: str = "detection"): # Expired non-drones head for the edge before the retire filter sees # them past the edge (see _route_out / _should_retire). for ac in self.aircraft: + if ac.source == "live": + continue if not ac.departing and ac.object_type != "drone" and self._time - ac.created_at >= ac.lifetime_s: self._route_out(ac) - # Retire expired aircraft (edge-gated — see _should_retire) - self.aircraft = [ac for ac in self.aircraft if not self._should_retire(ac)] - - # Spawn to maintain target count - while len(self.aircraft) < self.min_aircraft: + # Retire expired aircraft (edge-gated — see _should_retire). Live + # aircraft leave when the feed stops reporting them instead. + self._expire_live_aircraft() + self.aircraft = [ac for ac in self.aircraft if ac.source == "live" or not self._should_retire(ac)] + + # Spawn to maintain target count — synthetic aircraft only. The feed + # sets the live population; min/max_aircraft stay the operator's + # knobs for the synthetic traffic layered on top of it, so raising + # live traffic never starves synthetic spawns and vice versa. + n_synth = self.synthetic_count() + while n_synth < self.min_aircraft: self.aircraft.append(self._spawn_aircraft(mode)) - if len(self.aircraft) < self.max_aircraft and random.random() < 0.01: + n_synth += 1 + if n_synth < self.max_aircraft and random.random() < 0.01: self.aircraft.append(self._spawn_aircraft(mode)) # Update each aircraft @@ -807,6 +847,10 @@ def _enforce_separation(self, dt: float): break blend = 1.0 - math.exp(-dt / 3.0) for ac in flow: + if ac.source == "live": + # Real traffic is a leader for synthetic trailers to yield + # to, never a trailer: its speed is the feed's to set. + continue base = ac.base_speed_km_s or ac.speed_km_s target = base * 0.7 if ac.object_id in slowed else base ac.speed_km_s += (target - ac.speed_km_s) * blend @@ -836,6 +880,14 @@ def _maybe_schedule_anomaly(self, is_anomalous: bool, object_type: str) -> dict: def _update_aircraft(self, ac: SimulatedAircraft, dt: float): """Update aircraft position and navigate toward waypoints.""" + if ac.source == "live": + # The feed owns the trajectory; between polls the aircraft + # coasts on its last reported velocity. No waypoints, no + # perturbation, no anomaly events — those would put the echo + # somewhere the real aircraft is not. + self._dead_reckon(ac, dt) + return + # ── Fire scheduled mid-flight anomaly event ────────────────────────── if ac.anomaly_event and not ac.anomaly_fired and self._time >= ac.anomaly_trigger_at: self._fire_anomaly_event(ac) @@ -1147,10 +1199,164 @@ def get_aircraft_summary(self) -> list[dict]: "adsb_callsign": ac.adsb_callsign, "adsb_silent": ac.adsb_silent, "anomaly_event": ac.anomaly_event, + "source": ac.source, } for ac in self.aircraft ] + # ── Live ADS-B seeding ─────────────────────────────────────────────────── + + def synthetic_count(self) -> int: + """Aircraft spawned by this world's own planner (excludes live).""" + return sum(1 for ac in self.aircraft if ac.source != "live") + + def live_role_is_dark(self, hex_code: str) -> bool: + """Whether a feed aircraft is cast as dark at the current frac_live_dark. + + A stable hash of the hex against the fraction rather than a random + roll: the same aircraft gets the same answer on every ingest, so a + knob change flips exactly the aircraft the new fraction implies + (raising it only ever turns more aircraft dark, lowering it only + ever restores transponders) and nothing flickers between polls. + """ + if self.frac_live_dark <= 0.0: + return False + if self.frac_live_dark >= 1.0: + return True + u = int(hashlib.sha256(hex_code.encode()).hexdigest()[:8], 16) / 0x100000000 + return u < self.frac_live_dark + + def _cast_live_role(self, ac: SimulatedAircraft) -> None: + """Apply the dark/ADS-B cast to one live aircraft. + + Dark means the world mirrors the aircraft WITHOUT its transponder: + has_adsb off and adsb_hex None, so the node frames tag no echo with + it, the ADS-B push omits it, and the ground truth keys it by its + ``live-`` object id — exactly the shape of a synthetic dark + spawn, with a real trajectory underneath. + """ + dark = self.live_role_is_dark(ac.live_hex or "") + ac.has_adsb = not dark + ac.adsb_hex = None if dark else ac.live_hex + + def set_frac_live_dark(self, value: float) -> int: + """Set the live dark share and re-cast every live aircraft in the air. + + Returns how many live aircraft are dark afterwards. + """ + self.frac_live_dark = min(1.0, max(0.0, float(value))) + for ac in self.live_aircraft.values(): + self._cast_live_role(ac) + return sum(1 for ac in self.live_aircraft.values() if not ac.has_adsb) + + def ingest_live_aircraft(self, rows: list[dict], now_wall: float | None = None) -> dict: + """Merge one feed poll (live_adsb.parse_point_response rows) into the world. + + A row's position is advanced from its capture time to ``now_wall`` + along its reported velocity before it is applied, so a 10 s old fix + does not drag the echo 2 km behind the real aircraft. Known hexes + are updated in place (their ids, and so their ground-truth keys, are + stable); new ones join as ``live-`` aircraft. Rows never make + an aircraft leave — that is _expire_live_aircraft's job, on the + world clock, so a missed poll is a coast and not a blink. + """ + now_wall = time.time() if now_wall is None else now_wall + created = updated = 0 + for row in rows: + hex_code = str(row.get("hex") or "").strip().lower() + lat = row.get("lat") + lon = row.get("lon") + if not hex_code or not isinstance(lat, (int, float)) or not isinstance(lon, (int, float)): + continue + speed_km_s = max(0.0, float(row.get("gs") or 0.0)) * _KNOTS_TO_KM_S + heading = float(row.get("track") or 0.0) % 360.0 + alt_km = max(0.05, float(row.get("alt_baro") or 0.0) * _FT_TO_KM) + vel_up = float(row.get("baro_rate") or 0.0) * _FT_TO_KM / 60.0 + heading_rad = math.radians(heading) + vel_east = speed_km_s * math.sin(heading_rad) + vel_north = speed_km_s * math.cos(heading_rad) + captured_at = row.get("captured_at") + age = now_wall - captured_at if isinstance(captured_at, (int, float)) else 0.0 + age = min(max(age, 0.0), self.live_stale_s) + lat = float(lat) + (vel_north / R_EARTH) * (180 / math.pi) * age + lon = float(lon) + (vel_east / (R_EARTH * math.cos(math.radians(float(lat))))) * (180 / math.pi) * age + alt_km = max(0.05, alt_km + vel_up * age) + flight = str(row.get("flight") or "").strip() or None + + ac = self.live_aircraft.get(hex_code) + if ac is None: + ac = SimulatedAircraft( + object_id=f"live-{hex_code}", + lat=lat, + lon=lon, + alt_km=alt_km, + vel_east=vel_east, + vel_north=vel_north, + vel_up=vel_up, + heading_deg=heading, + speed_km_s=speed_km_s, + base_speed_km_s=speed_km_s, + object_type="aircraft", + adsb_callsign=flight, + created_at=self._time, + lifetime_s=float("inf"), + waypoints=[], + waypoint_idx=0, + sim_now_s=self._time, + source="live", + live_hex=hex_code, + ) + self._cast_live_role(ac) + # Same outage roll a synthetic transponder aircraft gets at + # spawn: frac_adsb_outage is a fraction of the ADS-B + # population, and live ADS-B aircraft are part of it. + ac.adsb_outage_start_s, ac.adsb_outage_end_s = self._roll_adsb_outage(ac.has_adsb, _ADSB_OUTAGE_START_S) + self.live_aircraft[hex_code] = ac + self.aircraft.append(ac) + created += 1 + else: + ac.lat = lat + ac.lon = lon + ac.alt_km = alt_km + ac.vel_east = vel_east + ac.vel_north = vel_north + ac.vel_up = vel_up + ac.heading_deg = heading + ac.speed_km_s = speed_km_s + ac.base_speed_km_s = speed_km_s + if flight: + ac.adsb_callsign = flight + self._cast_live_role(ac) + updated += 1 + ac.live_seen_s = self._time + ac.sim_now_s = self._time + return {"created": created, "updated": updated, "live": len(self.live_aircraft)} + + def _expire_live_aircraft(self) -> int: + """Drop live aircraft the feed has not refreshed within live_stale_s.""" + stale = [h for h, ac in self.live_aircraft.items() if self._time - ac.live_seen_s > self.live_stale_s] + if not stale: + return 0 + for h in stale: + del self.live_aircraft[h] + gone = {f"live-{h}" for h in stale} + self.aircraft = [ac for ac in self.aircraft if ac.object_id not in gone] + return len(stale) + + def clear_live_aircraft(self) -> int: + """Remove every live aircraft (the feed was switched off).""" + n = len(self.live_aircraft) + if n: + self.live_aircraft.clear() + self.aircraft = [ac for ac in self.aircraft if ac.source != "live"] + return n + + def _dead_reckon(self, ac: SimulatedAircraft, dt: float) -> None: + """Coast a live aircraft on its last reported velocity.""" + ac.lat += (ac.vel_north / R_EARTH) * (180 / math.pi) * dt + ac.lon += (ac.vel_east / (R_EARTH * math.cos(math.radians(ac.lat)))) * (180 / math.pi) * dt + ac.alt_km = max(0.05, ac.alt_km + ac.vel_up * dt) + # ── ML Training Data Batch Export ──────────────────────────────────────── def generate_training_batch(self, n_frames: int, dt: float = 0.5, mode: str = "adsb") -> list[dict]: diff --git a/tests/test_dual_fraction.py b/tests/test_dual_fraction.py index 08cca92..63949c8 100644 --- a/tests/test_dual_fraction.py +++ b/tests/test_dual_fraction.py @@ -146,9 +146,16 @@ class _StubWorld: min_aircraft = 1 max_aircraft = 1 + frac_live_dark = 0.0 + live_aircraft: dict = {} + def schedule_adsb_outages(self): return 0 + def set_frac_live_dark(self, value): + self.frac_live_dark = value + return 0 + class _StubOrchestrator: def __init__(self, max_range_km=0.0): diff --git a/tests/test_live_adsb.py b/tests/test_live_adsb.py new file mode 100644 index 0000000..657cafc --- /dev/null +++ b/tests/test_live_adsb.py @@ -0,0 +1,310 @@ +"""Live ADS-B seeding: real feed aircraft in the simulated world. + +Feed rows (adsb.retina.fm, adsb.lol shape) become world aircraft the synthetic +nodes echo. Two independent knobs: frac_live_dark casts a stable share of the +live aircraft as dark (mirrored without their transponder), and min/max_aircraft ++ frac_dark keep governing the synthetic spawns layered on top, untouched by +how many live aircraft the feed happens to report. +""" + +import io +import json +import math + +from retina_simulation.live_adsb import LiveAdsbClient, parse_point_response +from retina_simulation.orchestrator import build_adsb_push_payload, build_ground_truth_payload +from retina_simulation.world import NodeConfig, SimulationWorld + + +def _row(hex_code="ab1388", **overrides) -> dict: + base = { + "hex": hex_code, + "flight": "N81227", + "lat": 34.88, + "lon": -82.40, + "alt_baro": 11500.0, + "gs": 126.0, + "track": 80.0, + "baro_rate": 0.0, + "captured_at": 1000.0, + } + base.update(overrides) + return base + + +def _world(**knobs) -> SimulationWorld: + w = SimulationWorld(center_lat=34.85, center_lon=-82.39) + w.frac_anomalous = 0.0 + w.frac_drone = 0.0 + w.frac_dark = 0.0 + w.min_aircraft = 0 + w.max_aircraft = 0 + for k, v in knobs.items(): + setattr(w, k, v) + return w + + +# ── parse_point_response ───────────────────────────────────────────────────── + + +class TestParsePointResponse: + def test_normalises_tar1090_rows(self): + data = { + "ac": [ + { + "hex": "AB1388", + "flight": "N81227 ", + "lat": 34.88, + "lon": -82.4, + "alt_baro": 11500, + "gs": 126.0, + "track": 80.0, + "baro_rate": -192, + "seen_pos": 11.0, + }, + ] + } + rows = parse_point_response(data, fetched_at=2000.0) + assert len(rows) == 1 + r = rows[0] + assert r["hex"] == "ab1388" + assert r["flight"] == "N81227" + assert r["captured_at"] == 2000.0 - 11.0 + assert r["baro_rate"] == -192 + + def test_drops_ground_and_positionless_rows(self): + data = { + "ac": [ + {"hex": "aaaaaa", "lat": 34.0, "lon": -82.0, "alt_baro": "ground", "gs": 5}, + {"hex": "bbbbbb", "alt_baro": 3000}, + {"hex": "", "lat": 34.0, "lon": -82.0, "alt_baro": 3000}, + {"hex": "cccccc", "lat": 34.0, "lon": -82.0, "alt_baro": 3000}, + ] + } + assert [r["hex"] for r in parse_point_response(data, 0.0)] == ["cccccc"] + + +class TestLiveAdsbClient: + def test_builds_point_url_and_serves_last_good_on_failure(self, monkeypatch): + calls = [] + + class _Resp(io.BytesIO): + headers = {} + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def fake_urlopen(req, timeout=0): + calls.append(req.full_url) + if len(calls) == 1: + return _Resp(json.dumps({"ac": [{"hex": "ab1388", "lat": 1.0, "lon": 2.0, "alt_baro": 100}]}).encode()) + raise OSError("boom") + + monkeypatch.setattr("retina_simulation.live_adsb.urllib.request.urlopen", fake_urlopen) + c = LiveAdsbClient( + [{"name": "Greenville", "lat": 34.852, "lon": -82.394, "radius_nm": 60}], base_url="https://adsb.retina.fm/" + ) + first = c.fetch_all() + assert calls[0] == "https://adsb.retina.fm/v2/point/34.852/-82.394/60" + assert [r["hex"] for r in first] == ["ab1388"] + assert c.last_status["Greenville"] is True + second = c.fetch_all() # network fails → last good result stands in + assert [r["hex"] for r in second] == ["ab1388"] + assert c.last_status["Greenville"] is False + + +# ── ingest ─────────────────────────────────────────────────────────────────── + + +class TestIngest: + def test_creates_live_aircraft_with_real_kinematics(self): + w = _world() + stats = w.ingest_live_aircraft([_row()], now_wall=1000.0) + assert stats == {"created": 1, "updated": 0, "live": 1} + ac = w.live_aircraft["ab1388"] + assert ac.source == "live" + assert ac.object_id == "live-ab1388" + assert ac.live_hex == "ab1388" + assert ac.has_adsb is True and ac.adsb_hex == "ab1388" + assert ac.adsb_callsign == "N81227" + assert math.isclose(ac.speed_km_s, 126.0 * 1.852 / 3600.0) + assert math.isclose(ac.alt_km, 11500.0 * 0.0003048) + assert ac.heading_deg == 80.0 + assert ac in w.aircraft + + def test_extrapolates_a_stale_fix_to_now(self): + w = _world() + # Due north at 360 kt = 0.1852 km/s; a 10 s old fix should sit ~1.85 km north. + w.ingest_live_aircraft([_row(track=0.0, gs=360.0, captured_at=990.0)], now_wall=1000.0) + ac = w.live_aircraft["ab1388"] + dlat_km = (ac.lat - 34.88) * 111.19 + assert 1.7 < dlat_km < 2.0 + + def test_updates_in_place_and_keeps_id(self): + w = _world() + w.ingest_live_aircraft([_row()], now_wall=1000.0) + first = w.live_aircraft["ab1388"] + stats = w.ingest_live_aircraft([_row(lat=34.9, gs=200.0)], now_wall=1005.0) + assert stats == {"created": 0, "updated": 1, "live": 1} + assert w.live_aircraft["ab1388"] is first + assert first.lat > 34.88 + assert len([a for a in w.aircraft if a.source == "live"]) == 1 + + def test_expires_when_feed_goes_quiet_but_coasts_first(self): + w = _world(live_stale_s=30.0) + w.ingest_live_aircraft([_row(track=90.0, gs=360.0)], now_wall=1000.0) + ac = w.live_aircraft["ab1388"] + lon0 = ac.lon + for _ in range(10): + w.step(1.0, mode="adsb") + assert "ab1388" in w.live_aircraft + assert ac.lon > lon0 # dead-reckoned east while the feed was silent + for _ in range(25): + w.step(1.0, mode="adsb") + assert "ab1388" not in w.live_aircraft + assert all(a.source != "live" for a in w.aircraft) + + def test_clear_removes_every_live_aircraft(self): + w = _world(min_aircraft=3) + w.step(1.0, mode="adsb") + w.ingest_live_aircraft([_row("aaaaaa"), _row("bbbbbb")], now_wall=1000.0) + assert len(w.aircraft) == 5 + assert w.clear_live_aircraft() == 2 + assert len(w.aircraft) == 3 and w.synthetic_count() == 3 + + +# ── independence from the synthetic knobs ──────────────────────────────────── + + +class TestSyntheticIndependence: + def test_live_aircraft_do_not_count_toward_min_max_aircraft(self): + w = _world(min_aircraft=5, max_aircraft=5) + w.ingest_live_aircraft([_row(f"{i:06x}") for i in range(20)], now_wall=1000.0) + w.step(1.0, mode="adsb") + assert w.synthetic_count() == 5 # spawned up to min despite 20 live aircraft + assert len(w.aircraft) == 25 + + def test_live_aircraft_never_route_out_or_retire(self): + w = _world() + w.ingest_live_aircraft([_row()], now_wall=1000.0) + ac = w.live_aircraft["ab1388"] + for _ in range(50): + ac.live_seen_s = w._time # feed keeps reporting it + w.step(60.0, mode="adsb") # 50 minutes, far past any lifetime + assert ac in w.aircraft + assert ac.departing is False + + def test_separation_never_slows_a_live_aircraft(self): + w = _world() + w.ingest_live_aircraft( + [_row("aaaaaa", lat=34.88, lon=-82.40), _row("bbbbbb", lat=34.881, lon=-82.40)], now_wall=1000.0 + ) + speeds = {h: ac.speed_km_s for h, ac in w.live_aircraft.items()} + for _ in range(5): + for ac in w.live_aircraft.values(): + ac.live_seen_s = w._time + w.step(1.0, mode="adsb") + for h, ac in w.live_aircraft.items(): + assert math.isclose(ac.speed_km_s, speeds[h]) + + +# ── dark cast ──────────────────────────────────────────────────────────────── + + +class TestLiveDarkCast: + def test_zero_and_one_are_absolute(self): + w = _world(frac_live_dark=0.0) + assert not any(w.live_role_is_dark(f"{i:06x}") for i in range(200)) + w.frac_live_dark = 1.0 + assert all(w.live_role_is_dark(f"{i:06x}") for i in range(200)) + + def test_share_is_roughly_the_fraction_and_stable(self): + w = _world(frac_live_dark=0.3) + hexes = [f"{i * 7919:06x}" for i in range(1000)] + dark = [h for h in hexes if w.live_role_is_dark(h)] + assert 240 < len(dark) < 360 + assert dark == [h for h in hexes if w.live_role_is_dark(h)] # deterministic + + def test_raising_the_knob_only_adds_dark_aircraft(self): + w = _world(frac_live_dark=0.2) + hexes = [f"{i * 7919:06x}" for i in range(500)] + low = {h for h in hexes if w.live_role_is_dark(h)} + w.frac_live_dark = 0.5 + high = {h for h in hexes if w.live_role_is_dark(h)} + assert low <= high + + def test_dark_cast_strips_transponder_but_keeps_hex_for_recast(self): + w = _world(frac_live_dark=1.0) + w.ingest_live_aircraft([_row()], now_wall=1000.0) + ac = w.live_aircraft["ab1388"] + assert ac.has_adsb is False and ac.adsb_hex is None + assert ac.live_hex == "ab1388" + # Slider back to 0 → transponder restored on the aircraft already flying. + assert w.set_frac_live_dark(0.0) == 0 + assert ac.has_adsb is True and ac.adsb_hex == "ab1388" + + def test_set_frac_recasts_in_flight_aircraft(self): + w = _world(frac_live_dark=0.0) + w.ingest_live_aircraft([_row(f"{i * 7919:06x}") for i in range(300)], now_wall=1000.0) + assert all(ac.has_adsb for ac in w.live_aircraft.values()) + n_dark = w.set_frac_live_dark(0.5) + assert 120 < n_dark < 180 + assert n_dark == sum(1 for ac in w.live_aircraft.values() if not ac.has_adsb) + + def test_dark_live_aircraft_is_untagged_in_frames_and_pushes(self): + w = _world(frac_live_dark=1.0) + node = NodeConfig( + node_id="n1", + rx_lat=34.85, + rx_lon=-82.39, + tx_lat=34.80, + tx_lon=-82.30, + beam_width_deg=360.0, + max_range_km=200.0, + ) + w.add_node(node) + w.ingest_live_aircraft([_row()], now_wall=1000.0) + # Frames: many draws so the p_miss roll cannot hide a tagged echo. + tagged = 0 + for _ in range(50): + frame = w.generate_detections_for_node("n1", 0) + tagged += sum(1 for e in frame.get("adsb", []) if e) + assert tagged == 0 + summary = w.get_aircraft_summary() + assert summary[0]["source"] == "live" + assert build_adsb_push_payload(summary) == [] + gt = build_ground_truth_payload(summary) + assert gt[0]["hex"] == "live-ab1388" + assert gt[0]["has_adsb"] is False + assert gt[0]["source"] == "live" + + def test_adsb_live_aircraft_is_tagged_with_its_real_hex(self): + w = _world(frac_live_dark=0.0) + node = NodeConfig( + node_id="n1", + rx_lat=34.85, + rx_lon=-82.39, + tx_lat=34.80, + tx_lon=-82.30, + beam_width_deg=360.0, + max_range_km=200.0, + ) + w.add_node(node) + w.ingest_live_aircraft([_row()], now_wall=1000.0) + tags = [] + for _ in range(50): + tags += [e for e in w.generate_detections_for_node("n1", 0).get("adsb", []) if e] + assert tags and all(t["hex"] == "ab1388" and t["flight"] == "N81227" for t in tags) + summary = w.get_aircraft_summary() + push = build_adsb_push_payload(summary) + assert push[0]["hex"] == "ab1388" and push[0]["flight"] == "N81227" + assert build_ground_truth_payload(summary)[0]["hex"] == "ab1388" + + def test_synthetic_spawns_default_source_sim(self): + w = _world(min_aircraft=2) + w.step(1.0, mode="adsb") + assert {ac.source for ac in w.aircraft} == {"sim"} + assert all(e["source"] == "sim" for e in build_ground_truth_payload(w.get_aircraft_summary()))