From 33c32e7afc22497c998fc909c66ae6569c2bf261 Mon Sep 17 00:00:00 2001 From: Philip Van Baren Date: Tue, 14 Jul 2026 13:12:50 -0400 Subject: [PATCH 1/4] Dead-reckon aircraft between fetches; redraw at 4 Hz Aircraft positions only moved on each ~3 s ADS-B fetch, so motion was steppy. Now extrapolate each aircraft from its last fetched fix along its ground track (track_deg) at its ground speed (gs_knots) using elapsed time, and redraw at 4 Hz between fetches. - adsb_client: record lastUpdateMs() at each successful fetch (the time the stored positions are valid). - radar_display: extrapolatedLatLon() dead-reckons position using the same flat 111 km/deg projection as the drawing path (round-trips exactly); drawAircraft uses it for symbols, tags, and beyond-ring dots. - main loop: redraw every kRadarRedrawIntervalMs (250 ms) between fetches. The fetch interval goes 3 s -> 5 s, since smooth motion no longer depends on how often the data is refreshed. Extrapolation also accounts for the age of the fix itself. adsb.fi reports seen_pos: seconds since each aircraft's position was measured (often 10-30 s old). Starting from the fetch time drew a stale fix lagging its true position, so seen_pos is stored per aircraft (pos_age_ms, capped at 30 s) and added to the elapsed time. Positions snap to real data on each fetch. Motion pauses briefly during the blocking fetch itself. Applies to both targets. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/config.h | 6 +++++- include/services/adsb_client.h | 12 +++++++++++ src/main.cpp | 7 ++++++ src/services/adsb_client.cpp | 15 +++++++++++++ src/ui/radar_display.cpp | 39 ++++++++++++++++++++++++++++++---- 5 files changed, 74 insertions(+), 5 deletions(-) diff --git a/include/config.h b/include/config.h index 6555c0342..d084875e7 100644 --- a/include/config.h +++ b/include/config.h @@ -49,7 +49,11 @@ constexpr double kDefaultRadarLat = 52.3676; constexpr double kDefaultRadarLon = 4.9041; /** Poll adsb.fi (API public limit: 1 req/s). */ -constexpr unsigned long kAdsbFetchIntervalMs = 3000; +constexpr unsigned long kAdsbFetchIntervalMs = 5000; +/** Redraw cadence; aircraft are dead-reckoned along their track/speed so motion + * is smooth. ~4 Hz = 250 ms (panel scanout caps at ~24 Hz; the full-frame + * recompose+present is the practical limit ~10-15 Hz). */ +constexpr unsigned long kRadarRedrawIntervalMs = 250; /** Legacy scale unused — fetch uses radar::fetchRadiusKm() to screen edge. */ constexpr float kAdsbFetchRadiusScale = 1.0f; /** false = hide aircraft with alt_baro "ground"; true = show them too. */ diff --git a/include/services/adsb_client.h b/include/services/adsb_client.h index 2b5dd787c..a82f50ac1 100644 --- a/include/services/adsb_client.h +++ b/include/services/adsb_client.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace services::adsb { @@ -10,6 +11,10 @@ struct Aircraft { float nose_deg; float track_deg; float gs_knots; + /** Age of the position fix at fetch time (ms), from the feed's seen_pos. + * Added to the elapsed time when dead-reckoning so the drawn position + * reflects the estimated current location, not the already-stale fix. */ + uint32_t pos_age_ms; char callsign[9]; char type[5]; char alt[12]; @@ -20,6 +25,13 @@ constexpr size_t kMaxAircraft = 64; size_t aircraftCount(); const Aircraft* aircraftList(); +/** + * millis() timestamp of the last successful fetch (0 before the first). The + * stored lat/lon are the values as of this time; callers can dead-reckon + * newer positions from track_deg/gs_knots and the elapsed time. + */ +unsigned long lastUpdateMs(); + /** Hook invoked during long HTTP I/O (e.g. wifiLoop). Optional. */ using PollFn = void (*)(); void setPollFn(PollFn fn); diff --git a/src/main.cpp b/src/main.cpp index 797aecc09..17b3998e0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,6 +20,7 @@ bool g_radar_visible = false; unsigned long g_wifi_down_since = 0; unsigned long g_last_reconnect_ms = 0; unsigned long g_last_adsb_fetch_ms = 0; +unsigned long g_last_redraw_ms = 0; void showRadarIfConnected() { if (WiFi.status() != WL_CONNECTED) { @@ -110,8 +111,14 @@ void loop() { if (!g_radar_visible) { showRadarIfConnected(); } else if (millis() - g_last_adsb_fetch_ms >= config::kAdsbFetchIntervalMs) { + // Refresh aircraft data from adsb.fi (~every 3 s); also redraws. g_last_adsb_fetch_ms = millis(); fetchAndDrawAircraft(); + g_last_redraw_ms = millis(); + } else if (millis() - g_last_redraw_ms >= config::kRadarRedrawIntervalMs) { + // Between fetches, redraw at 4 Hz with dead-reckoned positions. + g_last_redraw_ms = millis(); + ui::radarDisplayRefreshAircraft(); } } diff --git a/src/services/adsb_client.cpp b/src/services/adsb_client.cpp index dcbbfcbea..77d3a2386 100644 --- a/src/services/adsb_client.cpp +++ b/src/services/adsb_client.cpp @@ -20,6 +20,7 @@ constexpr unsigned long kRequestTimeoutMs = 10000; Aircraft s_aircraft[kMaxAircraft]; size_t s_aircraft_count = 0; +unsigned long s_last_update_ms = 0; PollFn s_poll_fn = nullptr; void pollNetwork() { @@ -205,6 +206,8 @@ size_t aircraftCount() { return s_aircraft_count; } const Aircraft* aircraftList() { return s_aircraft; } +unsigned long lastUpdateMs() { return s_last_update_ms; } + bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { const float dist_nm = kmToNauticalMiles(fetch_radius_km); @@ -248,6 +251,9 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { return false; } + // Base time for dead-reckoning: the fetched positions are valid as of now. + s_last_update_ms = millis(); + JsonArray ac = doc["ac"].as(); if (ac.isNull()) { s_aircraft_count = 0; @@ -271,6 +277,15 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { s_aircraft[n].nose_deg = pickNoseHeading(plane); s_aircraft[n].track_deg = pickTrackHeading(plane); s_aircraft[n].gs_knots = pickGroundSpeed(plane); + + // seen_pos: seconds since this position was measured. Use it as the + // dead-reckoning age offset, capped so a very stale fix isn't flung far. + float seen_pos = 0.0f; + readJsonFloat(plane, "seen_pos", &seen_pos); + if (seen_pos < 0.0f) seen_pos = 0.0f; + if (seen_pos > 30.0f) seen_pos = 30.0f; + s_aircraft[n].pos_age_ms = static_cast(seen_pos * 1000.0f); + fillTagFields(&s_aircraft[n], plane); ++n; } diff --git a/src/ui/radar_display.cpp b/src/ui/radar_display.cpp index 3ececd6e8..add948f6d 100644 --- a/src/ui/radar_display.cpp +++ b/src/ui/radar_display.cpp @@ -1,5 +1,6 @@ #include "ui/radar_display.h" +#include #include #include @@ -213,6 +214,32 @@ void offsetKmFromCenter(float lat, float lon, float* dx_km, float* dy_km, *dist_km = sqrtf((*dx_km) * (*dx_km) + (*dy_km) * (*dy_km)); } +/** + * Dead-reckon an aircraft's position from its last-fetched fix along its ground + * track, so it moves smoothly between ADS-B updates. Uses the same flat + * 1° ≈ 111 km projection as offsetKmFromCenter(), so it round-trips exactly. + */ +void extrapolatedLatLon(const services::adsb::Aircraft& plane, float* lat, + float* lon) { + *lat = plane.lat; + *lon = plane.lon; + const unsigned long base_ms = services::adsb::lastUpdateMs(); + if (base_ms == 0 || plane.gs_knots <= 0.0f) { + return; + } + // Elapsed since the fix was measured = time since fetch + the fix's own age. + const unsigned long elapsed_ms = (millis() - base_ms) + plane.pos_age_ms; + const float elapsed_h = static_cast(elapsed_ms) / 3600000.0f; + const float dist_km = plane.gs_knots * 1.852f * elapsed_h; // knots -> km + if (dist_km <= 0.0f) { + return; + } + constexpr float kDegToRad = 0.01745329252f; + const float rad = plane.track_deg * kDegToRad; // track: 0 = N, 90 = E + *lat = plane.lat + (dist_km * cosf(rad)) / kKmPerDeg; + *lon = plane.lon + (dist_km * sinf(rad)) / kKmPerDeg; +} + float innerRingMaxKm() { const float outer_km = radar::rangeCurrent().outer_km; return outer_km * (static_cast(radar::kGridOuterRadius - @@ -497,15 +524,20 @@ void drawAircraft() { size_t dot_count = 0; for (size_t i = 0; i < n; ++i) { + // Dead-reckoned position for smooth motion between fetches. + float lat = 0.0f; + float lon = 0.0f; + extrapolatedLatLon(planes[i], &lat, &lon); + float dx_km = 0.0f; float dy_km = 0.0f; float dist_km = 0.0f; - offsetKmFromCenter(planes[i].lat, planes[i].lon, &dx_km, &dy_km, &dist_km); + offsetKmFromCenter(lat, lon, &dx_km, &dy_km, &dist_km); if (isInsideOuterRingKm(dist_km)) { int x = 0; int y = 0; - latLonToScreen(planes[i].lat, planes[i].lon, &x, &y); + latLonToScreen(lat, lon, &x, &y); items[draw_count].index = i; items[draw_count].x = x; items[draw_count].y = y; @@ -516,8 +548,7 @@ void drawAircraft() { int dot_x = 0; int dot_y = 0; - if (!beyondRingEdgeDotFromLatLon(planes[i].lat, planes[i].lon, &dot_x, - &dot_y)) { + if (!beyondRingEdgeDotFromLatLon(lat, lon, &dot_x, &dot_y)) { continue; } dots[dot_count].x = dot_x; From 42bf1392872ea6ba48fb0a5018115fdf26ae1f51 Mon Sep 17 00:00:00 2001 From: Philip Van Baren Date: Tue, 14 Jul 2026 13:04:35 -0400 Subject: [PATCH 2/4] Run the ADS-B HTTPS fetch on a background task The fetch blocked the main loop for ~1-2 s every 3 s, pausing the 2 Hz dead-reckoned redraw. Move it to a FreeRTOS task pinned to core 0 so the render loop (core 1) keeps refreshing smoothly throughout the request. - adsb_client: init() creates a mutex; fetchUpdate() parses into a local buffer and publish()es atomically under the lock; snapshotAircraft() copies the list + base timestamp under the lock for readers on another thread. - radar_display: drawAircraft() renders from a locked snapshot; extrapolation takes the base time as a parameter. - main: adsbFetchTask loops fetchUpdate every kAdsbFetchIntervalMs; the loop just redraws at 2 Hz. No more inline fetch / poll-fn. Applies to both targets (on the single-core C3 the scheduler interleaves the task with the loop). Co-Authored-By: Claude Opus 4.8 (1M context) --- include/services/adsb_client.h | 11 ++++ src/main.cpp | 35 ++++++----- src/services/adsb_client.cpp | 111 ++++++++++++++++++++++----------- src/ui/radar_display.cpp | 14 +++-- 4 files changed, 114 insertions(+), 57 deletions(-) diff --git a/include/services/adsb_client.h b/include/services/adsb_client.h index a82f50ac1..94ce29a18 100644 --- a/include/services/adsb_client.h +++ b/include/services/adsb_client.h @@ -22,6 +22,17 @@ struct Aircraft { constexpr size_t kMaxAircraft = 64; +/** Create the internal lock. Call once before any fetch/snapshot. */ +void init(); + +/** + * Copy the current aircraft into `out` (up to `max_out`) and report the fetch + * timestamp, all under a lock — safe to call from a different thread than the + * one running fetchUpdate(). Returns the number copied. + */ +size_t snapshotAircraft(Aircraft* out, size_t max_out, + unsigned long* out_last_update_ms); + size_t aircraftCount(); const Aircraft* aircraftList(); diff --git a/src/main.cpp b/src/main.cpp index 17b3998e0..6e98c651b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,7 +19,6 @@ namespace { bool g_radar_visible = false; unsigned long g_wifi_down_since = 0; unsigned long g_last_reconnect_ms = 0; -unsigned long g_last_adsb_fetch_ms = 0; unsigned long g_last_redraw_ms = 0; void showRadarIfConnected() { @@ -50,15 +49,19 @@ void handleBootButton() { } } -void fetchAndDrawAircraft() { - const float fetch_km = ui::radar::fetchRadiusKm(); - if (!services::adsb::fetchUpdate(services::location::lat(), - services::location::lon(), fetch_km)) { - handleBootButton(); - return; +// ADS-B fetch runs on its own task: the HTTPS request blocks for ~1-2 s, and +// keeping it off the main loop lets the radar keep redrawing (dead-reckoned) at +// 4 Hz throughout. The task publishes into the shared aircraft buffer under a +// lock; the render loop reads a snapshot. +void adsbFetchTask(void*) { + for (;;) { + if (WiFi.status() == WL_CONNECTED) { + services::adsb::fetchUpdate(services::location::lat(), + services::location::lon(), + ui::radar::fetchRadiusKm()); + } + vTaskDelay(pdMS_TO_TICKS(config::kAdsbFetchIntervalMs)); } - ui::radarDisplayRefreshAircraft(); - handleBootButton(); } } // namespace @@ -76,11 +79,15 @@ void setup() { } services::location::init(); ui::radar::rangeInit(); - services::adsb::setPollFn(wifiLoop); + services::adsb::init(); if (wifiSetupConnect()) { showRadarIfConnected(); } + + // Start the background ADS-B fetch (pinned to core 0, away from the render + // loop on core 1). It checks Wi-Fi state each cycle. + xTaskCreatePinnedToCore(adsbFetchTask, "adsb", 16384, nullptr, 1, nullptr, 0); } void loop() { @@ -110,13 +117,9 @@ void loop() { g_wifi_down_since = 0; if (!g_radar_visible) { showRadarIfConnected(); - } else if (millis() - g_last_adsb_fetch_ms >= config::kAdsbFetchIntervalMs) { - // Refresh aircraft data from adsb.fi (~every 3 s); also redraws. - g_last_adsb_fetch_ms = millis(); - fetchAndDrawAircraft(); - g_last_redraw_ms = millis(); } else if (millis() - g_last_redraw_ms >= config::kRadarRedrawIntervalMs) { - // Between fetches, redraw at 4 Hz with dead-reckoned positions. + // Redraw at 4 Hz with dead-reckoned positions; the ADS-B fetch runs on + // its own task (adsbFetchTask). g_last_redraw_ms = millis(); ui::radarDisplayRefreshAircraft(); } diff --git a/src/services/adsb_client.cpp b/src/services/adsb_client.cpp index 77d3a2386..6b6af6751 100644 --- a/src/services/adsb_client.cpp +++ b/src/services/adsb_client.cpp @@ -5,6 +5,9 @@ #include +#include +#include + #include #include "config.h" @@ -22,6 +25,22 @@ Aircraft s_aircraft[kMaxAircraft]; size_t s_aircraft_count = 0; unsigned long s_last_update_ms = 0; PollFn s_poll_fn = nullptr; +SemaphoreHandle_t s_mutex = nullptr; + +/** Publish parsed aircraft to the shared buffer atomically. */ +void publish(const Aircraft* src, size_t count) { + if (s_mutex != nullptr) { + xSemaphoreTake(s_mutex, portMAX_DELAY); + } + for (size_t i = 0; i < count; ++i) { + s_aircraft[i] = src[i]; + } + s_aircraft_count = count; + s_last_update_ms = millis(); // base time for dead-reckoning + if (s_mutex != nullptr) { + xSemaphoreGive(s_mutex); + } +} void pollNetwork() { if (s_poll_fn != nullptr) { @@ -200,6 +219,12 @@ void fillTagFields(Aircraft* ac, const JsonObject& plane) { } // namespace +void init() { + if (s_mutex == nullptr) { + s_mutex = xSemaphoreCreateMutex(); + } +} + void setPollFn(PollFn fn) { s_poll_fn = fn; } size_t aircraftCount() { return s_aircraft_count; } @@ -208,6 +233,25 @@ const Aircraft* aircraftList() { return s_aircraft; } unsigned long lastUpdateMs() { return s_last_update_ms; } +size_t snapshotAircraft(Aircraft* out, size_t max_out, + unsigned long* out_last_update_ms) { + if (s_mutex != nullptr) { + xSemaphoreTake(s_mutex, portMAX_DELAY); + } + const size_t count = + s_aircraft_count < max_out ? s_aircraft_count : max_out; + for (size_t i = 0; i < count; ++i) { + out[i] = s_aircraft[i]; + } + if (out_last_update_ms != nullptr) { + *out_last_update_ms = s_last_update_ms; + } + if (s_mutex != nullptr) { + xSemaphoreGive(s_mutex); + } + return count; +} + bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { const float dist_nm = kmToNauticalMiles(fetch_radius_km); @@ -251,46 +295,43 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { return false; } - // Base time for dead-reckoning: the fetched positions are valid as of now. - s_last_update_ms = millis(); - + // Parse into a local buffer, then publish atomically so a reader on another + // thread never sees a half-updated list. + Aircraft parsed[kMaxAircraft]; + size_t n = 0; JsonArray ac = doc["ac"].as(); - if (ac.isNull()) { - s_aircraft_count = 0; - return true; - } + if (!ac.isNull()) { + for (JsonObject plane : ac) { + if (n >= kMaxAircraft) { + break; + } + if (!plane["lat"].is() || !plane["lon"].is()) { + continue; + } + if (isOnGround(plane) && !config::kAdsbShowGroundAircraft) { + continue; + } - size_t n = 0; - for (JsonObject plane : ac) { - if (n >= kMaxAircraft) { - break; - } - if (!plane["lat"].is() || !plane["lon"].is()) { - continue; - } - if (isOnGround(plane) && !config::kAdsbShowGroundAircraft) { - continue; + parsed[n].lat = plane["lat"].as(); + parsed[n].lon = plane["lon"].as(); + parsed[n].nose_deg = pickNoseHeading(plane); + parsed[n].track_deg = pickTrackHeading(plane); + parsed[n].gs_knots = pickGroundSpeed(plane); + + // seen_pos: seconds since this position was measured. Use it as the + // dead-reckoning age offset, capped so a very stale fix isn't flung far. + float seen_pos = 0.0f; + readJsonFloat(plane, "seen_pos", &seen_pos); + if (seen_pos < 0.0f) seen_pos = 0.0f; + if (seen_pos > 30.0f) seen_pos = 30.0f; + parsed[n].pos_age_ms = static_cast(seen_pos * 1000.0f); + + fillTagFields(&parsed[n], plane); + ++n; } - - s_aircraft[n].lat = plane["lat"].as(); - s_aircraft[n].lon = plane["lon"].as(); - s_aircraft[n].nose_deg = pickNoseHeading(plane); - s_aircraft[n].track_deg = pickTrackHeading(plane); - s_aircraft[n].gs_knots = pickGroundSpeed(plane); - - // seen_pos: seconds since this position was measured. Use it as the - // dead-reckoning age offset, capped so a very stale fix isn't flung far. - float seen_pos = 0.0f; - readJsonFloat(plane, "seen_pos", &seen_pos); - if (seen_pos < 0.0f) seen_pos = 0.0f; - if (seen_pos > 30.0f) seen_pos = 30.0f; - s_aircraft[n].pos_age_ms = static_cast(seen_pos * 1000.0f); - - fillTagFields(&s_aircraft[n], plane); - ++n; } - s_aircraft_count = n; + publish(parsed, n); Serial.printf("adsb: %u aircraft\n", static_cast(n)); return true; } diff --git a/src/ui/radar_display.cpp b/src/ui/radar_display.cpp index add948f6d..a374a29d5 100644 --- a/src/ui/radar_display.cpp +++ b/src/ui/radar_display.cpp @@ -219,11 +219,10 @@ void offsetKmFromCenter(float lat, float lon, float* dx_km, float* dy_km, * track, so it moves smoothly between ADS-B updates. Uses the same flat * 1° ≈ 111 km projection as offsetKmFromCenter(), so it round-trips exactly. */ -void extrapolatedLatLon(const services::adsb::Aircraft& plane, float* lat, - float* lon) { +void extrapolatedLatLon(const services::adsb::Aircraft& plane, + unsigned long base_ms, float* lat, float* lon) { *lat = plane.lat; *lon = plane.lon; - const unsigned long base_ms = services::adsb::lastUpdateMs(); if (base_ms == 0 || plane.gs_knots <= 0.0f) { return; } @@ -515,8 +514,11 @@ void sortBeyondDotsFarFirst(BeyondDotDrawItem* items, size_t count) { void drawAircraft() { initLabelMetrics(); - const size_t n = services::adsb::aircraftCount(); - const services::adsb::Aircraft* planes = services::adsb::aircraftList(); + // Snapshot under the adsb lock (the fetch may run on another thread). + static services::adsb::Aircraft planes[services::adsb::kMaxAircraft]; + unsigned long base_ms = 0; + const size_t n = services::adsb::snapshotAircraft( + planes, services::adsb::kMaxAircraft, &base_ms); AircraftDrawItem items[services::adsb::kMaxAircraft]; BeyondDotDrawItem dots[services::adsb::kMaxAircraft]; @@ -527,7 +529,7 @@ void drawAircraft() { // Dead-reckoned position for smooth motion between fetches. float lat = 0.0f; float lon = 0.0f; - extrapolatedLatLon(planes[i], &lat, &lon); + extrapolatedLatLon(planes[i], base_ms, &lat, &lon); float dx_km = 0.0f; float dy_km = 0.0f; From 4efc3a9a8d382b1ce67ebbe1d6d1aa234a72b400 Mon Sep 17 00:00:00 2001 From: Philip Van Baren Date: Wed, 9 Sep 2026 22:52:02 -0400 Subject: [PATCH 3/4] Stream the ADS-B response into a filtered JSON parse The fetch buffered the whole response body into a String and then handed that to ArduinoJson, so peak heap held the payload and the document at the same time -- and the document stored all ~40 fields of every aircraft record when the radar reads only 15 of them. Replace readResponseBodyWithPoll with PollingBodyReader, which ArduinoJson pulls from directly. It drains the socket in 512-byte blocks and serves single bytes, since the JSON deserializer never calls readBytes, and it runs the network poll callback on every refill -- the reason HTTPClients --- src/services/adsb_client.cpp | 155 +++++++++++++++++++++++++++-------- 1 file changed, 120 insertions(+), 35 deletions(-) diff --git a/src/services/adsb_client.cpp b/src/services/adsb_client.cpp index 6b6af6751..0bd52ef3b 100644 --- a/src/services/adsb_client.cpp +++ b/src/services/adsb_client.cpp @@ -66,43 +66,114 @@ int performGetWithPoll(HTTPClient& http) { return HTTPC_ERROR_READ_TIMEOUT; } -bool readResponseBodyWithPoll(HTTPClient& http, String& payload) { - WiFiClient* stream = http.getStreamPtr(); - if (stream == nullptr) { - return false; - } - - const int content_length = http.getSize(); - if (content_length > 0) { - payload.reserve(static_cast(content_length + 1)); +/** + * Feeds the response body to the JSON parser one byte at a time without + * buffering the whole thing. + * + * ArduinoJson pulls single bytes, so the socket is drained in blocks and + * handed out from buffer_. Each refill runs the network poll callback, which + * is why HTTPClient's own body readers (getString/writeToStream) can't be + * used here -- they block without giving the fetch task a chance to poll. + */ +class PollingBodyReader { + public: + PollingBodyReader(HTTPClient& http, WiFiClient& stream, int content_length, + unsigned long deadline) + : http_(&http), + stream_(&stream), + remaining_(content_length), + deadline_(deadline) {} + + /** Next body byte, or -1 at end of body / timeout. Never returns 0. */ + int read() { + if (pos_ >= len_ && !refill()) { + return -1; + } + ++total_; + return static_cast(buffer_[pos_++]); } - uint8_t buffer[512]; - const unsigned long deadline = millis() + kRequestTimeoutMs; - while (millis() < deadline) { - pollNetwork(); - const int available = stream->available(); - if (available > 0) { - const int to_read = - available > static_cast(sizeof(buffer)) ? static_cast(sizeof(buffer)) - : available; - const int read_bytes = stream->readBytes(buffer, to_read); - if (read_bytes > 0) { - payload.concat(reinterpret_cast(buffer), - static_cast(read_bytes)); + size_t readBytes(char* out, size_t length) { + size_t n = 0; + while (n < length) { + const int c = read(); + if (c < 0) { + break; } + out[n++] = static_cast(c); } - if (content_length > 0 && - static_cast(payload.length()) >= content_length) { - break; + return n; + } + + size_t bytesRead() const { return total_; } + + private: + bool refill() { + pos_ = 0; + len_ = 0; + if (remaining_ == 0) { + return false; // Content-Length fully consumed } - if (!http.connected() && stream->available() <= 0) { - break; + while (millis() < deadline_) { + pollNetwork(); + const int available = stream_->available(); + if (available > 0) { + int to_read = available > static_cast(sizeof(buffer_)) + ? static_cast(sizeof(buffer_)) + : available; + if (remaining_ > 0 && remaining_ < to_read) { + to_read = remaining_; // never read past the end of the body + } + const int read_bytes = stream_->readBytes(buffer_, to_read); + if (read_bytes > 0) { + len_ = static_cast(read_bytes); + if (remaining_ > 0) { + remaining_ -= read_bytes; + } + return true; + } + } + if (!http_->connected() && stream_->available() <= 0) { + break; // server closed and the socket is drained: end of body + } + delay(1); } - delay(1); + return false; } - return payload.length() > 0; + HTTPClient* http_; + WiFiClient* stream_; + int remaining_; // bytes left per Content-Length, or < 0 when unknown + unsigned long deadline_; + char buffer_[512]; + size_t pos_ = 0; + size_t len_ = 0; + size_t total_ = 0; +}; + +/** + * Builds the deserialization filter. The keys below are the only ones the + * radar reads, out of the ~40 each adsb.fi v3 record carries; the parser + * skips the rest (rssi, mlat, tisb, nic, messages, ...) without storing them. + */ +void buildAircraftFilter(JsonDocument& filter) { + // A filter array applies its first element to every element of the input. + JsonObject plane = filter["ac"].add(); + plane["lat"] = true; + plane["lon"] = true; + plane["track"] = true; + plane["true_heading"] = true; + plane["mag_heading"] = true; + plane["dir"] = true; + plane["gs"] = true; + plane["tas"] = true; + plane["ias"] = true; + plane["alt_baro"] = true; + plane["alt_geom"] = true; + plane["seen_pos"] = true; + plane["flight"] = true; + plane["hex"] = true; + plane["t"] = true; } float kmToNauticalMiles(float km) { return km / kKmPerNm; } @@ -280,18 +351,32 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { return false; } - String payload; - if (!readResponseBodyWithPoll(http, payload)) { - Serial.println("adsb: empty response"); + WiFiClient* stream = http.getStreamPtr(); + if (stream == nullptr) { + Serial.println("adsb: no response stream"); http.end(); return false; } - http.end(); + // Parse straight off the socket, with a filter that keeps only the fields + // the radar reads. The body is never held in RAM as a whole and the skipped + // fields never get a document slot, so peak heap stays flat no matter how + // many aircraft the API returns. + JsonDocument filter; + buildAircraftFilter(filter); + + PollingBodyReader body(http, *stream, http.getSize(), + millis() + kRequestTimeoutMs); JsonDocument doc; - const DeserializationError err = deserializeJson(doc, payload); + const DeserializationError err = + deserializeJson(doc, body, DeserializationOption::Filter(filter)); + http.end(); if (err) { - Serial.printf("adsb: JSON parse error: %s\n", err.c_str()); + if (body.bytesRead() == 0) { + Serial.println("adsb: empty response"); + } else { + Serial.printf("adsb: JSON parse error: %s\n", err.c_str()); + } return false; } From 5cec5d1c8508708b7fb3a1fe3bd8749048740dbf Mon Sep 17 00:00:00 2001 From: Philip Van Baren Date: Wed, 9 Sep 2026 23:50:14 -0400 Subject: [PATCH 4/4] Fetch adsb.fi over HTTP/1.1, decoding the chunked body e2e94c6 forced HTTP/1.0 because the response body was arriving chunked and the JSON parser choked on the chunk-size lines. The cause is that getStreamPtr() hands back the raw socket -- HTTPClient de-chunks only inside its own body readers, which this code can't use because they block without running the network poll callback. Decode the framing here instead. BodyFramer unwraps an identity or chunked body from a byte source, stripping chunk sizes, extensions, the inter-chunk CRLFs and any trailer section. It pulls a byte at a time, so a chunk boundary landing anywhere in the socket's own buffering -- mid-hex-digit, or between the CR and the LF -- needs no special handling. PollingBodyReader loses its length bookkeeping to the framer and becomes PollingSocketSource, a plain pump that keeps the poll-callback refill loop. Transfer-Encoding has to be requested via collectHeaders() before the GET; HTTPClient's own _transferEncoding is private. Dropping useHTTP10 also re-enables keep-alive on the request, but nothing is reused yet -- the client and HTTPClient are still torn down every fetch. The post-parse drain() is groundwork for that: a parser stops at the closing brace, and a connection reused without consuming the terminating chunk would read it as the head of the next response. The header is free of Arduino includes so the framing can be tested on a host. Compiles clean on both arduino-esp32 2.x and 3.x. Co-Authored-By: Claude Opus 5 (1M context) --- include/services/http_body_framing.h | 303 +++++++++++++++++++++++++++ src/services/adsb_client.cpp | 94 ++++----- 2 files changed, 348 insertions(+), 49 deletions(-) create mode 100644 include/services/http_body_framing.h diff --git a/include/services/http_body_framing.h b/include/services/http_body_framing.h new file mode 100644 index 000000000..a7a63c4de --- /dev/null +++ b/include/services/http_body_framing.h @@ -0,0 +1,303 @@ +#pragma once + +#include +#include + +namespace services::http { + +/** How the response body is delimited (RFC 9112 section 6). */ +enum class BodyFraming { + /** Content-Length, or -- when the length is unknown -- read until close. */ + kIdentity, + /** Transfer-Encoding: chunked. */ + kChunked, +}; + +/** + * Unwraps an HTTP/1.1 message body from a raw byte source. + * + * `Source` supplies the body's wire image one byte at a time through + * + * int read(); // next byte, or < 0 when no more is coming + * + * and this class hands back the decoded body, stripping chunk sizes, chunk + * extensions, the inter-chunk CRLFs and any trailer section. Reading a byte at + * a time is what makes the decoder correct: the framing state lives in the + * object, so a chunk boundary landing anywhere in the source's own buffering + * -- mid-hex-digit, or between the CR and the LF -- needs no special handling. + * Callers that want throughput buffer inside the source, not out here. + * + * read()/readBytes() match the shape ArduinoJson expects of a custom input, + * so an instance can be handed straight to deserializeJson(). + */ +template +class BodyFramer { + public: + /** + * `content_length` is the Content-Length value, or < 0 when the header was + * absent. It is ignored for a chunked body, which carries its own lengths. + */ + BodyFramer(Source& source, BodyFraming framing, int content_length) + : source_(&source), + framing_(framing), + remaining_(framing == BodyFraming::kIdentity ? content_length : -1), + state_(framing == BodyFraming::kChunked ? State::kSize + : State::kBody) {} + + /** Next decoded body byte, or -1 once the body ends or the source dries + * up. */ + int read() { + return framing_ == BodyFraming::kChunked ? readChunked() : readIdentity(); + } + + size_t readBytes(char* out, size_t length) { + size_t n = 0; + while (n < length) { + const int c = read(); + if (c < 0) { + break; + } + out[n++] = static_cast(c); + } + return n; + } + + /** + * Consume whatever is left of the body, so the source is positioned at the + * start of the next message. A parser stops at the closing brace and leaves + * the terminating chunk (and any trailer) unread; a connection reused + * without this would read that leftover as the next response. + * + * Does nothing for a close-delimited body, whose end only arrives when the + * peer hangs up -- there is nothing to reuse in that case anyway. + */ + void drain() { + if (framing_ == BodyFraming::kIdentity && remaining_ < 0) { + return; + } + while (read() >= 0) { + } + } + + /** Decoded body bytes handed out so far. */ + size_t bytesRead() const { return total_; } + + /** True once the body has been read through to its end. */ + bool complete() const { return state_ == State::kDone; } + + /** True when the chunk framing was malformed, rather than merely cut + * short. */ + bool framingError() const { return framing_error_; } + + /** True when the source ran dry before the body ended. */ + bool truncated() const { return truncated_; } + + private: + enum class State { + kBody, // identity: body bytes + kSize, // chunked: hex digits of the chunk-size line + kSizeEol, // chunked: chunk extensions / CR, up to the line's LF + kData, // chunked: chunk payload + kDataCR, // chunked: CR of the CRLF that follows the payload + kDataLF, // chunked: LF of that CRLF + kTrailer, // chunked: trailer section, up to its blank line + kDone, + kError, + }; + + /** A chunk size is a 32-bit count; more digits than that is malformed. */ + static constexpr int kMaxSizeDigits = 8; + + static bool isHexDigit(int c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); + } + + static uint32_t hexValue(int c) { + if (c >= '0' && c <= '9') { + return static_cast(c - '0'); + } + if (c >= 'a' && c <= 'f') { + return static_cast(c - 'a' + 10); + } + return static_cast(c - 'A' + 10); + } + + int malformed() { + framing_error_ = true; + state_ = State::kError; + return -1; + } + + int cutShort() { + truncated_ = true; + state_ = State::kError; + return -1; + } + + int readIdentity() { + if (state_ != State::kBody) { + return -1; + } + if (remaining_ == 0) { + state_ = State::kDone; + return -1; + } + const int c = source_->read(); + if (c < 0) { + // Without a Content-Length the close *is* the terminator; with one, a + // short read means the body was cut off. + if (remaining_ < 0) { + state_ = State::kDone; + return -1; + } + return cutShort(); + } + if (remaining_ > 0) { + --remaining_; + } + ++total_; + return c; + } + + /** Chunk-size line is parsed; decide what follows it. */ + void finishSizeLine() { + if (chunk_remaining_ == 0) { + trailer_line_len_ = 0; + state_ = State::kTrailer; + } else { + state_ = State::kData; + } + } + + int readChunked() { + for (;;) { + switch (state_) { + case State::kSize: { + const int c = source_->read(); + if (c < 0) { + return cutShort(); + } + if (isHexDigit(c)) { + if (size_digits_ >= kMaxSizeDigits) { + return malformed(); + } + chunk_remaining_ = chunk_remaining_ * 16 + hexValue(c); + ++size_digits_; + break; + } + if (size_digits_ == 0) { + return malformed(); // a chunk-size line must start with hex + } + if (c == ';' || c == '\r') { + state_ = State::kSizeEol; // chunk extensions, then the CRLF + break; + } + if (c == '\n') { + finishSizeLine(); + break; + } + return malformed(); + } + + case State::kSizeEol: { + const int c = source_->read(); + if (c < 0) { + return cutShort(); + } + if (c == '\n') { + finishSizeLine(); + } + break; // anything else is extension text, discarded + } + + case State::kData: { + const int c = source_->read(); + if (c < 0) { + return cutShort(); + } + if (--chunk_remaining_ == 0) { + state_ = State::kDataCR; + } + ++total_; + return c; + } + + case State::kDataCR: { + const int c = source_->read(); + if (c < 0) { + return cutShort(); + } + if (c == '\r') { + state_ = State::kDataLF; + break; + } + if (c == '\n') { // tolerate a bare LF between chunks + startSizeLine(); + break; + } + return malformed(); + } + + case State::kDataLF: { + const int c = source_->read(); + if (c < 0) { + return cutShort(); + } + if (c != '\n') { + return malformed(); + } + startSizeLine(); + break; + } + + case State::kTrailer: { + const int c = source_->read(); + if (c < 0) { + // The zero-length chunk already ended the body; a peer that hangs + // up before the closing CRLF has still delivered all of it. + state_ = State::kDone; + return -1; + } + if (c == '\r') { + break; + } + if (c == '\n') { + if (trailer_line_len_ == 0) { + state_ = State::kDone; + return -1; + } + trailer_line_len_ = 0; // end of one trailer field + break; + } + ++trailer_line_len_; + break; + } + + case State::kBody: + case State::kDone: + case State::kError: + return -1; + } + } + } + + void startSizeLine() { + chunk_remaining_ = 0; + size_digits_ = 0; + state_ = State::kSize; + } + + Source* source_; + BodyFraming framing_; + int remaining_; // identity: bytes left per Content-Length, < 0 if unknown + State state_; + uint32_t chunk_remaining_ = 0; + int size_digits_ = 0; + size_t trailer_line_len_ = 0; + size_t total_ = 0; + bool framing_error_ = false; + bool truncated_ = false; +}; + +} // namespace services::http diff --git a/src/services/adsb_client.cpp b/src/services/adsb_client.cpp index 0bd52ef3b..c46e66e92 100644 --- a/src/services/adsb_client.cpp +++ b/src/services/adsb_client.cpp @@ -11,6 +11,7 @@ #include #include "config.h" +#include "services/http_body_framing.h" namespace services::adsb { @@ -67,74 +68,51 @@ int performGetWithPoll(HTTPClient& http) { } /** - * Feeds the response body to the JSON parser one byte at a time without - * buffering the whole thing. + * Pumps the socket one byte at a time, in blocks, without buffering the whole + * response. * - * ArduinoJson pulls single bytes, so the socket is drained in blocks and - * handed out from buffer_. Each refill runs the network poll callback, which - * is why HTTPClient's own body readers (getString/writeToStream) can't be - * used here -- they block without giving the fetch task a chance to poll. + * Each refill runs the network poll callback, which is why HTTPClient's own + * body readers (getString/writeToStream) can't be used here -- they block + * without giving the fetch task a chance to poll. That also means they can't + * de-chunk for us, so the wire image comes out raw and BodyFramer unwraps it. + * + * Where the body ends is BodyFramer's business, not this class's: it stops + * pulling at the right byte. Reading a little past that point into buffer_ is + * harmless while the connection is torn down after every fetch. */ -class PollingBodyReader { +class PollingSocketSource { public: - PollingBodyReader(HTTPClient& http, WiFiClient& stream, int content_length, - unsigned long deadline) - : http_(&http), - stream_(&stream), - remaining_(content_length), - deadline_(deadline) {} - - /** Next body byte, or -1 at end of body / timeout. Never returns 0. */ + PollingSocketSource(HTTPClient& http, WiFiClient& stream, + unsigned long deadline) + : http_(&http), stream_(&stream), deadline_(deadline) {} + + /** Next raw byte, or -1 once the socket closes or the deadline passes. */ int read() { if (pos_ >= len_ && !refill()) { return -1; } - ++total_; return static_cast(buffer_[pos_++]); } - size_t readBytes(char* out, size_t length) { - size_t n = 0; - while (n < length) { - const int c = read(); - if (c < 0) { - break; - } - out[n++] = static_cast(c); - } - return n; - } - - size_t bytesRead() const { return total_; } - private: bool refill() { pos_ = 0; len_ = 0; - if (remaining_ == 0) { - return false; // Content-Length fully consumed - } while (millis() < deadline_) { pollNetwork(); const int available = stream_->available(); if (available > 0) { - int to_read = available > static_cast(sizeof(buffer_)) - ? static_cast(sizeof(buffer_)) - : available; - if (remaining_ > 0 && remaining_ < to_read) { - to_read = remaining_; // never read past the end of the body - } + const int to_read = available > static_cast(sizeof(buffer_)) + ? static_cast(sizeof(buffer_)) + : available; const int read_bytes = stream_->readBytes(buffer_, to_read); if (read_bytes > 0) { len_ = static_cast(read_bytes); - if (remaining_ > 0) { - remaining_ -= read_bytes; - } return true; } } if (!http_->connected() && stream_->available() <= 0) { - break; // server closed and the socket is drained: end of body + break; // server closed and the socket is drained } delay(1); } @@ -143,14 +121,14 @@ class PollingBodyReader { HTTPClient* http_; WiFiClient* stream_; - int remaining_; // bytes left per Content-Length, or < 0 when unknown unsigned long deadline_; char buffer_[512]; size_t pos_ = 0; size_t len_ = 0; - size_t total_ = 0; }; +using BodyReader = services::http::BodyFramer; + /** * Builds the deserialization filter. The keys below are the only ones the * radar reads, out of the ~40 each adsb.fi v3 record carries; the parser @@ -342,7 +320,11 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { return false; } - http.useHTTP10(true); + // HTTPClient only records Transfer-Encoding in the collected headers when + // it is asked for up front; _transferEncoding itself is private. + static const char* kWantedHeaders[] = {"Transfer-Encoding"}; + http.collectHeaders(kWantedHeaders, 1); + http.setTimeout(kRequestTimeoutMs); const int code = performGetWithPoll(http); if (code != HTTP_CODE_OK) { @@ -365,14 +347,28 @@ bool fetchUpdate(double center_lat, double center_lon, float fetch_radius_km) { JsonDocument filter; buildAircraftFilter(filter); - PollingBodyReader body(http, *stream, http.getSize(), - millis() + kRequestTimeoutMs); + // On HTTP/1.1 the CDN answers with Transfer-Encoding: chunked, and + // getStreamPtr() hands back the raw socket -- chunk sizes and all. BodyFramer + // strips that framing back off. + const services::http::BodyFraming framing = + http.header("Transfer-Encoding").equalsIgnoreCase("chunked") + ? services::http::BodyFraming::kChunked + : services::http::BodyFraming::kIdentity; + + PollingSocketSource source(http, *stream, millis() + kRequestTimeoutMs); + BodyReader body(source, framing, http.getSize()); JsonDocument doc; const DeserializationError err = deserializeJson(doc, body, DeserializationOption::Filter(filter)); + // Read off the terminating chunk the parser stopped short of, so the socket + // sits at the end of the message. Not needed while every fetch builds its own + // connection, but a prerequisite for ever reusing one. + body.drain(); http.end(); if (err) { - if (body.bytesRead() == 0) { + if (body.framingError()) { + Serial.println("adsb: malformed chunked body"); + } else if (body.bytesRead() == 0) { Serial.println("adsb: empty response"); } else { Serial.printf("adsb: JSON parse error: %s\n", err.c_str());