Handle chunked encoding

platformio.ini

@@ -35,6 +35,7 @@ lib_deps =
   ricmoo/QRCode @ 0.0.1 ; for QR code
   marvinroger/AsyncMqttClient @ 0.9.0 ; FOR MQTT
   bblanchon/ArduinoJson @ 7.4.3 ; for JSON
+  bblanchon/StreamUtils @ 1.9.2
   https://github.com/tzapu/WiFiManager.git#v2.0.17 ; WiFi config portal
 
 ; set the default target to compile, upload, and monitor

src/network.cpp

@@ -1,5 +1,6 @@
 #include <WiFi.h>
 #include <HTTPClient.h>
+#include <StreamUtils.h>
 #include "homeplate.h"
 
 #define WIFI_TASK_PRIORITY 2
@@ -205,7 +206,6 @@ uint8_t* httpGet(const char* url, std::map<String, String> *headers, int32_t* de
     WiFi.setSleep(false);
 
     HTTPClient http;
-    http.getStream().setNoDelay(true);
     delaySleep(timeout_sec);
 
     // Connect with HTTP
@@ -217,13 +217,23 @@ uint8_t* httpGet(const char* url, std::map<String, String> *headers, int32_t* de
         }
     }
 
+    http.addHeader("Connection", "close"); // terminate connection when request is finished
+    const char *keys[] = {"Transfer-Encoding"};
+    http.collectHeaders(keys, 1);
+
     int httpCode = http.GET();
 
+    if (httpCode != HTTP_CODE_OK) {
+        Serial.printf("[NET] Non-200 response: %d from URL %s\n", httpCode, url);
+        Serial.printf("[NET] HTTP response buffer: \n\n%s\n\n", http.getString());
+        return nullptr;
+    }
+
     int32_t size = http.getSize();
     if (size == -1)
         size = *defaultLen;
-    else
-        *defaultLen = size;
+
+    *defaultLen = size;
 
     // Validate size to prevent buffer overflow attacks
     const int32_t MAX_HTTP_BUFFER_SIZE = 1024 * 1024; // 1MB limit
@@ -248,13 +258,19 @@ uint8_t* httpGet(const char* url, std::map<String, String> *headers, int32_t* de
 
     uint8_t buff[512] = {0};
 
-    WiFiClient* stream = http.getStreamPtr();
+
+    Stream &rawStream = http.getStream();
+    ChunkDecodingStream decodedStream(http.getStream());
+
+    // Choose the stream based on the Transfer-Encoding header
+    Stream &response = http.header("Transfer-Encoding") == "chunked" ? decodedStream : rawStream;
+
     while (http.connected() && (len > 0 || len == -1)) {
-        size_t size = stream->available();
+        size_t remainingSize = response.available();
 
-        if (size) {
-            int c = stream->readBytes(
-                buff, ((size > sizeof(buff)) ? sizeof(buff) : size));
+        if (remainingSize) {
+            int c = response.readBytes(
+                buff, ((remainingSize > sizeof(buff)) ? sizeof(buff) : remainingSize));
             memcpy(buffPtr, buff, c);
 
             if (len > 0) len -= c;
@@ -264,15 +280,6 @@ uint8_t* httpGet(const char* url, std::map<String, String> *headers, int32_t* de
         }
     }
 
-    if (httpCode != HTTP_CODE_OK) {
-        Serial.printf("[NET] Non-200 response: %d from URL %s\n", httpCode, url);
-        if (size) {
-            Serial.printf("[NET] HTTP response buffer: \n\n%s\n\n", buffer);
-        }
-        free(buffer);
-        buffer = 0;
-    }
-
     http.end();
     WiFi.setSleep(sleep);