From 4986cf4bc37a89c3f3ef62df3f9d8974de8e58e5 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 28 Apr 2026 09:54:47 -0300 Subject: [PATCH 01/16] Avoid holding a large JSON document while initiating SSL connections We ran into issues with allocation failures when attempting to set up SSL connections. After a lot of searching and trial-and-error, we realized that as the number of log files on the SD card grew, the size of the JSON document returned by GenerateFilelist() grew to consume a significant chunk of heap memory. This document was resident while the upload loop ran, which is the exact time we needed the memory most. Once this was realized, we almost immediately notices that we didn't need this JSON at all, as the only thing we were pulling from it was the file numbers, something we can easily get from a CountLogFiles() call. --- LoggerFirmware/src/AutoUpload.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/LoggerFirmware/src/AutoUpload.cpp b/LoggerFirmware/src/AutoUpload.cpp index 77dd02a5..e9b27e84 100644 --- a/LoggerFirmware/src/AutoUpload.cpp +++ b/LoggerFirmware/src/AutoUpload.cpp @@ -81,16 +81,17 @@ void UploadManager::UploadCycle(void) m_lastUploadCycle); return; } - DynamicJsonDocument files(logger::status::GenerateFilelist(m_logManager)); - int filecount = files["files"]["count"].as(); - for (int n = 0; n < filecount; ++n) { - uint32_t file_id = files["files"]["detail"][n]["id"].as(); - if (TransferFile(m_logManager->FileSystem(), file_id)) { + + auto * fileNumbers = new uint32_t[logger::MaxLogFiles]; + auto fileCount = m_logManager->CountLogFiles(fileNumbers); + for (int i = 0; i < fileCount; ++i) { + auto fileNumber = fileNumbers[i]; + bool success = TransferFile(m_logManager->FileSystem(), fileNumber); + if (success) { // File transferred to the server successfully, so we can delete locally - m_logManager->RemoveLogFile(file_id); + m_logManager->RemoveLogFile(fileNumber); } else { // File did not transfer, so we update the upload attempt metadata and move on - } unsigned long current_elapsed = millis(); if ((current_elapsed - start_time) > m_uploadDuration) { @@ -99,6 +100,7 @@ void UploadManager::UploadCycle(void) break; } } + delete [] fileNumbers; } class SecureClient { From 90f0daa35c161b6f5c9f649630fcad624728ec80 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 28 Apr 2026 12:33:44 -0300 Subject: [PATCH 02/16] Shrink file number lists to their contents While dealing with heap shortages, I realized that we were always holding onto 4 KiB array, even if only the first few entries were actually populated. Four kibibytes isn't a massive amount, but when running near capacity, it might be enough to push us over the edge into SSL allocation failures. To do this shrink-to-fit operation consistently, I replaced one overload of logging::Manager::CountLogFiles() with a new GetLogFileNumbers() method that returns a vector, which provides much nicer ergonomics to the caller. --- LoggerFirmware/include/LogManager.h | 5 ++- LoggerFirmware/src/AutoUpload.cpp | 6 +-- LoggerFirmware/src/LogManager.cpp | 57 +++++++++++++++-------------- LoggerFirmware/src/Status.cpp | 17 +++++---- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/LoggerFirmware/include/LogManager.h b/LoggerFirmware/include/LogManager.h index b0df99fd..ae13d9f5 100644 --- a/LoggerFirmware/include/LogManager.h +++ b/LoggerFirmware/include/LogManager.h @@ -73,10 +73,11 @@ class Manager { /// \brief Remove all log files currently available (use judiciously!) void RemoveAllLogfiles(void); - /// \brief Count the number of log files on the system - uint32_t CountLogFiles(uint32_t filenumbers[MaxLogFiles]); /// \brief Count the number of log files on the system uint32_t CountLogFiles(void); + + /// \brief Collect the file numbers of all log files on the system + std::vector GetLogFileNumbers(); class MD5Hash { public: diff --git a/LoggerFirmware/src/AutoUpload.cpp b/LoggerFirmware/src/AutoUpload.cpp index e9b27e84..2e5bcccf 100644 --- a/LoggerFirmware/src/AutoUpload.cpp +++ b/LoggerFirmware/src/AutoUpload.cpp @@ -82,10 +82,7 @@ void UploadManager::UploadCycle(void) return; } - auto * fileNumbers = new uint32_t[logger::MaxLogFiles]; - auto fileCount = m_logManager->CountLogFiles(fileNumbers); - for (int i = 0; i < fileCount; ++i) { - auto fileNumber = fileNumbers[i]; + for (auto fileNumber : m_logManager->GetLogFileNumbers()) { bool success = TransferFile(m_logManager->FileSystem(), fileNumber); if (success) { // File transferred to the server successfully, so we can delete locally @@ -100,7 +97,6 @@ void UploadManager::UploadCycle(void) break; } } - delete [] fileNumbers; } class SecureClient { diff --git a/LoggerFirmware/src/LogManager.cpp b/LoggerFirmware/src/LogManager.cpp index afc451e4..65702046 100644 --- a/LoggerFirmware/src/LogManager.cpp +++ b/LoggerFirmware/src/LogManager.cpp @@ -297,15 +297,14 @@ void Manager::RemoveAllLogfiles(void) /// Normally, this finds all of the log files on the SD card, counts the total number, and returns /// the internal reference numbers that are being used so that the user can request more details /// about a specific file later. In this dummy version, it simply prints a debug message, and returns -/// zero. +/// an empty vector. /// -/// \param filenumbers Fixed size array (of \a MaxLogFiles) written with the internal reference numbers for the files -/// \return Total number of files available on the SD card +/// \return An empty vector -int Manager::CountLogFiles(int filenumbers[MaxLogFiles]) +std::vector Manager::GetLogFileNumbers() { - Serial.println("DBG: Call to count log files; returning zero."); - return 0; + Serial.println("DBG: Call to get log files numbers; returning empty vector."); + return {}; } /// \brief Determine the size and file name of the specified file @@ -506,54 +505,56 @@ bool Manager::RemoveLogFile(uint32_t file_num) void Manager::RemoveAllLogfiles(void) { - uint32_t *filenumbers = new uint32_t[MaxLogFiles]; + auto fileNumbers = GetLogFileNumbers(); CloseLogfile(); // All means all ... - uint32_t filecount = CountLogFiles(filenumbers); uint32_t files_closed = 0; - for (uint32_t f = 0; f < filecount; ++f) { - String filename = MakeLogName(filenumbers[f]); + for (auto fileNumber : fileNumbers) { + String filename = MakeLogName(fileNumber); Serial.printf("INFO: erasing log file: \"%s\".\n", filename.c_str()); bool rc = m_storage->Controller().remove(filename); if (rc) { m_consoleLog.printf("INFO: erased log file \"%s\" by user command.\n", filename.c_str()); ++files_closed; - if (m_inventory != nullptr) m_inventory->RemoveLogFile(filenumbers[f]); + if (m_inventory != nullptr) m_inventory->RemoveLogFile(fileNumber); } else { m_consoleLog.printf("ERR: failed to erase log file \"%s\" by user command.\n", filename.c_str()); } } - delete[] filenumbers; - m_consoleLog.printf("INFO: erased %u log files of %u.\n", files_closed, filecount); + + m_consoleLog.printf("INFO: erased %u log files of %u.\n", files_closed, fileNumbers.size()); m_consoleLog.flush(); StartNewLog(); // We need to have something running for the logging effort! } -/// Count the number of log files on the SD card, so that the client can enumerate them -/// and report to the user. -/// -/// \param filenumbers (Out) Array of the log file numbers on card -/// \return Number of files on the SD card - -uint32_t Manager::CountLogFiles(uint32_t filenumbers[MaxLogFiles]) +uint32_t Manager::CountLogFiles(void) { uint32_t filecount; if (m_inventory != nullptr) { - filecount = m_inventory->CountLogFiles(filenumbers); + filecount = m_inventory->CountLogFiles(); } else - filecount = count(filenumbers); + filecount = count(nullptr); return filecount; } -uint32_t Manager::CountLogFiles(void) +std::vector Manager::GetLogFileNumbers() { - uint32_t filecount; + // Temporarily allocate an array of max size... + auto * temp = new uint32_t[logger::MaxLogFiles]; + + size_t fileCount; if (m_inventory != nullptr) { - filecount = m_inventory->CountLogFiles(); - } else - filecount = count(nullptr); - return filecount; + fileCount = m_inventory->CountLogFiles(temp); + } else { + fileCount = count(temp); + } + + std::vector fileNumbers; + fileNumbers.assign(temp, temp + fileCount); + + delete [] temp; + return fileNumbers; } /// Make a list of all of the files that exist on the SD card in the log directory, along with their diff --git a/LoggerFirmware/src/Status.cpp b/LoggerFirmware/src/Status.cpp index 04daa051..de91141b 100644 --- a/LoggerFirmware/src/Status.cpp +++ b/LoggerFirmware/src/Status.cpp @@ -44,19 +44,21 @@ namespace status { DynamicJsonDocument GenerateFilelist(logger::Manager *m) { - uint32_t *filenumbers = new uint32_t[logger::MaxLogFiles]; - uint32_t n_files = m->CountLogFiles(filenumbers); - DynamicJsonDocument doc(100*n_files + 256); // Approximate guess, but can expand + auto fileNumbers = m->GetLogFileNumbers(); + + DynamicJsonDocument doc(100 * fileNumbers.size() + 256); // Approximate guess, but can expand + + doc["files"]["count"] = fileNumbers.size(); + for (int n = 0; n < fileNumbers.size(); ++n) { + auto fileNumber = fileNumbers[n]; - doc["files"]["count"] = n_files; - for (int n = 0; n < n_files; ++n) { String filename; uint32_t filesize; logger::Manager::MD5Hash filehash; uint16_t uploadCount; - m->EnumerateLogFile(filenumbers[n], filename, filesize, filehash, uploadCount); + m->EnumerateLogFile(fileNumber, filename, filesize, filehash, uploadCount); StaticJsonDocument<256> entry; - entry["id"] = filenumbers[n]; + entry["id"] = fileNumber; entry["len"] = filesize; if (!filehash.Empty()) entry["md5"] = filehash.Value(); @@ -88,7 +90,6 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) doc["files"]["detail"].add(entry.as()); } } - delete[] filenumbers; return doc; } From 9c91835c0aad0c8588863d2b6da55fab65bd819b Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Wed, 29 Apr 2026 12:10:46 -0300 Subject: [PATCH 03/16] Show question marks for missing file detail fields Previously any missing field would cause the whole row to be dropped. Given that MD5 hashes aren't guaranteed to be present, that felt like overkill. --- LoggerFirmware/website/js/status.js | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/LoggerFirmware/website/js/status.js b/LoggerFirmware/website/js/status.js index a22b8092..67d719ab 100644 --- a/LoggerFirmware/website/js/status.js +++ b/LoggerFirmware/website/js/status.js @@ -224,21 +224,16 @@ function updateStatus(tablePrefix) { let detail = document.getElementById(detailTable); if (detail !== null) { detail.replaceChildren(assembleDetailHeader()); - for (let n = 0; n < data.files.count; ++n) { - if (data.files.detail[n].hasOwnProperty('id') && - data.files.detail[n].hasOwnProperty('len') && - data.files.detail[n].hasOwnProperty('md5') && - data.files.detail[n].hasOwnProperty('url') && - data.files.detail[n].hasOwnProperty('uploads')) - detail.appendChild( - assembleDetailRow( - data.files.detail[n].id, - data.files.detail[n].len, - data.files.detail[n].md5, - data.files.detail[n].url, - data.files.detail[n].uploads - ) - ); + for (const entry of data.files.detail) { + detail.appendChild( + assembleDetailRow( + entry.id ?? "?", + entry.len ?? "?", + entry.md5 ?? "?", + entry.url ?? "?", + entry.uploads ?? "?" + ) + ); } } }); From 1d44de77030312fd324d60ddf2d8238c694d2177 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Wed, 29 Apr 2026 18:38:35 -0300 Subject: [PATCH 04/16] Allow CountLogFiles to optionally determine total size My goal is to pull the file list out of the status JSON document, which will require calculating the total file count and size separately from the full file list. --- LoggerFirmware/include/LogManager.h | 6 ++-- LoggerFirmware/src/LogManager.cpp | 47 ++++++++++++----------------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/LoggerFirmware/include/LogManager.h b/LoggerFirmware/include/LogManager.h index ae13d9f5..2ef71b0a 100644 --- a/LoggerFirmware/include/LogManager.h +++ b/LoggerFirmware/include/LogManager.h @@ -74,7 +74,7 @@ class Manager { void RemoveAllLogfiles(void); /// \brief Count the number of log files on the system - uint32_t CountLogFiles(void); + uint32_t CountLogFiles(uint64_t * fileSize = nullptr); /// \brief Collect the file numbers of all log files on the system std::vector GetLogFileNumbers(); @@ -161,7 +161,7 @@ class Manager { bool Update(uint32_t filenum, MD5Hash *hash = nullptr); void RemoveLogFile(uint32_t filenum); uint32_t CountLogFiles(uint32_t filenumbers[MaxLogFiles]); - uint32_t CountLogFiles(void); + uint32_t CountLogFiles(uint64_t * fileSize); uint32_t GetNextLogNumber(void); uint32_t Filesize(uint32_t filenum); uint16_t UploadCount(uint32_t filenum); @@ -193,7 +193,7 @@ class Manager { /// \brief Extract a log number from a filename (if valid) int32_t ExtractLogNumber(String const& filename); /// \brief Count the number of log files on the system - uint32_t count(uint32_t *filenumbers); + uint32_t scanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize); /// \brief Extract information on a single log file void enumerate(uint32_t lognumber, String& filename, uint32_t& filesize); /// \brief Generate a hash for a given file diff --git a/LoggerFirmware/src/LogManager.cpp b/LoggerFirmware/src/LogManager.cpp index 65702046..e01211a5 100644 --- a/LoggerFirmware/src/LogManager.cpp +++ b/LoggerFirmware/src/LogManager.cpp @@ -99,8 +99,7 @@ Manager::Inventory::~Inventory(void) bool Manager::Inventory::Reinitialise(void) { uint32_t *filenumbers = new uint32_t[MaxLogFiles]; - uint32_t filecount = m_logManager->count(filenumbers); - String filename; + auto filecount = m_logManager->scanLogFolder(filenumbers, nullptr); Manager::MD5Hash emptyhash; if (m_verbose) @@ -166,12 +165,15 @@ uint32_t Manager::Inventory::CountLogFiles(uint32_t filenumbers[MaxLogFiles]) return filecount; } -uint32_t Manager::Inventory::CountLogFiles(void) +uint32_t Manager::Inventory::CountLogFiles(uint64_t * totalSizePtr) { + uint64_t totalSize = 0; uint32_t filecount = 0; - for (uint32_t entry = 0; entry < MaxLogFiles; ++entry) { - if (m_filesize[entry] != 0) ++filecount; + for (auto fileSize : m_filesize) { + totalSize += fileSize; + filecount += (fileSize != 0); } + if (totalSizePtr) *totalSizePtr = totalSize; return filecount; } @@ -528,14 +530,10 @@ void Manager::RemoveAllLogfiles(void) StartNewLog(); // We need to have something running for the logging effort! } -uint32_t Manager::CountLogFiles(void) +uint32_t Manager::CountLogFiles(uint64_t * fileSize) { - uint32_t filecount; - if (m_inventory != nullptr) { - filecount = m_inventory->CountLogFiles(); - } else - filecount = count(nullptr); - return filecount; + return m_inventory ? m_inventory->CountLogFiles(fileSize) + : scanLogFolder(nullptr, fileSize); } std::vector Manager::GetLogFileNumbers() @@ -543,15 +541,9 @@ std::vector Manager::GetLogFileNumbers() // Temporarily allocate an array of max size... auto * temp = new uint32_t[logger::MaxLogFiles]; - size_t fileCount; - if (m_inventory != nullptr) { - fileCount = m_inventory->CountLogFiles(temp); - } else { - fileCount = count(temp); - } - - std::vector fileNumbers; - fileNumbers.assign(temp, temp + fileCount); + auto fileCount = m_inventory ? m_inventory->CountLogFiles(temp) + : scanLogFolder(temp, nullptr); + auto fileNumbers = std::vector(temp, temp + fileCount); delete [] temp; return fileNumbers; @@ -819,23 +811,22 @@ void Manager::TransferLogFile(uint32_t file_num, MD5Hash const& filehash, Stream Serial.printf("Sent %u B in %lu s.\n", bytes_transferred, duration); } -uint32_t Manager::count(uint32_t *filenumbers) +uint32_t Manager::scanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize) { uint32_t file_count = 0; File logdir = m_storage->Controller().open("/logs"); - File entry = logdir.openNextFile(); - - while (entry) { + while (File entry = logdir.openNextFile()) { // Because we can store snapshots of configuration information in the /logs directory // (so that they can be seen through the webserver's static website for download), we // need to count only the valid log files in the directory. int32_t lognumber = ExtractLogNumber(String(entry.name())); - if (lognumber >= 0) { - if (filenumbers != nullptr) filenumbers[file_count] = lognumber; + auto size = entry.size(); + if (lognumber >= 0 && size > 0) { + if (fileNumbers != nullptr) fileNumbers[file_count] = lognumber; + if (totalSize) *totalSize += size; ++file_count; } entry.close(); - entry = logdir.openNextFile(); } logdir.close(); return file_count; From c2f1724c4e9330fe6d187883729c52950dcc9ae4 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Mon, 11 May 2026 15:15:33 -0300 Subject: [PATCH 05/16] Allow JSON to be moved directly into the WiFi message buffer I'm running into situations where a large JSON document is being successfully generated, but then allocation fails when trying to copy it into ESP32WiFiAdapter::m_messages. Move semantics to the rescue! --- LoggerFirmware/include/WiFiAdapter.h | 6 ++++-- LoggerFirmware/src/SerialCommand.cpp | 12 +++++------- LoggerFirmware/src/WiFiAdapter.cpp | 14 ++++++-------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/LoggerFirmware/include/WiFiAdapter.h b/LoggerFirmware/include/WiFiAdapter.h index cfa84042..f9d4c7c5 100644 --- a/LoggerFirmware/include/WiFiAdapter.h +++ b/LoggerFirmware/include/WiFiAdapter.h @@ -57,7 +57,8 @@ class WiFiAdapter { void AddMessage(String const& message); /// \brief Replace the entire message to be returned to the client for the current transaction - void SetMessage(DynamicJsonDocument const& message); + void SetMessage(JsonDocument const& message) { setMessage(message); } + void SetMessage(DynamicJsonDocument && message) { setMessage(std::move(message)); } enum HTTPReturnCodes { OK = 200, // The request succeeded @@ -103,7 +104,8 @@ class WiFiAdapter { virtual void accumulateMessage(String const& message) = 0; /// \brief Sub-class implementation of code to replace message for transmission - virtual void setMessage(DynamicJsonDocument const& message) = 0; + virtual void setMessage(JsonDocument const& message) = 0; + virtual void setMessage(DynamicJsonDocument && message) = 0; /// \brief Sub-class implementation of code to set the status code for the transaction virtual void setStatusCode(HTTPReturnCodes status_code) = 0; diff --git a/LoggerFirmware/src/SerialCommand.cpp b/LoggerFirmware/src/SerialCommand.cpp index c4e3c29b..6766d45b 100644 --- a/LoggerFirmware/src/SerialCommand.cpp +++ b/LoggerFirmware/src/SerialCommand.cpp @@ -787,7 +787,7 @@ void SerialCommand::ReportConfigurationJSON(CommandSource src, bool secure) EmitMessage(s + "\n", src); } else { if (m_wifi != nullptr) - m_wifi->SetMessage(json); + m_wifi->SetMessage(std::move(json)); } } @@ -922,8 +922,7 @@ void SerialCommand::DisplayAlgorithmStore(logger::AlgoRequestStore& store, Comma String algorithms(store.JSONRepresentation(true)); EmitMessage(algorithms + '\n', src); } else if (src == CommandSource::WirelessPort) { - DynamicJsonDocument doc(store.GetContents()); - m_wifi->SetMessage(doc); + m_wifi->SetMessage(store.GetContents()); } else { EmitMessage("ERR: request for unknown CommandSource - who are you?\n", src); } @@ -1022,8 +1021,7 @@ void SerialCommand::DisplayNMEAFilter(logger::N0183IDStore& filter, CommandSourc String filter_ids(filter.JSONRepresentation(true)); EmitMessage(filter_ids + '\n', src); } else if (src == CommandSource::WirelessPort) { - DynamicJsonDocument doc(filter.GetContents()); - m_wifi->SetMessage(doc); + m_wifi->SetMessage(filter.GetContents()); } else { EmitMessage("ERR: request for unknown CommandSource - who are you?\n", src); } @@ -1174,7 +1172,7 @@ void SerialCommand::ReportCurrentStatus(CommandSource src) EmitMessage(json+"\n", src); } else { if (m_wifi != nullptr) { - m_wifi->SetMessage(status); + m_wifi->SetMessage(std::move(status)); } } } @@ -1789,7 +1787,7 @@ bool SerialCommand::EmitJSON(String const& source, CommandSource chan) break; case CommandSource::WirelessPort: if (m_wifi != nullptr) { - m_wifi->SetMessage(json); + m_wifi->SetMessage(std::move(json)); } break; default: diff --git a/LoggerFirmware/src/WiFiAdapter.cpp b/LoggerFirmware/src/WiFiAdapter.cpp index 9bbc332d..19c49ca4 100644 --- a/LoggerFirmware/src/WiFiAdapter.cpp +++ b/LoggerFirmware/src/WiFiAdapter.cpp @@ -553,11 +553,15 @@ class ESP32WiFiAdapter : public WiFiAdapter { /// /// \param message \a String to use for the return message. /// \return N/A - - void setMessage(DynamicJsonDocument const& message) + void setMessage(JsonDocument const& message) { *m_messages = message; } + void setMessage(DynamicJsonDocument && message) + { + *m_messages = std::move(message); + + } /// Configure the HTTP status code that the response should use. By default, the status code used /// when the response is finally sent to the client is 200 OK. However, if there is an error @@ -633,12 +637,6 @@ bool WiFiAdapter::TransferFile(String const& filename, uint32_t filesize, logger /// \return N/A void WiFiAdapter::AddMessage(String const& message) { accumulateMessage(message); } -/// Pass-through implementation to the sub-class code to reset the message from straight JSON -/// -/// @param message JSON document to send back to the client -/// @return N/A -void WiFiAdapter::SetMessage(DynamicJsonDocument const& message) { setMessage(message); } - /// Pass-through implementation to the sub-class code to set status code /// /// \param status_code HTTP status code to set From 9e0d49547fc27529aad87b7d83a0df2332dfe040 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 09:03:42 -0300 Subject: [PATCH 06/16] Store ESP32WiFiAdapter::m_messages by value instead of pointer Keeping the document itself on the heap only introduced complexity which we didn't really need. Also store the default buffer size in a named constant. --- LoggerFirmware/src/WiFiAdapter.cpp | 35 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/LoggerFirmware/src/WiFiAdapter.cpp b/LoggerFirmware/src/WiFiAdapter.cpp index 19c49ca4..09ae2bbd 100644 --- a/LoggerFirmware/src/WiFiAdapter.cpp +++ b/LoggerFirmware/src/WiFiAdapter.cpp @@ -381,14 +381,11 @@ class ESP32WiFiAdapter : public WiFiAdapter { /// Default constructor for the ESP32 adapter. This brings up the parameter store to use /// for WiFi parameters, but takes no other action until the user explicitly starts the AccessPoint. ESP32WiFiAdapter(void) - : m_storage(nullptr), m_server(nullptr), m_messages(nullptr), m_statusCode(HTTPReturnCodes::OK) + : m_storage(nullptr), m_server(nullptr), m_messages(defaultMessageBufferSize), m_statusCode(HTTPReturnCodes::OK) { if ((m_storage = mem::MemControllerFactory::Create()) == nullptr) { return; } - if ((m_messages = new DynamicJsonDocument(1024)) == nullptr) { - return; - } } /// Default destructor for the ESP32 adapter. This stops the Access Point, if it isn't already down, /// and then deletes the ParamStore manipulation object. @@ -396,14 +393,15 @@ class ESP32WiFiAdapter : public WiFiAdapter { { stop(); delete m_storage; // Note that we're not stopping the interface, since it may still be required elsewhere - delete m_messages; } private: + static constexpr size_t defaultMessageBufferSize = 1024; + mem::MemController *m_storage; ///< Pointer to the storage object to use ExtendedWebServer *m_server; ///< Pointer to the server object, if started. std::queue m_commands; ///< Queue to handle commands sent by the user - DynamicJsonDocument *m_messages; ///< Accumulating message content to be send to the client + DynamicJsonDocument m_messages; ///< Accumulating message content to be send to the client HTTPReturnCodes m_statusCode; ///< Status code to return to the user with the transaction response ConnectionStateMachine m_state; ///< Manager for connection state @@ -533,19 +531,18 @@ class ESP32WiFiAdapter : public WiFiAdapter { void accumulateMessage(String const& message) { - if ((m_messages->memoryUsage() + message.length()) > 0.95*m_messages->capacity()) { + if ((m_messages.memoryUsage() + message.length()) > 0.95 * m_messages.capacity()) { // Expand capacity to ensure that the message will be added successfully - size_t new_capacity = m_messages->capacity() * 2; - DynamicJsonDocument *new_doc = new DynamicJsonDocument(new_capacity); - if (new_doc != nullptr) { - new_doc->set(*m_messages); - delete m_messages; - m_messages = new_doc; + size_t new_capacity = m_messages.capacity() * 2; + DynamicJsonDocument new_doc(new_capacity); + if (new_doc.capacity() != 0) { + new_doc.set(m_messages); + m_messages = std::move(new_doc); } } - if (!(*m_messages)["messages"].add(message)) { + if (!m_messages["messages"].add(message)) { Serial.printf("ERR: failed to add message to accumulation buffer (capacity %d bytes); messages may be truncated.\n", - m_messages->capacity()); + m_messages.capacity()); } } @@ -555,11 +552,11 @@ class ESP32WiFiAdapter : public WiFiAdapter { /// \return N/A void setMessage(JsonDocument const& message) { - *m_messages = message; + m_messages = message; } void setMessage(DynamicJsonDocument && message) { - *m_messages = std::move(message); + m_messages = std::move(message); } @@ -586,10 +583,10 @@ class ESP32WiFiAdapter : public WiFiAdapter { bool transmitMessages(void) { String message; - serializeJson(*m_messages, message); + serializeJson(m_messages, message); //Serial.printf("DBG: WiFi transmitting response |%s|\n", message.c_str()); m_server->send(m_statusCode, "application/json", message); - m_messages->clear(); + m_messages.clear(); m_statusCode = HTTPReturnCodes::OK; // "OK" by default return true; } From ac066fecd20723270f0a3d36a135b707d764a62b Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 09:07:43 -0300 Subject: [PATCH 07/16] Remove declared-but-never-defined SerialCommand::GenerateFilelist --- LoggerFirmware/include/SerialCommand.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/LoggerFirmware/include/SerialCommand.h b/LoggerFirmware/include/SerialCommand.h index b8b41f8d..b3ab7c28 100644 --- a/LoggerFirmware/include/SerialCommand.h +++ b/LoggerFirmware/include/SerialCommand.h @@ -195,8 +195,6 @@ class SerialCommand { void EmitMessage(String const& msg, CommandSource src); /// \brief Convert a stringified JSON into a document, with error reporting bool EmitJSON(String const& source, CommandSource src); - /// @brief Generate a list of files into a JSON document - DynamicJsonDocument GenerateFilelist(void); /// @brief Display a NMEA0183 filter ID list void DisplayNMEAFilter(logger::N0183IDStore& filter, CommandSource src); /// @brief Display an Algorithm Store list From 3ba1ce6412cd9f28581af8a4cb6e485061eb42b1 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 11:06:13 -0300 Subject: [PATCH 08/16] Shrink ESP32WiFiAdapter's JSON document if it gets to large This document's capacity increased every time a large message was sent, but that memory was never recovered. So if we succeeded in sending a giant file list, ESP32WiFiAdapter would hold onto that memory forever. --- LoggerFirmware/src/WiFiAdapter.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/LoggerFirmware/src/WiFiAdapter.cpp b/LoggerFirmware/src/WiFiAdapter.cpp index 09ae2bbd..b9247866 100644 --- a/LoggerFirmware/src/WiFiAdapter.cpp +++ b/LoggerFirmware/src/WiFiAdapter.cpp @@ -586,7 +586,15 @@ class ESP32WiFiAdapter : public WiFiAdapter { serializeJson(m_messages, message); //Serial.printf("DBG: WiFi transmitting response |%s|\n", message.c_str()); m_server->send(m_statusCode, "application/json", message); - m_messages.clear(); + + // If the JSON document got too large, replace it with a smaller one to avoid needlessly + // holding on to too much memory. Otherwise, just clear the document and reuse it. + if (defaultMessageBufferSize < m_messages.capacity()) { + m_messages = DynamicJsonDocument(defaultMessageBufferSize); + } else { + m_messages.clear(); + } + m_statusCode = HTTPReturnCodes::OK; // "OK" by default return true; } From 1054b78327e133da7c0703efe98ff8bf716cf3ee Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 12:28:00 -0300 Subject: [PATCH 09/16] Introduce a common functions to grow the capacity of a DynamicJsonDocument This is a subtle dance that was repeated a few times. All of those copies were missing a std::move() call that significantly reduces the peak memory usage of the operation. To me that justifies abstracting it out. --- LoggerFirmware/include/JSONUtilities.h | 27 ++++++++++++++++++++++++++ LoggerFirmware/src/NVMFile.cpp | 6 ++---- LoggerFirmware/src/Status.cpp | 12 ++++-------- LoggerFirmware/src/WiFiAdapter.cpp | 13 +++++-------- 4 files changed, 38 insertions(+), 20 deletions(-) create mode 100644 LoggerFirmware/include/JSONUtilities.h diff --git a/LoggerFirmware/include/JSONUtilities.h b/LoggerFirmware/include/JSONUtilities.h new file mode 100644 index 00000000..226fd088 --- /dev/null +++ b/LoggerFirmware/include/JSONUtilities.h @@ -0,0 +1,27 @@ + + +#ifndef JSONUTILITIES_H +#define JSONUTILITIES_H + +#include + +inline bool GrowJsonDocumentBy(DynamicJsonDocument & json, size_t capacityIncrease) +{ + auto newCapacity = json.capacity() + capacityIncrease; + DynamicJsonDocument temp(newCapacity); + + // Allocation failed. Nothing we can do. + if (temp.capacity() == 0) return false; + + temp.set(json); + json = std::move(temp); + return true; +} + +inline bool GrowJsonDocument(DynamicJsonDocument & json) +{ + // Double the current capacity. + return GrowJsonDocumentBy(json, json.capacity()); +} + +#endif // JSONUTILITIES_H diff --git a/LoggerFirmware/src/NVMFile.cpp b/LoggerFirmware/src/NVMFile.cpp index eb2aff47..891338aa 100644 --- a/LoggerFirmware/src/NVMFile.cpp +++ b/LoggerFirmware/src/NVMFile.cpp @@ -26,6 +26,7 @@ #include #include "NVMFile.h" +#include "JSONUtilities.h" #include "LittleFS.h" #include "LogManager.h" #include "serialisation.h" @@ -348,10 +349,7 @@ void AlgoRequestStore::AddAlgorithm(String const& alg_name, String const& alg_pa entry["parameters"] = alg_params; if ((doc.memoryUsage() + entry.capacity()) > 0.95*doc.capacity()) { // The document needs to be bigger to handle the new entry - int capacity = doc.memoryUsage() + entry.capacity() + 256; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; + GrowJsonDocumentBy(doc, entry.capacity()); } doc["algorithm"].add(entry.as()); doc["count"] = count + 1; diff --git a/LoggerFirmware/src/Status.cpp b/LoggerFirmware/src/Status.cpp index de91141b..b90d9b7f 100644 --- a/LoggerFirmware/src/Status.cpp +++ b/LoggerFirmware/src/Status.cpp @@ -25,6 +25,7 @@ */ #include "ArduinoJson.h" +#include "JSONUtilities.h" #include "LogManager.h" #include "Configuration.h" #include "N0183Logger.h" @@ -70,10 +71,7 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) // It's likely that adding the entry will cause the document to run out of // space. We therefore double the capacity (since it's expensive, and we // don't want to have to do it more than once) before we attempt the addition. - int capacity = doc.capacity() * 2; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; + if(!GrowJsonDocument(doc)) return doc; } if (!doc["files"]["detail"].add(entry.as())) { // Out of memory, even after adding more above (most likely), @@ -83,10 +81,8 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) // element before continuing -- luckily, it should be at the index in // the array of the file number. doc["files"]["detail"].remove(n); - int capacity = doc.capacity() * 2; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; + + if(!GrowJsonDocument(doc)) return doc; doc["files"]["detail"].add(entry.as()); } } diff --git a/LoggerFirmware/src/WiFiAdapter.cpp b/LoggerFirmware/src/WiFiAdapter.cpp index b9247866..831a58c5 100644 --- a/LoggerFirmware/src/WiFiAdapter.cpp +++ b/LoggerFirmware/src/WiFiAdapter.cpp @@ -35,6 +35,7 @@ #include #include +#include "JSONUtilities.h" #include "LogManager.h" #include "WiFiAdapter.h" #include "Configuration.h" @@ -531,14 +532,10 @@ class ESP32WiFiAdapter : public WiFiAdapter { void accumulateMessage(String const& message) { - if ((m_messages.memoryUsage() + message.length()) > 0.95 * m_messages.capacity()) { - // Expand capacity to ensure that the message will be added successfully - size_t new_capacity = m_messages.capacity() * 2; - DynamicJsonDocument new_doc(new_capacity); - if (new_doc.capacity() != 0) { - new_doc.set(m_messages); - m_messages = std::move(new_doc); - } + bool needToGrowBuffer = (m_messages.memoryUsage() + message.length()) > 0.95 * m_messages.capacity(); + if (needToGrowBuffer && !GrowJsonDocument(m_messages)) { + Serial.printf("ERR: failed to grow buffer beyond %d bytes; messages will be lost.\n", m_messages.capacity()); + return; } if (!m_messages["messages"].add(message)) { Serial.printf("ERR: failed to add message to accumulation buffer (capacity %d bytes); messages may be truncated.\n", From cc8e8af7ceacb1d407d1d459173bc4fea7054d49 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 12:39:00 -0300 Subject: [PATCH 10/16] =?UTF-8?q?Reduce=20DynamicJsonDocument=20growth=20f?= =?UTF-8?q?actor=20to=201.5=C3=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given that our large JSON documents can grow to fill temporarily fill all available memory, a more conservate growth factor seems appropriate. 1. Less aggressive growth means we are more likely to "just" fit in memory. 2. A growth factor less than the golden ratio allows the possibility of reusing memory blocks the document previously occupied. A growth factor of 2 prevents this. --- LoggerFirmware/include/JSONUtilities.h | 5 +++-- LoggerFirmware/src/Status.cpp | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LoggerFirmware/include/JSONUtilities.h b/LoggerFirmware/include/JSONUtilities.h index 226fd088..2dcb7e99 100644 --- a/LoggerFirmware/include/JSONUtilities.h +++ b/LoggerFirmware/include/JSONUtilities.h @@ -20,8 +20,9 @@ inline bool GrowJsonDocumentBy(DynamicJsonDocument & json, size_t capacityIncrea inline bool GrowJsonDocument(DynamicJsonDocument & json) { - // Double the current capacity. - return GrowJsonDocumentBy(json, json.capacity()); + // Grow capacity by 50%. A growth rate less than the golden ratio allows + // the document to potentially reuse allocations it previously occupied. + return GrowJsonDocumentBy(json, json.capacity() / 2); } #endif // JSONUTILITIES_H diff --git a/LoggerFirmware/src/Status.cpp b/LoggerFirmware/src/Status.cpp index b90d9b7f..630cbb1e 100644 --- a/LoggerFirmware/src/Status.cpp +++ b/LoggerFirmware/src/Status.cpp @@ -69,8 +69,7 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) int entry_size = entry.memoryUsage(); if (doc.memoryUsage() + entry_size > 0.95*doc.capacity()) { // It's likely that adding the entry will cause the document to run out of - // space. We therefore double the capacity (since it's expensive, and we - // don't want to have to do it more than once) before we attempt the addition. + // space. We therefore increase the capacity before we attempt the addition. if(!GrowJsonDocument(doc)) return doc; } if (!doc["files"]["detail"].add(entry.as())) { From 9dbc0aec7450d4f3f6c45c136e0979eef8ccc0ba Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 13:27:23 -0300 Subject: [PATCH 11/16] Add a SerialCommand::EmitJSON() overload that takes a JSON document There was a lot of unnecessary repetition between commands. Other improvements: * Serializing directly to Serial instead of allocating a large temporary string, like some jobs were doing. * Using Serial.println() to print the trailing new line instead of appending a single character to a large string, potentially resulting in another large allocation and copy. * Avoid some unnecessary JsonDocument->String->JsonDocument round-trips. --- LoggerFirmware/include/SerialCommand.h | 4 +- LoggerFirmware/src/SerialCommand.cpp | 63 ++++++++++---------------- 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/LoggerFirmware/include/SerialCommand.h b/LoggerFirmware/include/SerialCommand.h index b3ab7c28..ec69ddfb 100644 --- a/LoggerFirmware/include/SerialCommand.h +++ b/LoggerFirmware/include/SerialCommand.h @@ -193,8 +193,10 @@ class SerialCommand { /// \brief Generate a string on the appropriate output stream void EmitMessage(String const& msg, CommandSource src); - /// \brief Convert a stringified JSON into a document, with error reporting + /// \brief Send stringified JSON, with error reporting bool EmitJSON(String const& source, CommandSource src); + /// \brief Send a JSON document, with error reporting + void EmitJSON(DynamicJsonDocument && json, CommandSource src); /// @brief Display a NMEA0183 filter ID list void DisplayNMEAFilter(logger::N0183IDStore& filter, CommandSource src); /// @brief Display an Algorithm Store list diff --git a/LoggerFirmware/src/SerialCommand.cpp b/LoggerFirmware/src/SerialCommand.cpp index 6766d45b..3413ba08 100644 --- a/LoggerFirmware/src/SerialCommand.cpp +++ b/LoggerFirmware/src/SerialCommand.cpp @@ -780,15 +780,7 @@ void SerialCommand::ConfigurePassthrough(String const& params, CommandSource src void SerialCommand::ReportConfigurationJSON(CommandSource src, bool secure) { - DynamicJsonDocument json(logger::ConfigJSON::ExtractConfig()); - if (src == CommandSource::SerialPort) { - String s; - serializeJsonPretty(json, s); - EmitMessage(s + "\n", src); - } else { - if (m_wifi != nullptr) - m_wifi->SetMessage(std::move(json)); - } + EmitJSON(logger::ConfigJSON::ExtractConfig(), src); } /// Provide summary report of all of the configuration parameters being managed by the Confgiuration @@ -1164,17 +1156,7 @@ void SerialCommand::ConfigureWebserver(String const& params, CommandSource src) void SerialCommand::ReportCurrentStatus(CommandSource src) { - DynamicJsonDocument status(logger::status::CurrentStatus(m_logManager)); - - if (src == CommandSource::SerialPort) { - String json; - serializeJsonPretty(status, json); - EmitMessage(json+"\n", src); - } else { - if (m_wifi != nullptr) { - m_wifi->SetMessage(std::move(status)); - } - } + EmitJSON(logger::status::CurrentStatus(m_logManager), src); } /// Report the current set of "lab default" configuration parameters set on the logger. These @@ -1382,14 +1364,12 @@ void SerialCommand::SnapshotResource(String const& resource, CommandSource src) EmitMessage("ERR: unknown snapshot resource requested.\n", src); } - StaticJsonDocument<256> responseJson; + DynamicJsonDocument responseJson(256); if (rc) responseJson["url"] = url; else responseJson["url"] = ""; - String response; - serializeJson(responseJson, response); - EmitJSON(response, src); + EmitJSON(std::move(responseJson), src); } void SerialCommand::ReportUploadConfig(CommandSource src) @@ -1780,19 +1760,26 @@ bool SerialCommand::EmitJSON(String const& source, CommandSource chan) EmitMessage("No data in JSON.\n", chan); return false; } - DynamicJsonDocument json(logger::status::GenerateJSON(source)); - switch (chan) { - case CommandSource::SerialPort: - serializeJsonPretty(json, Serial); - break; - case CommandSource::WirelessPort: - if (m_wifi != nullptr) { - m_wifi->SetMessage(std::move(json)); - } - break; - default: - Serial.println("ERR: command source not recognised."); - break; - } + EmitJSON(logger::status::GenerateJSON(source), chan); return true; } + +/// Generate a message on the output stream associated with the given source, with +/// the message in JSON format. +/// +/// @param json JSON document +/// @param src Input channel on which the command arrived + +void SerialCommand::EmitJSON(DynamicJsonDocument && json, CommandSource src) +{ + if (src == CommandSource::SerialPort) { + serializeJsonPretty(json, Serial); + Serial.println(); + } else if (src == CommandSource::WirelessPort) { + if (m_wifi != nullptr) { + m_wifi->SetMessage(std::move(json)); + } + } else { + Serial.println("ERR: command source not recognised."); + } +} From cc63c325f4535651069d7f3f93d8bf89c2d39b5d Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 13:29:09 -0300 Subject: [PATCH 12/16] Add a new "catalog" command that sends the JSON file list Easy peasy. --- LoggerFirmware/include/SerialCommand.h | 2 ++ LoggerFirmware/src/SerialCommand.cpp | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/LoggerFirmware/include/SerialCommand.h b/LoggerFirmware/include/SerialCommand.h index ec69ddfb..3099e56a 100644 --- a/LoggerFirmware/include/SerialCommand.h +++ b/LoggerFirmware/include/SerialCommand.h @@ -166,6 +166,8 @@ class SerialCommand { void ReportScalesElement(CommandSource src); /// \brief Report the number of log files available on the SD card void ReportFileCount(CommandSource src); + /// \brief Report the properties of all log files on the SD card + void ReportCatalog(CommandSource src); /// \brief Report the configuration of the web-server void ReportWebserverConfig(CommandSource src); /// \brief Configure the web-server for system config/management diff --git a/LoggerFirmware/src/SerialCommand.cpp b/LoggerFirmware/src/SerialCommand.cpp index 3413ba08..ec3894d2 100644 --- a/LoggerFirmware/src/SerialCommand.cpp +++ b/LoggerFirmware/src/SerialCommand.cpp @@ -1089,6 +1089,11 @@ void SerialCommand::ReportFileCount(CommandSource src) EmitMessage(String(file_count) + "\n", src); } +void SerialCommand::ReportCatalog(CommandSource src) +{ + EmitJSON(logger::status::GenerateFilelist(m_logManager), src); +} + void SerialCommand::ReportWebserverConfig(CommandSource src) { if (src == CommandSource::SerialPort) { @@ -1538,6 +1543,8 @@ void SerialCommand::Execute(String const& cmd, CommandSource src) } else { SetAuthorisation(cmd.substring(5), src); } + } else if (cmd == "catalog") { + ReportCatalog(src); } else if (cmd.startsWith("configure")) { if (cmd.length() == 9) { ReportConfiguration(src); From 9b08ed4cec2766dae0f2644840c3b69c7c06c5c6 Mon Sep 17 00:00:00 2001 From: Parker Coates Date: Tue, 12 May 2026 13:56:53 -0300 Subject: [PATCH 13/16] Remove the file list from the status JSON message Just include the total file count and total file size. The file list JSON just isn't guaranteed to fit in memory, so it doesn't make sense to try to cram it inside the status JSON. A failed allocation means no status is sent, which is much more problematic than an incomplete file list. Moving "catalog" out into its own command means the WIBL can serve the status, release memory, then serve the file list. This approach also lets us handle a partial file list by appending a row of ellipses to the file table. --- LoggerFirmware/src/Status.cpp | 43 ++++++++++++++++------------- LoggerFirmware/website/js/status.js | 33 ++++++++++------------ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/LoggerFirmware/src/Status.cpp b/LoggerFirmware/src/Status.cpp index 630cbb1e..92122c12 100644 --- a/LoggerFirmware/src/Status.cpp +++ b/LoggerFirmware/src/Status.cpp @@ -47,9 +47,12 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) { auto fileNumbers = m->GetLogFileNumbers(); - DynamicJsonDocument doc(100 * fileNumbers.size() + 256); // Approximate guess, but can expand + // Start small. We really don't want this first allocation to fail. + DynamicJsonDocument doc(512); - doc["files"]["count"] = fileNumbers.size(); + // Assume the list will be truncated, because we might not have enough memory + // to set it to true later when allocation fails. + doc["files"]["isTruncated"] = true; for (int n = 0; n < fileNumbers.size(); ++n) { auto fileNumber = fileNumbers[n]; @@ -66,38 +69,35 @@ DynamicJsonDocument GenerateFilelist(logger::Manager *m) entry["url"] = filename; entry["uploads"] = uploadCount; - int entry_size = entry.memoryUsage(); - if (doc.memoryUsage() + entry_size > 0.95*doc.capacity()) { - // It's likely that adding the entry will cause the document to run out of - // space. We therefore increase the capacity before we attempt the addition. - if(!GrowJsonDocument(doc)) return doc; - } - if (!doc["files"]["detail"].add(entry.as())) { - // Out of memory, even after adding more above (most likely), - // so make a new allocation. Note that the addition can partially - // work, so you can end up with a malformed entry in the array (and - // one more entry than you expect). We therefore remove this last + while (!doc["files"]["detail"].add(entry.as())) { + // Not enough free capacity to add the entry. Note that the addition + // can partially work, so you can end up with ain malformed entry at + // the end of the array. We therefore remove this last // element before continuing -- luckily, it should be at the index in // the array of the file number. doc["files"]["detail"].remove(n); - - if(!GrowJsonDocument(doc)) return doc; - doc["files"]["detail"].add(entry.as()); + + if(!GrowJsonDocument(doc)) + { + // Allocation failed. Not really anything we can do other than + // return what we have. + return doc; + } } } + doc["files"]["isTruncated"] = false; return doc; } DynamicJsonDocument CurrentStatus(logger::Manager *m) { - DynamicJsonDocument filelist(GenerateFilelist(m)); DynamicJsonDocument lkg(logger::Metrics.LastKnownGood()); // The total capacity should be, approximately, the sum of the biggest components // (above), plus some limited information on versions, elapsed time, and boot // status. We're assuming here that 1024B is enough for the extras ... that // might not always be the case. - int capacity = filelist.capacity() + lkg.capacity() + 1024; + int capacity = lkg.capacity() + 1024; DynamicJsonDocument status(capacity); @@ -120,7 +120,12 @@ DynamicJsonDocument CurrentStatus(logger::Manager *m) status["webserver"]["boot"] = boot_status; status["data"] = lkg; - status["files"] = filelist["files"]; + + uint64_t totalFileSize = 0; + auto totalFileCount = m->CountLogFiles(&totalFileSize); + auto files = status.createNestedObject("files"); + files["totalFileCount"] = totalFileCount; + files["totalFileSize"] = totalFileSize; return status; } diff --git a/LoggerFirmware/website/js/status.js b/LoggerFirmware/website/js/status.js index 67d719ab..ee4db883 100644 --- a/LoggerFirmware/website/js/status.js +++ b/LoggerFirmware/website/js/status.js @@ -174,22 +174,15 @@ function updateStatus(tablePrefix) { versions.appendChild(assembleSummaryRow("NMEA2000", data.version.nmea2000)); versions.appendChild(assembleSummaryRow("IMU", data.version.imu)); versions.appendChild(assembleSummaryRow("Serialiser", data.version.serialiser)); - - let totalFileSize = 0; - for (let n = 0; n < data.files.count; ++n) { - if ('len' in data.files.detail[n] && isFinite(data.files.detail[n].len)) { - totalFileSize += data.files.detail[n].len; - } - } - + let stats = document.getElementById(statsTable); stats.replaceChildren(assembleSummaryHeader("Status")); stats.appendChild(assembleSummaryRow("Elapsed Time", translateTime(data.elapsed))); stats.appendChild(assembleSummaryRow("Supply Voltage", roundVoltage(data.supply))); stats.appendChild(assembleSummaryRow("Webserver Status Current", data.webserver.current)); stats.appendChild(assembleSummaryRow("Webserver Status Boot", data.webserver.boot)); - stats.appendChild(assembleSummaryRow("Files on Logger", data.files.count)); - stats.appendChild(assembleSummaryRow("Total Size", translateSize(totalFileSize))); + stats.appendChild(assembleSummaryRow("Files on Logger", data.files.totalFileCount)); + stats.appendChild(assembleSummaryRow("Total Size", translateSize(data.files.totalFileSize))); let nmea0183 = document.getElementById(n0183Table); if (nmea0183 !== null) { @@ -220,21 +213,23 @@ function updateStatus(tablePrefix) { data.data.nmea2000.detail[n].display)); } } - + let detail = document.getElementById(detailTable); if (detail !== null) { - detail.replaceChildren(assembleDetailHeader()); - for (const entry of data.files.detail) { - detail.appendChild( - assembleDetailRow( + sendCommand('catalog').then((data) => { + detail.replaceChildren(assembleDetailHeader()); + for (const entry of data.files.detail) { + detail.appendChild(assembleDetailRow( entry.id ?? "?", entry.len ?? "?", entry.md5 ?? "?", entry.url ?? "?", - entry.uploads ?? "?" - ) - ); - } + entry.uploads ?? "?")); + } + if (data.files.isTruncated) { + detail.appendChild(assembleDetailRow("...", "...", "...", "...", "...")); + } + }); } }); } From 4f22660719bebf109df285635a5eef6e5735fed4 Mon Sep 17 00:00:00 2001 From: brian-r-calder Date: Fri, 22 May 2026 18:05:40 -0400 Subject: [PATCH 14/16] fw: Mods to upload server to compensate for new status message structure on check-in. Bumped version to 1.6.2. --- LoggerFirmware/ReleaseNotes.md | 8 ++++++++ LoggerFirmware/include/Configuration.h | 2 +- UploadServer/README.md | 6 ++++-- UploadServer/src/api/api.go | 11 ++--------- UploadServer/wibl-monitor.go | 4 ++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/LoggerFirmware/ReleaseNotes.md b/LoggerFirmware/ReleaseNotes.md index 6b66b42d..83cc3949 100644 --- a/LoggerFirmware/ReleaseNotes.md +++ b/LoggerFirmware/ReleaseNotes.md @@ -1,5 +1,13 @@ # Release Notes: Logger Firmware +## Firmware 1.6.2 + +Firmware 1.6.2 addresses [issue 107](https://github.com/CCOMJHC/WIBL/issues/107) in the repository, where the size of the JSON documents being constructed in the firmware were getting to the size where they took up enough memory that they caused problems when generating SSL connections during upload. This also seemed to have significantly slowed down the webserver, so this version of the firmware should show much snappier connections on the server as a (very welcome) side effect. + +In addition to managing the memory better in the firmware, the modifications here change the status message so that it no longer provides all of the file information (names, sizes, checksums, etc.) The code adds a "catalog" command that now provides this information; the "snapshot catalog" command can also be used to generate this information as a file that can be downloaded using the standard browser mechanism. + +The [original PR](https://github.com/CCOMJHC/WIBL/pull/108) for this (which is used here with slight adjustments for the upload server end of the connection) was contributed by the awesome team at [Spatialnetics](https://spatialnetics.com). + ## Firmware 1.6.1 Firmware 1.6.1 addresses [issue 85](https://github.com/CCOMJHC/WIBL/issues/85) in the repository, which is a bug in the generation of "last known good" data in the JSON response to the "status" command, which shows up as failure to parse the JSON in the JavaScript. A consequence of this is that the hardware data simulator required updates to (a) add a Depth datagram in the NMEA2000 output (fixed depth rather than fully simulated like the NMEA0183 output), and (b) flushing of buffers to ensure that NMEA0183 messages are sent correctly on the GGA/ZDA channel without problems. diff --git a/LoggerFirmware/include/Configuration.h b/LoggerFirmware/include/Configuration.h index 3d11cdab..6c722b8f 100644 --- a/LoggerFirmware/include/Configuration.h +++ b/LoggerFirmware/include/Configuration.h @@ -38,7 +38,7 @@ namespace logger { // Firmware software version (i.e., overall firmware, rather than components like Command Processor, etc.) const int firmware_major = 1; const int firmware_minor = 6; -const int firmware_patch = 1; +const int firmware_patch = 2; /// @brief Stringify the version information for the firmware itself String FirmwareVersion(void); diff --git a/UploadServer/README.md b/UploadServer/README.md index 491b25b6..704371cb 100644 --- a/UploadServer/README.md +++ b/UploadServer/README.md @@ -672,7 +672,7 @@ TNNAME-12CEC8B4-0C42-424C-82CD-FB4E96CD7153|JDJhJDEwJDc1Q0FrVG9WdEo2YUwwWWwxLjN0 Next, you can do a basic test of the upload-server by using the `/checkin` endpoint using `curl`: ```shell -$ curl -v --http1.1 \ +$ curl -v --http1.1 --insecure \ -u TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A:CC0E1FE1-46CA-4768-93A7-2252BF748118 \ --cacert ./certs/ca.crt --fail-with-body "https://localhost:8000/checkin" \ -H 'Content-Type: application/json' \ @@ -750,7 +750,9 @@ EOF ``` You can see from the above output that the request was successful because -the HTTP status code was 200, i.e., `HTTP/2 200`. +the HTTP status code was 200, i.e., `HTTP/2 200`. Note that the 'insecure' +option tells `curl` not to attempt to chase the CA chain, which won't work +anyway with a self-signed certificate. In the console in which you are running `docker compose up`, you should also see the following output: diff --git a/UploadServer/src/api/api.go b/UploadServer/src/api/api.go index 0954d905..5989a84b 100644 --- a/UploadServer/src/api/api.go +++ b/UploadServer/src/api/api.go @@ -58,16 +58,9 @@ type DataSummary struct { Nmea2000 DataInfo `json:"nmea2000"` } -type FileEntry struct { - Id uint `json:"id"` - Len uint32 `json:"len"` - MD5 string `json:"md5"` - Url string `json:"url"` -} - type FileInfo struct { - Count uint `json:"count"` - Detail []FileEntry `json:"detail"` + Count int `json:"totalFileCount"` + Size int `json:"totalFileSize"` } type Status struct { diff --git a/UploadServer/wibl-monitor.go b/UploadServer/wibl-monitor.go index 16ed356d..e99d281f 100644 --- a/UploadServer/wibl-monitor.go +++ b/UploadServer/wibl-monitor.go @@ -171,8 +171,8 @@ func status_updates(w http.ResponseWriter, r *http.Request) { return } - support.Infof("CHECKIN: status update from logger with firmware %s, command processor %s, total %d files.\n", - status.Versions.Firmware, status.Versions.CommandProcessor, status.Files.Count) + support.Infof("CHECKIN: status update from logger with firmware %s, command processor %s, total %d files of %d bytes.\n", + status.Versions.Firmware, status.Versions.CommandProcessor, status.Files.Count, status.Files.Size) support.LogAccess(r, http.StatusOK) } From ec8140f28a314fb0c8a1f9d232918fa54864cd26 Mon Sep 17 00:00:00 2001 From: brian-r-calder Date: Wed, 27 May 2026 14:46:42 -0400 Subject: [PATCH 15/16] Updated README for upload server, and pinned localstack (for local test) to v.4.8.1 so that we can run without having to use a license. --- UploadServer/README.md | 38 ++++++++++++++------------------ UploadServer/docker-compose.yaml | 2 +- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/UploadServer/README.md b/UploadServer/README.md index 704371cb..35878f7b 100644 --- a/UploadServer/README.md +++ b/UploadServer/README.md @@ -602,11 +602,22 @@ go 1.24 ``` ### Building and running with `docker compose` -Before running, you'll need to first create a self-signed TLS -certificate using the provided script [cert-gen.sh](scripts/cert-gen.sh). +Before running, you'll need to first create a self-signed TLS certificate using the provided script +[cert-gen.sh](scripts/cert-gen.sh), which by default will generate the certificates in `./certs` (i.e., in the +repository source directory), a requirement for `docker compose` to function correctly. -This should store the certs in the local directory called `certs` -(which will be created if it does not exist). +You will also have to build the code to add a logger to the database using the [script +available](./build-add-logger.bash), which will compile the code [from the Go +source](./src/tools/add-logger/insert-logger.go). Then, create a dummy logger with: +```shell +mkdir ./db +add-logger -config ./config-local.json \ + -logger TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A \ + -password CC0E1FE1-46CA-4768-93A7-2252BF748118 +``` +where `TNAME` is your DCDB identifier (e.g., `UNHJHC`). If you do not specify a password, one will be automatically +created; the database file will also be created if it does not already exist (which also needs to be in the root of the +upload server source directory, i.e., `./db/loggers.db`, for the `docker compose` to work). Now, build and start the server in a container using: ```shell @@ -650,19 +661,6 @@ wibl-upload | 2025/10/02 18:11:02.497117 INFO [::1] - - [02/Oct/2025:18:11:02 + wibl-upload | 2025/10/02 18:11:12.556534 INFO [::1] - - [02/Oct/2025:18:11:12 +0000] "GET / HTTP/2.0" 200 0 ``` -Before trying to interact with the service, you'll need to create a `loggers.db` -file in the `db` local directory. Before you can do that, you'll need to build -the `add-logger` command using the provided [script](build-add-logger.bash). - - -```shell -mkdir -p db -./add-logger -config config-local.json -logger TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A -password CC0E1FE1-46CA-4768-93A7-2252BF748118 -./add-logger -config config-local.json -logger TNNAME-12CEC8B4-0C42-424C-82CD-FB4E96CD7153 -password CAF1CA92-CB9E-437D-B391-7709A39D32B1 -``` - -> Where "TNNAME" is your trusted node identifier. - You can then verify that the loggers have been added by running: ```shell $ sqlite3 ./db/loggers.db 'SELECT * FROM loggers' @@ -672,7 +670,7 @@ TNNAME-12CEC8B4-0C42-424C-82CD-FB4E96CD7153|JDJhJDEwJDc1Q0FrVG9WdEo2YUwwWWwxLjN0 Next, you can do a basic test of the upload-server by using the `/checkin` endpoint using `curl`: ```shell -$ curl -v --http1.1 --insecure \ +$ curl -v --http1.1 \ -u TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A:CC0E1FE1-46CA-4768-93A7-2252BF748118 \ --cacert ./certs/ca.crt --fail-with-body "https://localhost:8000/checkin" \ -H 'Content-Type: application/json' \ @@ -750,9 +748,7 @@ EOF ``` You can see from the above output that the request was successful because -the HTTP status code was 200, i.e., `HTTP/2 200`. Note that the 'insecure' -option tells `curl` not to attempt to chase the CA chain, which won't work -anyway with a self-signed certificate. +the HTTP status code was 200, i.e., `HTTP/2 200`. In the console in which you are running `docker compose up`, you should also see the following output: diff --git a/UploadServer/docker-compose.yaml b/UploadServer/docker-compose.yaml index c71c0a5b..ce7b5a2f 100644 --- a/UploadServer/docker-compose.yaml +++ b/UploadServer/docker-compose.yaml @@ -2,7 +2,7 @@ name: wibl-upload services: localstack: - image: localstack/localstack + image: localstack/localstack:4.8.1 environment: - AWS_DEFAULT_REGION=us-east-2 - AWS_ACCESS_KEY_ID=test From b4ec6806c07aec7efcb915b66ccdb2ccb7d1addb Mon Sep 17 00:00:00 2001 From: brian-r-calder Date: Wed, 27 May 2026 15:07:55 -0400 Subject: [PATCH 16/16] fw: Small updates to resolve questions on PR #117. --- LoggerFirmware/include/JSONUtilities.h | 4 ++-- LoggerFirmware/include/LogManager.h | 4 ++-- LoggerFirmware/src/LogManager.cpp | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/LoggerFirmware/include/JSONUtilities.h b/LoggerFirmware/include/JSONUtilities.h index 2dcb7e99..7b5884ba 100644 --- a/LoggerFirmware/include/JSONUtilities.h +++ b/LoggerFirmware/include/JSONUtilities.h @@ -5,7 +5,7 @@ #include -inline bool GrowJsonDocumentBy(DynamicJsonDocument & json, size_t capacityIncrease) +inline bool GrowJsonDocumentBy(DynamicJsonDocument& json, size_t capacityIncrease) { auto newCapacity = json.capacity() + capacityIncrease; DynamicJsonDocument temp(newCapacity); @@ -18,7 +18,7 @@ inline bool GrowJsonDocumentBy(DynamicJsonDocument & json, size_t capacityIncrea return true; } -inline bool GrowJsonDocument(DynamicJsonDocument & json) +inline bool GrowJsonDocument(DynamicJsonDocument& json) { // Grow capacity by 50%. A growth rate less than the golden ratio allows // the document to potentially reuse allocations it previously occupied. diff --git a/LoggerFirmware/include/LogManager.h b/LoggerFirmware/include/LogManager.h index 2ef71b0a..599e6a66 100644 --- a/LoggerFirmware/include/LogManager.h +++ b/LoggerFirmware/include/LogManager.h @@ -161,7 +161,7 @@ class Manager { bool Update(uint32_t filenum, MD5Hash *hash = nullptr); void RemoveLogFile(uint32_t filenum); uint32_t CountLogFiles(uint32_t filenumbers[MaxLogFiles]); - uint32_t CountLogFiles(uint64_t * fileSize); + uint32_t CountLogFiles(uint64_t *totalFileSizes); uint32_t GetNextLogNumber(void); uint32_t Filesize(uint32_t filenum); uint16_t UploadCount(uint32_t filenum); @@ -193,7 +193,7 @@ class Manager { /// \brief Extract a log number from a filename (if valid) int32_t ExtractLogNumber(String const& filename); /// \brief Count the number of log files on the system - uint32_t scanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize); + uint32_t ScanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t *totalSize); /// \brief Extract information on a single log file void enumerate(uint32_t lognumber, String& filename, uint32_t& filesize); /// \brief Generate a hash for a given file diff --git a/LoggerFirmware/src/LogManager.cpp b/LoggerFirmware/src/LogManager.cpp index e01211a5..46e4b001 100644 --- a/LoggerFirmware/src/LogManager.cpp +++ b/LoggerFirmware/src/LogManager.cpp @@ -99,7 +99,7 @@ Manager::Inventory::~Inventory(void) bool Manager::Inventory::Reinitialise(void) { uint32_t *filenumbers = new uint32_t[MaxLogFiles]; - auto filecount = m_logManager->scanLogFolder(filenumbers, nullptr); + auto filecount = m_logManager->ScanLogFolder(filenumbers, nullptr); Manager::MD5Hash emptyhash; if (m_verbose) @@ -165,7 +165,7 @@ uint32_t Manager::Inventory::CountLogFiles(uint32_t filenumbers[MaxLogFiles]) return filecount; } -uint32_t Manager::Inventory::CountLogFiles(uint64_t * totalSizePtr) +uint32_t Manager::Inventory::CountLogFiles(uint64_t *totalFileSizes) { uint64_t totalSize = 0; uint32_t filecount = 0; @@ -173,7 +173,7 @@ uint32_t Manager::Inventory::CountLogFiles(uint64_t * totalSizePtr) totalSize += fileSize; filecount += (fileSize != 0); } - if (totalSizePtr) *totalSizePtr = totalSize; + if (totalFileSizes) *totalFileSizes = totalSize; return filecount; } @@ -530,19 +530,19 @@ void Manager::RemoveAllLogfiles(void) StartNewLog(); // We need to have something running for the logging effort! } -uint32_t Manager::CountLogFiles(uint64_t * fileSize) +uint32_t Manager::CountLogFiles(uint64_t *fileSize) { return m_inventory ? m_inventory->CountLogFiles(fileSize) - : scanLogFolder(nullptr, fileSize); + : ScanLogFolder(nullptr, fileSize); } std::vector Manager::GetLogFileNumbers() { // Temporarily allocate an array of max size... - auto * temp = new uint32_t[logger::MaxLogFiles]; + uint32_t *temp = new uint32_t[logger::MaxLogFiles]; - auto fileCount = m_inventory ? m_inventory->CountLogFiles(temp) - : scanLogFolder(temp, nullptr); + uint32_t fileCount = m_inventory ? m_inventory->CountLogFiles(temp) + : ScanLogFolder(temp, nullptr); auto fileNumbers = std::vector(temp, temp + fileCount); delete [] temp; @@ -811,7 +811,7 @@ void Manager::TransferLogFile(uint32_t file_num, MD5Hash const& filehash, Stream Serial.printf("Sent %u B in %lu s.\n", bytes_transferred, duration); } -uint32_t Manager::scanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize) +uint32_t Manager::ScanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize) { uint32_t file_count = 0; File logdir = m_storage->Controller().open("/logs");