diff --git a/LoggerFirmware/ReleaseNotes.md b/LoggerFirmware/ReleaseNotes.md index 6b66b42d..83cc3949 100644 --- a/LoggerFirmware/ReleaseNotes.md +++ b/LoggerFirmware/ReleaseNotes.md @@ -1,5 +1,13 @@ # Release Notes: Logger Firmware +## Firmware 1.6.2 + +Firmware 1.6.2 addresses [issue 107](https://github.com/CCOMJHC/WIBL/issues/107) in the repository, where the size of the JSON documents being constructed in the firmware were getting to the size where they took up enough memory that they caused problems when generating SSL connections during upload. This also seemed to have significantly slowed down the webserver, so this version of the firmware should show much snappier connections on the server as a (very welcome) side effect. + +In addition to managing the memory better in the firmware, the modifications here change the status message so that it no longer provides all of the file information (names, sizes, checksums, etc.) The code adds a "catalog" command that now provides this information; the "snapshot catalog" command can also be used to generate this information as a file that can be downloaded using the standard browser mechanism. + +The [original PR](https://github.com/CCOMJHC/WIBL/pull/108) for this (which is used here with slight adjustments for the upload server end of the connection) was contributed by the awesome team at [Spatialnetics](https://spatialnetics.com). + ## Firmware 1.6.1 Firmware 1.6.1 addresses [issue 85](https://github.com/CCOMJHC/WIBL/issues/85) in the repository, which is a bug in the generation of "last known good" data in the JSON response to the "status" command, which shows up as failure to parse the JSON in the JavaScript. A consequence of this is that the hardware data simulator required updates to (a) add a Depth datagram in the NMEA2000 output (fixed depth rather than fully simulated like the NMEA0183 output), and (b) flushing of buffers to ensure that NMEA0183 messages are sent correctly on the GGA/ZDA channel without problems. diff --git a/LoggerFirmware/include/Configuration.h b/LoggerFirmware/include/Configuration.h index 3d11cdab..6c722b8f 100644 --- a/LoggerFirmware/include/Configuration.h +++ b/LoggerFirmware/include/Configuration.h @@ -38,7 +38,7 @@ namespace logger { // Firmware software version (i.e., overall firmware, rather than components like Command Processor, etc.) const int firmware_major = 1; const int firmware_minor = 6; -const int firmware_patch = 1; +const int firmware_patch = 2; /// @brief Stringify the version information for the firmware itself String FirmwareVersion(void); diff --git a/LoggerFirmware/include/JSONUtilities.h b/LoggerFirmware/include/JSONUtilities.h new file mode 100644 index 00000000..7b5884ba --- /dev/null +++ b/LoggerFirmware/include/JSONUtilities.h @@ -0,0 +1,28 @@ + + +#ifndef JSONUTILITIES_H +#define JSONUTILITIES_H + +#include + +inline bool GrowJsonDocumentBy(DynamicJsonDocument& json, size_t capacityIncrease) +{ + auto newCapacity = json.capacity() + capacityIncrease; + DynamicJsonDocument temp(newCapacity); + + // Allocation failed. Nothing we can do. + if (temp.capacity() == 0) return false; + + temp.set(json); + json = std::move(temp); + return true; +} + +inline bool GrowJsonDocument(DynamicJsonDocument& json) +{ + // Grow capacity by 50%. A growth rate less than the golden ratio allows + // the document to potentially reuse allocations it previously occupied. + return GrowJsonDocumentBy(json, json.capacity() / 2); +} + +#endif // JSONUTILITIES_H diff --git a/LoggerFirmware/include/LogManager.h b/LoggerFirmware/include/LogManager.h index b0df99fd..599e6a66 100644 --- a/LoggerFirmware/include/LogManager.h +++ b/LoggerFirmware/include/LogManager.h @@ -74,9 +74,10 @@ class Manager { void RemoveAllLogfiles(void); /// \brief Count the number of log files on the system - uint32_t CountLogFiles(uint32_t filenumbers[MaxLogFiles]); - /// \brief Count the number of log files on the system - uint32_t CountLogFiles(void); + uint32_t CountLogFiles(uint64_t * fileSize = nullptr); + + /// \brief Collect the file numbers of all log files on the system + std::vector GetLogFileNumbers(); class MD5Hash { public: @@ -160,7 +161,7 @@ class Manager { bool Update(uint32_t filenum, MD5Hash *hash = nullptr); void RemoveLogFile(uint32_t filenum); uint32_t CountLogFiles(uint32_t filenumbers[MaxLogFiles]); - uint32_t CountLogFiles(void); + uint32_t CountLogFiles(uint64_t *totalFileSizes); uint32_t GetNextLogNumber(void); uint32_t Filesize(uint32_t filenum); uint16_t UploadCount(uint32_t filenum); @@ -192,7 +193,7 @@ class Manager { /// \brief Extract a log number from a filename (if valid) int32_t ExtractLogNumber(String const& filename); /// \brief Count the number of log files on the system - uint32_t count(uint32_t *filenumbers); + uint32_t ScanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t *totalSize); /// \brief Extract information on a single log file void enumerate(uint32_t lognumber, String& filename, uint32_t& filesize); /// \brief Generate a hash for a given file diff --git a/LoggerFirmware/include/SerialCommand.h b/LoggerFirmware/include/SerialCommand.h index b8b41f8d..3099e56a 100644 --- a/LoggerFirmware/include/SerialCommand.h +++ b/LoggerFirmware/include/SerialCommand.h @@ -166,6 +166,8 @@ class SerialCommand { void ReportScalesElement(CommandSource src); /// \brief Report the number of log files available on the SD card void ReportFileCount(CommandSource src); + /// \brief Report the properties of all log files on the SD card + void ReportCatalog(CommandSource src); /// \brief Report the configuration of the web-server void ReportWebserverConfig(CommandSource src); /// \brief Configure the web-server for system config/management @@ -193,10 +195,10 @@ class SerialCommand { /// \brief Generate a string on the appropriate output stream void EmitMessage(String const& msg, CommandSource src); - /// \brief Convert a stringified JSON into a document, with error reporting + /// \brief Send stringified JSON, with error reporting bool EmitJSON(String const& source, CommandSource src); - /// @brief Generate a list of files into a JSON document - DynamicJsonDocument GenerateFilelist(void); + /// \brief Send a JSON document, with error reporting + void EmitJSON(DynamicJsonDocument && json, CommandSource src); /// @brief Display a NMEA0183 filter ID list void DisplayNMEAFilter(logger::N0183IDStore& filter, CommandSource src); /// @brief Display an Algorithm Store list diff --git a/LoggerFirmware/include/WiFiAdapter.h b/LoggerFirmware/include/WiFiAdapter.h index cfa84042..f9d4c7c5 100644 --- a/LoggerFirmware/include/WiFiAdapter.h +++ b/LoggerFirmware/include/WiFiAdapter.h @@ -57,7 +57,8 @@ class WiFiAdapter { void AddMessage(String const& message); /// \brief Replace the entire message to be returned to the client for the current transaction - void SetMessage(DynamicJsonDocument const& message); + void SetMessage(JsonDocument const& message) { setMessage(message); } + void SetMessage(DynamicJsonDocument && message) { setMessage(std::move(message)); } enum HTTPReturnCodes { OK = 200, // The request succeeded @@ -103,7 +104,8 @@ class WiFiAdapter { virtual void accumulateMessage(String const& message) = 0; /// \brief Sub-class implementation of code to replace message for transmission - virtual void setMessage(DynamicJsonDocument const& message) = 0; + virtual void setMessage(JsonDocument const& message) = 0; + virtual void setMessage(DynamicJsonDocument && message) = 0; /// \brief Sub-class implementation of code to set the status code for the transaction virtual void setStatusCode(HTTPReturnCodes status_code) = 0; diff --git a/LoggerFirmware/src/AutoUpload.cpp b/LoggerFirmware/src/AutoUpload.cpp index 77dd02a5..2e5bcccf 100644 --- a/LoggerFirmware/src/AutoUpload.cpp +++ b/LoggerFirmware/src/AutoUpload.cpp @@ -81,16 +81,14 @@ void UploadManager::UploadCycle(void) m_lastUploadCycle); return; } - DynamicJsonDocument files(logger::status::GenerateFilelist(m_logManager)); - int filecount = files["files"]["count"].as(); - for (int n = 0; n < filecount; ++n) { - uint32_t file_id = files["files"]["detail"][n]["id"].as(); - if (TransferFile(m_logManager->FileSystem(), file_id)) { + + for (auto fileNumber : m_logManager->GetLogFileNumbers()) { + bool success = TransferFile(m_logManager->FileSystem(), fileNumber); + if (success) { // File transferred to the server successfully, so we can delete locally - m_logManager->RemoveLogFile(file_id); + m_logManager->RemoveLogFile(fileNumber); } else { // File did not transfer, so we update the upload attempt metadata and move on - } unsigned long current_elapsed = millis(); if ((current_elapsed - start_time) > m_uploadDuration) { diff --git a/LoggerFirmware/src/LogManager.cpp b/LoggerFirmware/src/LogManager.cpp index afc451e4..46e4b001 100644 --- a/LoggerFirmware/src/LogManager.cpp +++ b/LoggerFirmware/src/LogManager.cpp @@ -99,8 +99,7 @@ Manager::Inventory::~Inventory(void) bool Manager::Inventory::Reinitialise(void) { uint32_t *filenumbers = new uint32_t[MaxLogFiles]; - uint32_t filecount = m_logManager->count(filenumbers); - String filename; + auto filecount = m_logManager->ScanLogFolder(filenumbers, nullptr); Manager::MD5Hash emptyhash; if (m_verbose) @@ -166,12 +165,15 @@ uint32_t Manager::Inventory::CountLogFiles(uint32_t filenumbers[MaxLogFiles]) return filecount; } -uint32_t Manager::Inventory::CountLogFiles(void) +uint32_t Manager::Inventory::CountLogFiles(uint64_t *totalFileSizes) { + uint64_t totalSize = 0; uint32_t filecount = 0; - for (uint32_t entry = 0; entry < MaxLogFiles; ++entry) { - if (m_filesize[entry] != 0) ++filecount; + for (auto fileSize : m_filesize) { + totalSize += fileSize; + filecount += (fileSize != 0); } + if (totalFileSizes) *totalFileSizes = totalSize; return filecount; } @@ -297,15 +299,14 @@ void Manager::RemoveAllLogfiles(void) /// Normally, this finds all of the log files on the SD card, counts the total number, and returns /// the internal reference numbers that are being used so that the user can request more details /// about a specific file later. In this dummy version, it simply prints a debug message, and returns -/// zero. +/// an empty vector. /// -/// \param filenumbers Fixed size array (of \a MaxLogFiles) written with the internal reference numbers for the files -/// \return Total number of files available on the SD card +/// \return An empty vector -int Manager::CountLogFiles(int filenumbers[MaxLogFiles]) +std::vector Manager::GetLogFileNumbers() { - Serial.println("DBG: Call to count log files; returning zero."); - return 0; + Serial.println("DBG: Call to get log files numbers; returning empty vector."); + return {}; } /// \brief Determine the size and file name of the specified file @@ -506,54 +507,46 @@ bool Manager::RemoveLogFile(uint32_t file_num) void Manager::RemoveAllLogfiles(void) { - uint32_t *filenumbers = new uint32_t[MaxLogFiles]; + auto fileNumbers = GetLogFileNumbers(); CloseLogfile(); // All means all ... - uint32_t filecount = CountLogFiles(filenumbers); uint32_t files_closed = 0; - for (uint32_t f = 0; f < filecount; ++f) { - String filename = MakeLogName(filenumbers[f]); + for (auto fileNumber : fileNumbers) { + String filename = MakeLogName(fileNumber); Serial.printf("INFO: erasing log file: \"%s\".\n", filename.c_str()); bool rc = m_storage->Controller().remove(filename); if (rc) { m_consoleLog.printf("INFO: erased log file \"%s\" by user command.\n", filename.c_str()); ++files_closed; - if (m_inventory != nullptr) m_inventory->RemoveLogFile(filenumbers[f]); + if (m_inventory != nullptr) m_inventory->RemoveLogFile(fileNumber); } else { m_consoleLog.printf("ERR: failed to erase log file \"%s\" by user command.\n", filename.c_str()); } } - delete[] filenumbers; - m_consoleLog.printf("INFO: erased %u log files of %u.\n", files_closed, filecount); + + m_consoleLog.printf("INFO: erased %u log files of %u.\n", files_closed, fileNumbers.size()); m_consoleLog.flush(); StartNewLog(); // We need to have something running for the logging effort! } -/// Count the number of log files on the SD card, so that the client can enumerate them -/// and report to the user. -/// -/// \param filenumbers (Out) Array of the log file numbers on card -/// \return Number of files on the SD card - -uint32_t Manager::CountLogFiles(uint32_t filenumbers[MaxLogFiles]) +uint32_t Manager::CountLogFiles(uint64_t *fileSize) { - uint32_t filecount; - if (m_inventory != nullptr) { - filecount = m_inventory->CountLogFiles(filenumbers); - } else - filecount = count(filenumbers); - return filecount; + return m_inventory ? m_inventory->CountLogFiles(fileSize) + : ScanLogFolder(nullptr, fileSize); } -uint32_t Manager::CountLogFiles(void) +std::vector Manager::GetLogFileNumbers() { - uint32_t filecount; - if (m_inventory != nullptr) { - filecount = m_inventory->CountLogFiles(); - } else - filecount = count(nullptr); - return filecount; + // Temporarily allocate an array of max size... + uint32_t *temp = new uint32_t[logger::MaxLogFiles]; + + uint32_t fileCount = m_inventory ? m_inventory->CountLogFiles(temp) + : ScanLogFolder(temp, nullptr); + auto fileNumbers = std::vector(temp, temp + fileCount); + + delete [] temp; + return fileNumbers; } /// Make a list of all of the files that exist on the SD card in the log directory, along with their @@ -818,23 +811,22 @@ void Manager::TransferLogFile(uint32_t file_num, MD5Hash const& filehash, Stream Serial.printf("Sent %u B in %lu s.\n", bytes_transferred, duration); } -uint32_t Manager::count(uint32_t *filenumbers) +uint32_t Manager::ScanLogFolder(uint32_t fileNumbers[MaxLogFiles], uint64_t * totalSize) { uint32_t file_count = 0; File logdir = m_storage->Controller().open("/logs"); - File entry = logdir.openNextFile(); - - while (entry) { + while (File entry = logdir.openNextFile()) { // Because we can store snapshots of configuration information in the /logs directory // (so that they can be seen through the webserver's static website for download), we // need to count only the valid log files in the directory. int32_t lognumber = ExtractLogNumber(String(entry.name())); - if (lognumber >= 0) { - if (filenumbers != nullptr) filenumbers[file_count] = lognumber; + auto size = entry.size(); + if (lognumber >= 0 && size > 0) { + if (fileNumbers != nullptr) fileNumbers[file_count] = lognumber; + if (totalSize) *totalSize += size; ++file_count; } entry.close(); - entry = logdir.openNextFile(); } logdir.close(); return file_count; diff --git a/LoggerFirmware/src/NVMFile.cpp b/LoggerFirmware/src/NVMFile.cpp index eb2aff47..891338aa 100644 --- a/LoggerFirmware/src/NVMFile.cpp +++ b/LoggerFirmware/src/NVMFile.cpp @@ -26,6 +26,7 @@ #include #include "NVMFile.h" +#include "JSONUtilities.h" #include "LittleFS.h" #include "LogManager.h" #include "serialisation.h" @@ -348,10 +349,7 @@ void AlgoRequestStore::AddAlgorithm(String const& alg_name, String const& alg_pa entry["parameters"] = alg_params; if ((doc.memoryUsage() + entry.capacity()) > 0.95*doc.capacity()) { // The document needs to be bigger to handle the new entry - int capacity = doc.memoryUsage() + entry.capacity() + 256; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; + GrowJsonDocumentBy(doc, entry.capacity()); } doc["algorithm"].add(entry.as()); doc["count"] = count + 1; diff --git a/LoggerFirmware/src/SerialCommand.cpp b/LoggerFirmware/src/SerialCommand.cpp index c4e3c29b..ec3894d2 100644 --- a/LoggerFirmware/src/SerialCommand.cpp +++ b/LoggerFirmware/src/SerialCommand.cpp @@ -780,15 +780,7 @@ void SerialCommand::ConfigurePassthrough(String const& params, CommandSource src void SerialCommand::ReportConfigurationJSON(CommandSource src, bool secure) { - DynamicJsonDocument json(logger::ConfigJSON::ExtractConfig()); - if (src == CommandSource::SerialPort) { - String s; - serializeJsonPretty(json, s); - EmitMessage(s + "\n", src); - } else { - if (m_wifi != nullptr) - m_wifi->SetMessage(json); - } + EmitJSON(logger::ConfigJSON::ExtractConfig(), src); } /// Provide summary report of all of the configuration parameters being managed by the Confgiuration @@ -922,8 +914,7 @@ void SerialCommand::DisplayAlgorithmStore(logger::AlgoRequestStore& store, Comma String algorithms(store.JSONRepresentation(true)); EmitMessage(algorithms + '\n', src); } else if (src == CommandSource::WirelessPort) { - DynamicJsonDocument doc(store.GetContents()); - m_wifi->SetMessage(doc); + m_wifi->SetMessage(store.GetContents()); } else { EmitMessage("ERR: request for unknown CommandSource - who are you?\n", src); } @@ -1022,8 +1013,7 @@ void SerialCommand::DisplayNMEAFilter(logger::N0183IDStore& filter, CommandSourc String filter_ids(filter.JSONRepresentation(true)); EmitMessage(filter_ids + '\n', src); } else if (src == CommandSource::WirelessPort) { - DynamicJsonDocument doc(filter.GetContents()); - m_wifi->SetMessage(doc); + m_wifi->SetMessage(filter.GetContents()); } else { EmitMessage("ERR: request for unknown CommandSource - who are you?\n", src); } @@ -1099,6 +1089,11 @@ void SerialCommand::ReportFileCount(CommandSource src) EmitMessage(String(file_count) + "\n", src); } +void SerialCommand::ReportCatalog(CommandSource src) +{ + EmitJSON(logger::status::GenerateFilelist(m_logManager), src); +} + void SerialCommand::ReportWebserverConfig(CommandSource src) { if (src == CommandSource::SerialPort) { @@ -1166,17 +1161,7 @@ void SerialCommand::ConfigureWebserver(String const& params, CommandSource src) void SerialCommand::ReportCurrentStatus(CommandSource src) { - DynamicJsonDocument status(logger::status::CurrentStatus(m_logManager)); - - if (src == CommandSource::SerialPort) { - String json; - serializeJsonPretty(status, json); - EmitMessage(json+"\n", src); - } else { - if (m_wifi != nullptr) { - m_wifi->SetMessage(status); - } - } + EmitJSON(logger::status::CurrentStatus(m_logManager), src); } /// Report the current set of "lab default" configuration parameters set on the logger. These @@ -1384,14 +1369,12 @@ void SerialCommand::SnapshotResource(String const& resource, CommandSource src) EmitMessage("ERR: unknown snapshot resource requested.\n", src); } - StaticJsonDocument<256> responseJson; + DynamicJsonDocument responseJson(256); if (rc) responseJson["url"] = url; else responseJson["url"] = ""; - String response; - serializeJson(responseJson, response); - EmitJSON(response, src); + EmitJSON(std::move(responseJson), src); } void SerialCommand::ReportUploadConfig(CommandSource src) @@ -1560,6 +1543,8 @@ void SerialCommand::Execute(String const& cmd, CommandSource src) } else { SetAuthorisation(cmd.substring(5), src); } + } else if (cmd == "catalog") { + ReportCatalog(src); } else if (cmd.startsWith("configure")) { if (cmd.length() == 9) { ReportConfiguration(src); @@ -1782,19 +1767,26 @@ bool SerialCommand::EmitJSON(String const& source, CommandSource chan) EmitMessage("No data in JSON.\n", chan); return false; } - DynamicJsonDocument json(logger::status::GenerateJSON(source)); - switch (chan) { - case CommandSource::SerialPort: - serializeJsonPretty(json, Serial); - break; - case CommandSource::WirelessPort: - if (m_wifi != nullptr) { - m_wifi->SetMessage(json); - } - break; - default: - Serial.println("ERR: command source not recognised."); - break; - } + EmitJSON(logger::status::GenerateJSON(source), chan); return true; } + +/// Generate a message on the output stream associated with the given source, with +/// the message in JSON format. +/// +/// @param json JSON document +/// @param src Input channel on which the command arrived + +void SerialCommand::EmitJSON(DynamicJsonDocument && json, CommandSource src) +{ + if (src == CommandSource::SerialPort) { + serializeJsonPretty(json, Serial); + Serial.println(); + } else if (src == CommandSource::WirelessPort) { + if (m_wifi != nullptr) { + m_wifi->SetMessage(std::move(json)); + } + } else { + Serial.println("ERR: command source not recognised."); + } +} diff --git a/LoggerFirmware/src/Status.cpp b/LoggerFirmware/src/Status.cpp index 04daa051..92122c12 100644 --- a/LoggerFirmware/src/Status.cpp +++ b/LoggerFirmware/src/Status.cpp @@ -25,6 +25,7 @@ */ #include "ArduinoJson.h" +#include "JSONUtilities.h" #include "LogManager.h" #include "Configuration.h" #include "N0183Logger.h" @@ -44,64 +45,59 @@ namespace status { DynamicJsonDocument GenerateFilelist(logger::Manager *m) { - uint32_t *filenumbers = new uint32_t[logger::MaxLogFiles]; - uint32_t n_files = m->CountLogFiles(filenumbers); - DynamicJsonDocument doc(100*n_files + 256); // Approximate guess, but can expand + auto fileNumbers = m->GetLogFileNumbers(); + + // Start small. We really don't want this first allocation to fail. + DynamicJsonDocument doc(512); + + // Assume the list will be truncated, because we might not have enough memory + // to set it to true later when allocation fails. + doc["files"]["isTruncated"] = true; + for (int n = 0; n < fileNumbers.size(); ++n) { + auto fileNumber = fileNumbers[n]; - doc["files"]["count"] = n_files; - for (int n = 0; n < n_files; ++n) { String filename; uint32_t filesize; logger::Manager::MD5Hash filehash; uint16_t uploadCount; - m->EnumerateLogFile(filenumbers[n], filename, filesize, filehash, uploadCount); + m->EnumerateLogFile(fileNumber, filename, filesize, filehash, uploadCount); StaticJsonDocument<256> entry; - entry["id"] = filenumbers[n]; + entry["id"] = fileNumber; entry["len"] = filesize; if (!filehash.Empty()) entry["md5"] = filehash.Value(); entry["url"] = filename; entry["uploads"] = uploadCount; - int entry_size = entry.memoryUsage(); - if (doc.memoryUsage() + entry_size > 0.95*doc.capacity()) { - // It's likely that adding the entry will cause the document to run out of - // space. We therefore double the capacity (since it's expensive, and we - // don't want to have to do it more than once) before we attempt the addition. - int capacity = doc.capacity() * 2; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; - } - if (!doc["files"]["detail"].add(entry.as())) { - // Out of memory, even after adding more above (most likely), - // so make a new allocation. Note that the addition can partially - // work, so you can end up with a malformed entry in the array (and - // one more entry than you expect). We therefore remove this last + while (!doc["files"]["detail"].add(entry.as())) { + // Not enough free capacity to add the entry. Note that the addition + // can partially work, so you can end up with ain malformed entry at + // the end of the array. We therefore remove this last // element before continuing -- luckily, it should be at the index in // the array of the file number. doc["files"]["detail"].remove(n); - int capacity = doc.capacity() * 2; - DynamicJsonDocument new_doc(capacity); - new_doc.set(doc); - doc = new_doc; - doc["files"]["detail"].add(entry.as()); + + if(!GrowJsonDocument(doc)) + { + // Allocation failed. Not really anything we can do other than + // return what we have. + return doc; + } } } - delete[] filenumbers; + doc["files"]["isTruncated"] = false; return doc; } DynamicJsonDocument CurrentStatus(logger::Manager *m) { - DynamicJsonDocument filelist(GenerateFilelist(m)); DynamicJsonDocument lkg(logger::Metrics.LastKnownGood()); // The total capacity should be, approximately, the sum of the biggest components // (above), plus some limited information on versions, elapsed time, and boot // status. We're assuming here that 1024B is enough for the extras ... that // might not always be the case. - int capacity = filelist.capacity() + lkg.capacity() + 1024; + int capacity = lkg.capacity() + 1024; DynamicJsonDocument status(capacity); @@ -124,7 +120,12 @@ DynamicJsonDocument CurrentStatus(logger::Manager *m) status["webserver"]["boot"] = boot_status; status["data"] = lkg; - status["files"] = filelist["files"]; + + uint64_t totalFileSize = 0; + auto totalFileCount = m->CountLogFiles(&totalFileSize); + auto files = status.createNestedObject("files"); + files["totalFileCount"] = totalFileCount; + files["totalFileSize"] = totalFileSize; return status; } diff --git a/LoggerFirmware/src/WiFiAdapter.cpp b/LoggerFirmware/src/WiFiAdapter.cpp index 9bbc332d..831a58c5 100644 --- a/LoggerFirmware/src/WiFiAdapter.cpp +++ b/LoggerFirmware/src/WiFiAdapter.cpp @@ -35,6 +35,7 @@ #include #include +#include "JSONUtilities.h" #include "LogManager.h" #include "WiFiAdapter.h" #include "Configuration.h" @@ -381,14 +382,11 @@ class ESP32WiFiAdapter : public WiFiAdapter { /// Default constructor for the ESP32 adapter. This brings up the parameter store to use /// for WiFi parameters, but takes no other action until the user explicitly starts the AccessPoint. ESP32WiFiAdapter(void) - : m_storage(nullptr), m_server(nullptr), m_messages(nullptr), m_statusCode(HTTPReturnCodes::OK) + : m_storage(nullptr), m_server(nullptr), m_messages(defaultMessageBufferSize), m_statusCode(HTTPReturnCodes::OK) { if ((m_storage = mem::MemControllerFactory::Create()) == nullptr) { return; } - if ((m_messages = new DynamicJsonDocument(1024)) == nullptr) { - return; - } } /// Default destructor for the ESP32 adapter. This stops the Access Point, if it isn't already down, /// and then deletes the ParamStore manipulation object. @@ -396,14 +394,15 @@ class ESP32WiFiAdapter : public WiFiAdapter { { stop(); delete m_storage; // Note that we're not stopping the interface, since it may still be required elsewhere - delete m_messages; } private: + static constexpr size_t defaultMessageBufferSize = 1024; + mem::MemController *m_storage; ///< Pointer to the storage object to use ExtendedWebServer *m_server; ///< Pointer to the server object, if started. std::queue m_commands; ///< Queue to handle commands sent by the user - DynamicJsonDocument *m_messages; ///< Accumulating message content to be send to the client + DynamicJsonDocument m_messages; ///< Accumulating message content to be send to the client HTTPReturnCodes m_statusCode; ///< Status code to return to the user with the transaction response ConnectionStateMachine m_state; ///< Manager for connection state @@ -533,19 +532,14 @@ class ESP32WiFiAdapter : public WiFiAdapter { void accumulateMessage(String const& message) { - if ((m_messages->memoryUsage() + message.length()) > 0.95*m_messages->capacity()) { - // Expand capacity to ensure that the message will be added successfully - size_t new_capacity = m_messages->capacity() * 2; - DynamicJsonDocument *new_doc = new DynamicJsonDocument(new_capacity); - if (new_doc != nullptr) { - new_doc->set(*m_messages); - delete m_messages; - m_messages = new_doc; - } + bool needToGrowBuffer = (m_messages.memoryUsage() + message.length()) > 0.95 * m_messages.capacity(); + if (needToGrowBuffer && !GrowJsonDocument(m_messages)) { + Serial.printf("ERR: failed to grow buffer beyond %d bytes; messages will be lost.\n", m_messages.capacity()); + return; } - if (!(*m_messages)["messages"].add(message)) { + if (!m_messages["messages"].add(message)) { Serial.printf("ERR: failed to add message to accumulation buffer (capacity %d bytes); messages may be truncated.\n", - m_messages->capacity()); + m_messages.capacity()); } } @@ -553,10 +547,14 @@ class ESP32WiFiAdapter : public WiFiAdapter { /// /// \param message \a String to use for the return message. /// \return N/A - - void setMessage(DynamicJsonDocument const& message) + void setMessage(JsonDocument const& message) { - *m_messages = message; + m_messages = message; + } + void setMessage(DynamicJsonDocument && message) + { + m_messages = std::move(message); + } /// Configure the HTTP status code that the response should use. By default, the status code used @@ -582,10 +580,18 @@ class ESP32WiFiAdapter : public WiFiAdapter { bool transmitMessages(void) { String message; - serializeJson(*m_messages, message); + serializeJson(m_messages, message); //Serial.printf("DBG: WiFi transmitting response |%s|\n", message.c_str()); m_server->send(m_statusCode, "application/json", message); - m_messages->clear(); + + // If the JSON document got too large, replace it with a smaller one to avoid needlessly + // holding on to too much memory. Otherwise, just clear the document and reuse it. + if (defaultMessageBufferSize < m_messages.capacity()) { + m_messages = DynamicJsonDocument(defaultMessageBufferSize); + } else { + m_messages.clear(); + } + m_statusCode = HTTPReturnCodes::OK; // "OK" by default return true; } @@ -633,12 +639,6 @@ bool WiFiAdapter::TransferFile(String const& filename, uint32_t filesize, logger /// \return N/A void WiFiAdapter::AddMessage(String const& message) { accumulateMessage(message); } -/// Pass-through implementation to the sub-class code to reset the message from straight JSON -/// -/// @param message JSON document to send back to the client -/// @return N/A -void WiFiAdapter::SetMessage(DynamicJsonDocument const& message) { setMessage(message); } - /// Pass-through implementation to the sub-class code to set status code /// /// \param status_code HTTP status code to set diff --git a/LoggerFirmware/website/js/status.js b/LoggerFirmware/website/js/status.js index a22b8092..ee4db883 100644 --- a/LoggerFirmware/website/js/status.js +++ b/LoggerFirmware/website/js/status.js @@ -174,22 +174,15 @@ function updateStatus(tablePrefix) { versions.appendChild(assembleSummaryRow("NMEA2000", data.version.nmea2000)); versions.appendChild(assembleSummaryRow("IMU", data.version.imu)); versions.appendChild(assembleSummaryRow("Serialiser", data.version.serialiser)); - - let totalFileSize = 0; - for (let n = 0; n < data.files.count; ++n) { - if ('len' in data.files.detail[n] && isFinite(data.files.detail[n].len)) { - totalFileSize += data.files.detail[n].len; - } - } - + let stats = document.getElementById(statsTable); stats.replaceChildren(assembleSummaryHeader("Status")); stats.appendChild(assembleSummaryRow("Elapsed Time", translateTime(data.elapsed))); stats.appendChild(assembleSummaryRow("Supply Voltage", roundVoltage(data.supply))); stats.appendChild(assembleSummaryRow("Webserver Status Current", data.webserver.current)); stats.appendChild(assembleSummaryRow("Webserver Status Boot", data.webserver.boot)); - stats.appendChild(assembleSummaryRow("Files on Logger", data.files.count)); - stats.appendChild(assembleSummaryRow("Total Size", translateSize(totalFileSize))); + stats.appendChild(assembleSummaryRow("Files on Logger", data.files.totalFileCount)); + stats.appendChild(assembleSummaryRow("Total Size", translateSize(data.files.totalFileSize))); let nmea0183 = document.getElementById(n0183Table); if (nmea0183 !== null) { @@ -220,26 +213,23 @@ function updateStatus(tablePrefix) { data.data.nmea2000.detail[n].display)); } } - + let detail = document.getElementById(detailTable); if (detail !== null) { - detail.replaceChildren(assembleDetailHeader()); - for (let n = 0; n < data.files.count; ++n) { - if (data.files.detail[n].hasOwnProperty('id') && - data.files.detail[n].hasOwnProperty('len') && - data.files.detail[n].hasOwnProperty('md5') && - data.files.detail[n].hasOwnProperty('url') && - data.files.detail[n].hasOwnProperty('uploads')) - detail.appendChild( - assembleDetailRow( - data.files.detail[n].id, - data.files.detail[n].len, - data.files.detail[n].md5, - data.files.detail[n].url, - data.files.detail[n].uploads - ) - ); - } + sendCommand('catalog').then((data) => { + detail.replaceChildren(assembleDetailHeader()); + for (const entry of data.files.detail) { + detail.appendChild(assembleDetailRow( + entry.id ?? "?", + entry.len ?? "?", + entry.md5 ?? "?", + entry.url ?? "?", + entry.uploads ?? "?")); + } + if (data.files.isTruncated) { + detail.appendChild(assembleDetailRow("...", "...", "...", "...", "...")); + } + }); } }); } diff --git a/UploadServer/README.md b/UploadServer/README.md index 491b25b6..35878f7b 100644 --- a/UploadServer/README.md +++ b/UploadServer/README.md @@ -602,11 +602,22 @@ go 1.24 ``` ### Building and running with `docker compose` -Before running, you'll need to first create a self-signed TLS -certificate using the provided script [cert-gen.sh](scripts/cert-gen.sh). +Before running, you'll need to first create a self-signed TLS certificate using the provided script +[cert-gen.sh](scripts/cert-gen.sh), which by default will generate the certificates in `./certs` (i.e., in the +repository source directory), a requirement for `docker compose` to function correctly. -This should store the certs in the local directory called `certs` -(which will be created if it does not exist). +You will also have to build the code to add a logger to the database using the [script +available](./build-add-logger.bash), which will compile the code [from the Go +source](./src/tools/add-logger/insert-logger.go). Then, create a dummy logger with: +```shell +mkdir ./db +add-logger -config ./config-local.json \ + -logger TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A \ + -password CC0E1FE1-46CA-4768-93A7-2252BF748118 +``` +where `TNAME` is your DCDB identifier (e.g., `UNHJHC`). If you do not specify a password, one will be automatically +created; the database file will also be created if it does not already exist (which also needs to be in the root of the +upload server source directory, i.e., `./db/loggers.db`, for the `docker compose` to work). Now, build and start the server in a container using: ```shell @@ -650,19 +661,6 @@ wibl-upload | 2025/10/02 18:11:02.497117 INFO [::1] - - [02/Oct/2025:18:11:02 + wibl-upload | 2025/10/02 18:11:12.556534 INFO [::1] - - [02/Oct/2025:18:11:12 +0000] "GET / HTTP/2.0" 200 0 ``` -Before trying to interact with the service, you'll need to create a `loggers.db` -file in the `db` local directory. Before you can do that, you'll need to build -the `add-logger` command using the provided [script](build-add-logger.bash). - - -```shell -mkdir -p db -./add-logger -config config-local.json -logger TNNAME-F94E871E-8A66-4614-9E10-628FFC49540A -password CC0E1FE1-46CA-4768-93A7-2252BF748118 -./add-logger -config config-local.json -logger TNNAME-12CEC8B4-0C42-424C-82CD-FB4E96CD7153 -password CAF1CA92-CB9E-437D-B391-7709A39D32B1 -``` - -> Where "TNNAME" is your trusted node identifier. - You can then verify that the loggers have been added by running: ```shell $ sqlite3 ./db/loggers.db 'SELECT * FROM loggers' diff --git a/UploadServer/docker-compose.yaml b/UploadServer/docker-compose.yaml index c71c0a5b..ce7b5a2f 100644 --- a/UploadServer/docker-compose.yaml +++ b/UploadServer/docker-compose.yaml @@ -2,7 +2,7 @@ name: wibl-upload services: localstack: - image: localstack/localstack + image: localstack/localstack:4.8.1 environment: - AWS_DEFAULT_REGION=us-east-2 - AWS_ACCESS_KEY_ID=test diff --git a/UploadServer/src/api/api.go b/UploadServer/src/api/api.go index 0954d905..5989a84b 100644 --- a/UploadServer/src/api/api.go +++ b/UploadServer/src/api/api.go @@ -58,16 +58,9 @@ type DataSummary struct { Nmea2000 DataInfo `json:"nmea2000"` } -type FileEntry struct { - Id uint `json:"id"` - Len uint32 `json:"len"` - MD5 string `json:"md5"` - Url string `json:"url"` -} - type FileInfo struct { - Count uint `json:"count"` - Detail []FileEntry `json:"detail"` + Count int `json:"totalFileCount"` + Size int `json:"totalFileSize"` } type Status struct { diff --git a/UploadServer/wibl-monitor.go b/UploadServer/wibl-monitor.go index 16ed356d..e99d281f 100644 --- a/UploadServer/wibl-monitor.go +++ b/UploadServer/wibl-monitor.go @@ -171,8 +171,8 @@ func status_updates(w http.ResponseWriter, r *http.Request) { return } - support.Infof("CHECKIN: status update from logger with firmware %s, command processor %s, total %d files.\n", - status.Versions.Firmware, status.Versions.CommandProcessor, status.Files.Count) + support.Infof("CHECKIN: status update from logger with firmware %s, command processor %s, total %d files of %d bytes.\n", + status.Versions.Firmware, status.Versions.CommandProcessor, status.Files.Count, status.Files.Size) support.LogAccess(r, http.StatusOK) }