Fix MQTT client issues with JWT renewal, memory allocation, and NTP handling

.gitignore

@@ -32,3 +32,7 @@ platformio.local.ini
 .build-wt-*/
 .wt-*/
 scripts/__pycache__/*
+
+# Reduced-TLS mbedTLS archives, ~6 MB per arch, fetched by scripts/fetch_mbedtls_4k.sh.
+# Not committed because they must be rebuilt for every espressif32 platform bump.
+.mbedtls-4k/

docs/mbedtls-tls-footprint.md

@@ -136,6 +136,36 @@ fix was `rm -rf` the package and `pio pkg install` to re-download stock. A prepe
 search path keeps the change scoped to one env, because the linker takes each archive member
 from the first archive that satisfies an undefined symbol.
 
+### How the archives are distributed
+
+They are not committed: ~6 MB per architecture, and they have to be rebuilt for every
+`platformio/espressif32` bump, so committing them would grow history permanently and go
+stale silently. Instead they are published as a release asset and fetched on demand:
+
+```
+scripts/fetch_mbedtls_4k.sh esp32s3            # download + verify against the manifest
+MESHCORE_REDUCED_TLS=1 pio run -e Heltec_v3_repeater_observer_mqtt
+```
+
+- `scripts/mbedtls_4k_manifest.txt` — per-arch sha256 of each archive. **Update it on every
+  platform bump**, together with the published asset.
+- `scripts/fetch_mbedtls_4k.sh` — downloads into `.mbedtls-4k/<arch>/` (gitignored) and
+  verifies. `MBEDTLS_4K_LOCAL=<dir>` copies from a local build tree instead of downloading.
+- `scripts/mbedtls_4k.py` — wired into `esp32_base.extra_scripts`, but a **no-op unless
+  `MESHCORE_REDUCED_TLS=1`**, so ordinary builds need no artifact and behave as before.
+
+The opt-in path is deliberately loud, because both ways this can go wrong produce a
+firmware that looks correct and silently lacks the change:
+
+| failure | what happens without a guard | guard |
+|---|---|---|
+| directory missing or partial | linker ignores an unusable `-L` and resolves mbedTLS from the framework | pre-build: hard error |
+| archives stale after a platform bump | links the wrong build | pre-build: sha256 vs manifest |
+| `-L` present but outranked | framework archives win, flag is inert | post-link: `firmware.map` must resolve every `libmbed*.a` to `.mbedtls-4k/` |
+
+That last one is the reason the post-link check exists rather than trusting the flag: a
+build flag reaching the compiler proves nothing about what got linked.
+
 ## How to verify it worked
 
 1. `strings`/`grep` the new `sdkconfig.h` for the four settings.

lib/PsychicMqttClient/src/PsychicMqttClient.cpp

@@ -433,8 +433,18 @@ void PsychicMqttClient::connect()
         }
     }
 
-    ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_start(_client));
-    ESP_LOGI(TAG, "MQTT client started.");
+    esp_err_t start_result = esp_mqtt_client_start(_client);
+    ESP_ERROR_CHECK_WITHOUT_ABORT(start_result);
+    if (start_result == ESP_OK)
+    {
+        _started = true;
+        ESP_LOGI(TAG, "MQTT client started.");
+    }
+    else
+    {
+        // Reporting success here hides the one state reconnect() cannot recover from.
+        ESP_LOGE(TAG, "MQTT client failed to start: %s", esp_err_to_name(start_result));
+    }
 }
 
 void PsychicMqttClient::reconnect()
@@ -489,9 +499,40 @@ void PsychicMqttClient::disconnect()
     }
 
     esp_mqtt_client_stop(_client);
+    _started = false;
     ESP_LOGI(TAG, "MQTT client stopped.");
 }
 
+void PsychicMqttClient::softDisconnect(unsigned long timeout_ms)
+{
+    if (_client == nullptr)
+    {
+        ESP_LOGW(TAG, "MQTT client not started.");
+        return;
+    }
+
+    if (!_connected)
+    {
+        // Nothing to close; leaving the task alone is the whole point.
+        return;
+    }
+
+    ESP_LOGI(TAG, "Disconnecting MQTT transport (client task retained).");
+    _stopMqttClient = false;
+    esp_mqtt_client_disconnect(_client);
+
+    unsigned long waited = 0;
+    while (!_stopMqttClient && waited < timeout_ms)
+    {
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+        waited += 10;
+    }
+    if (!_stopMqttClient)
+    {
+        ESP_LOGW(TAG, "softDisconnect: no DISCONNECTED event in %lums", timeout_ms);
+    }
+}
+
 void PsychicMqttClient::forceStop()
 {
     if (_client == nullptr)
@@ -506,6 +547,7 @@ void PsychicMqttClient::forceStop()
     }
     ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_stop(_client));
     _connected = false;
+    _started = false;
     ESP_LOGI(TAG, "MQTT client forcefully stopped.");
 }
 

lib/PsychicMqttClient/src/PsychicMqttClient.h

@@ -368,6 +368,29 @@ class PsychicMqttClient
      */
     void disconnect();
 
+    /**
+     * @brief Closes the transport but leaves the client task running.
+     *
+     * disconnect() ends with esp_mqtt_client_stop(), which ends the client task
+     * and returns its 6 KiB stack to the heap right as a TLS handshake vacates
+     * two 16 KiB mbedTLS record buffers — the stack then lands in that hole and
+     * the largest free block ratchets down. This variant omits the stop, so the
+     * task and its stack stay put. Pair it with reconnect().
+     *
+     * @param timeout_ms how long to wait for the DISCONNECTED event before
+     *                   giving up. Bounded on purpose: disconnect()'s wait is
+     *                   unbounded and a lost event would wedge the caller.
+     */
+    void softDisconnect(unsigned long timeout_ms = 5000);
+
+    /**
+     * @brief True once esp_mqtt_client_start() has succeeded and no stop has run.
+     *
+     * reconnect() silently does nothing on a stopped client, so callers that
+     * want to avoid stop/start must check this and fall back to connect().
+     */
+    bool isStarted() const { return _started; }
+
     /**
      * @brief Forcefully stops the MQTT client and disconnects from the server.
      * This does not trigger the onDisconnect callbacks.
@@ -478,6 +501,7 @@ class PsychicMqttClient
     bool _connected = false;
     bool _stopMqttClient = false;
     bool _config_dirty = true;
+    bool _started = false;
 
     // Runtime cap on the esp-mqtt outbox for QoS 0 async publishes (bytes).
     // 0 = disabled. Enforced in publish(); not an esp-mqtt config field.

platformio.ini

@@ -64,6 +64,8 @@ platform = platformio/espressif32@6.11.0
 monitor_filters = esp32_exception_decoder
 extra_scripts =
   pre:scripts/generate_webconfig_html.py
+; No-op unless MESHCORE_REDUCED_TLS=1; see docs/mbedtls-tls-footprint.md.
+  pre:scripts/mbedtls_4k.py
   merge-bin.py
 build_flags = ${arduino_base.build_flags}
   -D ESP32_PLATFORM

scripts/fetch_mbedtls_4k.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Fetch the reduced-TLS mbedTLS archives into .mbedtls-4k/<arch>/.
+#
+# These archives are built from the shipped sdkconfig plus three lines
+# (CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y, IN_CONTENT_LEN 16384,
+# OUT_CONTENT_LEN 4096) and save ~12 KiB of internal DRAM per TLS connection.
+# See docs/mbedtls-tls-footprint.md for the rationale and the build recipe.
+#
+# They are not committed: ~6 MB per architecture, and they must be rebuilt for
+# every platform bump, so they are published as a release asset keyed on the
+# espressif32 platform version instead.
+#
+#   scripts/fetch_mbedtls_4k.sh [arch]        # default: esp32s3
+#
+# Set MBEDTLS_4K_LOCAL to skip the download and copy from a local build tree:
+#   MBEDTLS_4K_LOCAL=~/mbedtls-4k-esp32s3/staged scripts/fetch_mbedtls_4k.sh
+set -euo pipefail
+
+ARCH="${1:-esp32s3}"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DEST="$REPO_ROOT/.mbedtls-4k/$ARCH"
+MANIFEST="$REPO_ROOT/scripts/mbedtls_4k_manifest.txt"
+BASE_URL="${MBEDTLS_4K_BASE_URL:-https://github.com/agessaman/MeshCore/releases/download/mbedtls-4k}"
+
+if [ ! -f "$MANIFEST" ]; then
+  echo "error: missing $MANIFEST" >&2
+  exit 1
+fi
+
+# Manifest lines: <arch> <sha256> <filename>. Blank lines and # comments ignored.
+# The "platform" and "stock:<arch>" lines bind the archives to a framework version;
+# they are the build check's business, not ours, and are not architectures.
+expected="$(awk -v a="$ARCH" '$1 == a && $0 !~ /^#/ {print $2"  "$3}' "$MANIFEST")"
+if [ -z "$expected" ]; then
+  echo "error: no manifest entries for arch '$ARCH'" >&2
+  echo "known arches: $(awk '$0 !~ /^#/ && NF && $1 != "platform" && $1 !~ /^stock:/ {print $1}' \
+    "$MANIFEST" | sort -u | tr '\n' ' ')" >&2
+  exit 1
+fi
+
+mkdir -p "$DEST"
+
+if [ -n "${MBEDTLS_4K_LOCAL:-}" ]; then
+  echo "copying from $MBEDTLS_4K_LOCAL"
+  while read -r _sha name; do
+    cp "$MBEDTLS_4K_LOCAL/$name" "$DEST/$name"
+  done <<< "$expected"
+else
+  TARBALL="mbedtls-4k-$ARCH.tar.gz"
+  echo "downloading $BASE_URL/$TARBALL"
+  tmp="$(mktemp -d)"
+  trap 'rm -rf "$tmp"' EXIT
+  curl -fsSL "$BASE_URL/$TARBALL" -o "$tmp/$TARBALL"
+  tar -xzf "$tmp/$TARBALL" -C "$tmp"
+  while read -r _sha name; do
+    # Accept the archive whether or not the tarball has a leading directory.
+    found="$(find "$tmp" -name "$name" -type f | head -1)"
+    if [ -z "$found" ]; then
+      echo "error: $name missing from $TARBALL" >&2
+      exit 1
+    fi
+    cp "$found" "$DEST/$name"
+  done <<< "$expected"
+fi
+
+# Verify every archive against the manifest. A wrong or truncated archive would
+# otherwise link silently and produce a firmware without the reduced buffers.
+cd "$DEST"
+if command -v shasum >/dev/null 2>&1; then
+  echo "$expected" | shasum -a 256 -c -
+else
+  echo "$expected" | sha256sum -c -
+fi
+
+echo "ok: $ARCH archives verified in $DEST"