feat(#289): I2C dropout back-off and re-probe for every bus device

docs/DIAGNOSTICS.md

@@ -169,6 +169,7 @@ arduino-cli upload --fqbn esp32:esp32:adafruit_feather_esp32s3 --port <blPort> F
 | `--before usb-reset` from a running app | does not enter download mode |
 | bulk `read-flash` over native USB | fails repeatedly; small transactions like `chip-id` work |
 | holding a serial handle across a reset | port re-enumerates, handle dies |
+| `pio-flash confirm` with its output redirected to a file or pipe on Windows, default code page | looks like a hang at `Connecting...` with no `Writing at` line for ten minutes. Not the board: PlatformIO's echo thread dies with `UnicodeEncodeError` (cp1252) on esptool's Unicode progress bar, esptool blocks on the full pipe **mid-write**, and the chip is left in the stub. Set `PYTHONUTF8=1 PYTHONIOENCODING=utf-8` for the wrapper call. Seen twice on 2026-09-08 (#289), clean flash on the third run with the variables set |
 
 A full flash image backup needs a UART adapter wired to RX/TX — it is not
 achievable over native USB.

docs/HARDWARE.md

@@ -378,6 +378,20 @@ Source for all of these: UM's FeatherS3D pinout card,
   needs an external boost from `BAT`. IO34 "5V SENSE" reads the USB net through a
   2 kΩ / 3.3 kΩ divider. STEMMA1 (LDO1) = IO9 SCL / IO8 SDA; STEMMA2 (LDO2) = IO15 SCL /
   IO16 SDA, as the wiring page says.
+- **STEMMA QT chain order on the bench** (owner, 2026-09-08 18:5x, during #289): Feather →
+  BME688 → SHT41 → LSM6DSOX/LIS3MDL (one 4517 board) → TSL2591 (unused). The IMU board was
+  moved up the chain to this position. The bus failure it was moved for did not change
+  with the move: one failed transaction to the IMU board per loop pass, 22.4/s, steady for
+  75 minutes either side of it, and it moved from the LIS3MDL read (garbage magnetometer
+  data until 17:49) to the other read on the same board with no reboot. The scan's 0x29 is
+  that TSL2591, not a VEML7700 (label fixed in #289).
+  The 4517 itself (adafruit.com/product/4517 and learn.adafruit.com/st-9-dof-combo/pinouts,
+  read 2026-09-08): one on-board 3.3 V regulator for both chips (AP2112K, or a ≥150 mA
+  substitute since the 2022-06 parts shortage), FET level shifters on SDA/SCL with 10 kΩ
+  pull-ups, address jumpers on the back — `ADM` bridges the LIS3MDL 0x1C → 0x1E, `ADAG`
+  bridges the LSM6DSOX 0x6A → 0x6B, both open on the bench — and INT1/INT2/INTM/DRDY pins,
+  none wired. Regulator and level shifters are shared by both chips, so a fault in either
+  would hit both reads at once; the bench fault takes one chip at a time.
 - **First run results** (image 8af7ce5): TFT init and LVGL completed and the backlight lit,
   but the panel stayed blank, the I2C1 scan found zero devices (the stacked OLED wing
   included), and the first SD mount never returned. The SD hang was two SPI drivers on one

src/battery.cpp

@@ -2,10 +2,18 @@
 #include "battery.h"
 #include "fram.h"
 #include "logging.h"
+#include "i2c_health.h"
 
 Adafruit_MAX17048 battery;
 bool batteryAvailable = false;
 
+// Dropout tracking (#289). The gauge is read on a 10 s timer and by the
+// settings screen and /diags, never in a tight loop, so a health check every
+// BATT_CHECK_MS is enough. isDeviceReady() reads the version register and is
+// false on a NACK; every cellVoltage()/cellPercent() call already gates on it.
+static I2CDevice battDev = {"MAX17048", 0x36, &batteryAvailable};
+#define BATT_CHECK_MS 2000
+
 void initBattery() {
   logPrint("Initializing MAX17048... ");
 
@@ -15,9 +23,22 @@ void initBattery() {
   }
 
   batteryAvailable = true;
+  i2cNoteReady(battDev);
   logPrintln("OK");
 }
 
+// Called every loop pass (#289).
+void serviceBattery() {
+  if (!batteryAvailable) {
+    if (i2cReprobeDue(battDev)) initBattery();
+    return;
+  }
+  static unsigned long lastCheck = 0;
+  if (millis() - lastCheck < BATT_CHECK_MS) return;
+  lastCheck = millis();
+  if (battery.isDeviceReady()) i2cNoteOk(battDev); else i2cNoteFail(battDev);
+}
+
 // Log battery data to SD card for analysis
 void logBatteryToSD() {
   if (!sdHealth.available || !batteryAvailable) return;

src/battery.h

@@ -9,6 +9,7 @@ extern Adafruit_MAX17048 battery;
 extern bool batteryAvailable;
 
 void initBattery();
+void serviceBattery();   // health check; re-probe a dropped gauge and re-init it (#289)
 void logBatteryToSD();
 void logBatteryToFRAM();
 bool isBatteryConnected();

src/env.cpp

@@ -2,6 +2,7 @@
 #include "env.h"
 #include "fram.h"
 #include "logging.h"
+#include "i2c_health.h"
 
 // BSEC2 IAQ config for BME680/688 at 3.3V, 3-second sample rate, 4-day calibration
 const uint8_t bsec2_config[] = {
@@ -12,6 +13,18 @@ Bsec2 envSensor;
 Adafruit_SHT4x sht4 = Adafruit_SHT4x();  // SHT41 temp/humidity (#48)
 bool bmeAvailable = false;
 bool shtAvailable = false;          // SHT41 temp/humidity (#48)
+
+// Dropout tracking (#289). The BME688 address is whichever answered in
+// initBME688(), the other is the alternate the re-probe also tries, and its
+// chip-ID register is the re-probe check. The SHT41 has one address and no
+// ID register, so its re-probe is an address probe; initSHT41() (a serial
+// number read) is the real test when it answers.
+static I2CDevice shtDev = {"SHT41",  0x44, &shtAvailable};
+static I2CDevice bmeDev = {"BME688", 0x77, &bmeAvailable, BME68X_REG_CHIP_ID, BME68X_CHIP_ID, 0x76};
+// The BME688 is touched once per BSEC sample (LP mode = 3 s) and its status
+// persists until the next access, so its health is sampled on that cadence;
+// sampling every loop pass would count one failed read five times over.
+#define BME_HEALTH_MS 3000
 static uint8_t bsecState[BSEC_MAX_STATE_BLOB_SIZE];
 static unsigned long lastBsecStateSave = 0;
 bool bsecStateLoaded = false;
@@ -88,14 +101,19 @@ void initBME688() {
   };
 
   // Try primary address (0x77), then secondary (0x76)
-  if (!envSensor.begin(0x77, Wire)) {
-    if (!envSensor.begin(0x76, Wire)) {
+  uint8_t addr = 0x77;
+  if (!envSensor.begin(addr, Wire)) {
+    addr = 0x76;
+    if (!envSensor.begin(addr, Wire)) {
       logPrintln("NOT FOUND");
       logPrintf("  BSEC status: %d\n", envSensor.status);
       logPrintf("  Sensor status: %d\n", envSensor.sensor.status);
       return;
     }
   }
+  bmeDev.addr = addr;
+  bmeDev.altAddr = (addr == 0x77) ? 0x76 : 0x77;
+  i2cNoteReady(bmeDev);
 
   // Load BSEC2 IAQ config
   if (!envSensor.setConfig(bsec2_config)) {
@@ -137,6 +155,7 @@ void initSHT41() {
   sht4.setHeater(SHT4X_NO_HEATER);
 
   shtAvailable = true;
+  i2cNoteReady(shtDev);
   logPrintln("OK (0x44)");
 
   // Read initial values immediately
@@ -261,6 +280,9 @@ void readSHT41() {
   if (sht4.getEvent(&humEv, &tempEv)) {
     shtData.temperature = tempEv.temperature;
     shtData.humidity = humEv.relative_humidity;
+    i2cNoteOk(shtDev);
+  } else {
+    i2cNoteFail(shtDev);   // false on a NACK (before the measurement delay) or a bad CRC
   }
 }
 
@@ -272,6 +294,32 @@ void readBME688() {
       LOG_ERROR("BSEC error: %d", envSensor.status);
     }
   }
+
+  // Bus health (#289): the BME68x driver checks endTransmission() and leaves
+  // BME68X_E_COM_FAIL in sensor.status when the chip stops answering. Only
+  // that code counts: the other negative codes (null pointer, self-test,
+  // invalid length) are not bus faults and a re-probe would not clear them
+  // (review finding on #289). BSEC touches the chip in one burst per sample
+  // (setTPH, heater profile, forced mode, then fetchData), so a status
+  // sampled on the same 3 s cadence sees at most one or two samples per
+  // burst: one transient failure cannot reach I2C_FAIL_LIMIT.
+  static unsigned long lastHealth = 0;
+  if (millis() - lastHealth >= BME_HEALTH_MS) {
+    lastHealth = millis();
+    if (envSensor.sensor.status == BME68X_E_COM_FAIL) i2cNoteFail(bmeDev); else i2cNoteOk(bmeDev);
+  }
+}
+
+// Called every loop pass (#289): re-probe a dropped sensor on the slow timer
+// and run its normal init when it answers.
+void serviceSHT41() {
+  if (!shtAvailable && i2cReprobeDue(shtDev)) initSHT41();
+}
+
+void serviceBME688() {
+  if (bmeAvailable || !i2cReprobeDue(bmeDev)) return;
+  initBME688();   // fresh BSEC instance: restore its state the way setup() does
+  if (bmeAvailable && !loadBsecFromFRAM() && sdHealth.available) loadBsecState();
 }
 
 // Helper function to get IAQ accuracy as short text

src/env.h

@@ -41,6 +41,8 @@ bool saveBsecToFRAM();
 bool loadBsecFromFRAM();
 void readSHT41();
 void readBME688();
+void serviceSHT41();    // re-probe a dropped sensor and re-init it (#289)
+void serviceBME688();
 const char* getIaqAccuracyText(uint8_t accuracy);
 const char* getIaqQualityText(float iaq);
 float hPaToInHg(float hPa);

src/i2c_health.cpp

@@ -0,0 +1,147 @@
+// i2c_health.cpp -- per-device I2C failure tracking, back-off and re-probe (#289).
+#include "i2c_health.h"
+#include "logging.h"
+#include <Wire.h>
+#include <esp_system.h>
+
+void i2cNoteOk(I2CDevice& d) {
+  d.fails = 0;
+  d.recent <<= 1;
+  // Probation ends with good behaviour, not only at the next drop: a device
+  // that has answered for I2C_FLAP_MS since it came back is off the back-off
+  // ladder (review finding on #289: otherwise only the timing of the next
+  // failure could clear it).
+  if (d.backoff && millis() - d.lastReturn >= I2C_FLAP_MS) d.backoff = 0;
+}
+
+void i2cNoteReady(I2CDevice& d) {
+  d.fails = 0;
+  d.recent = 0;
+}
+
+bool i2cNoteFail(I2CDevice& d) {
+  d.totalFails++;
+  if (d.fails < 255) d.fails++;
+  d.recent = (d.recent << 1) | 1;
+  if (!*d.available) return false;          // already dropped
+  if (d.fails < I2C_FAIL_LIMIT && __builtin_popcount(d.recent) < I2C_FAIL_WINDOW) return false;
+  *d.available = false;
+  d.drops++;
+  d.droppedAt = millis();
+  d.lastProbe = d.droppedAt;                 // first re-probe one interval from now
+  // Flapping: dropped again soon after it came back, so wait longer each time.
+  if (d.lastReturn && d.droppedAt - d.lastReturn < I2C_FLAP_MS) {
+    if (d.backoff < I2C_BACKOFF_MAX) d.backoff++;
+  } else {
+    d.backoff = 0;
+  }
+  logPrintf("[I2C] %s (0x%02X) stopped answering: %u consecutive failures, %d of the last 16 checks; dropped, re-probing in %lu s\n",
+            d.name, d.addr, (unsigned)d.fails, __builtin_popcount(d.recent),
+            (unsigned long)(I2C_REPROBE_MS << d.backoff) / 1000);
+  return true;
+}
+
+bool i2cProbe(uint8_t addr) {
+  // Zero-length write: the core turns it into i2c_master_probe (address and
+  // ACK only), the same transaction scanI2CBus() uses.
+  Wire.beginTransmission(addr);
+  return Wire.endTransmission() == 0;
+}
+
+bool i2cReadReg(uint8_t addr, uint8_t reg, uint8_t* val) {
+  // The same write-then-read every Adafruit driver uses. endTransmission(false)
+  // only queues the register byte on this core; the combined transaction runs
+  // in requestFrom(), which returns 0 bytes when any part of it is NACKed.
+  Wire.beginTransmission(addr);
+  Wire.write(reg);
+  Wire.endTransmission(false);
+  if (Wire.requestFrom((uint8_t)addr, (uint8_t)1) != 1) return false;
+  *val = Wire.read();
+  return true;
+}
+
+static bool checkAt(uint8_t addr, const I2CDevice& d) {
+  if (d.idReg == I2C_NO_REG) return i2cProbe(addr);
+  uint8_t v;
+  return i2cReadReg(addr, d.idReg, &v) && v == d.idVal;
+}
+
+bool i2cCheck(const I2CDevice& d) {
+  return checkAt(d.addr, d);
+}
+
+// A slave left mid-transaction by a glitch can hold SDA low until it sees
+// enough clocks to finish its byte and then a STOP. Clock it out with the
+// pins as plain GPIO, then hand the pins back to the I2C peripheral. Same
+// end()/begin() pair the #283 bus diagnostic uses, verified on both boards.
+void i2cBusRecover() {
+  Wire.end();
+  // Pins back to GPIO as inputs first: the core refuses digitalWrite() on a
+  // pin still routed to the I2C peripheral (seen in the first recovery on the
+  // bench, 2026-09-08 20:44). Then set the latch high while nothing drives,
+  // then switch SCL to open-drain output so it does not dip on the change.
+  pinMode(SDA, INPUT_PULLUP);
+  pinMode(SCL, INPUT_PULLUP);
+  digitalWrite(SDA, HIGH);
+  digitalWrite(SCL, HIGH);
+  pinMode(SCL, OUTPUT_OPEN_DRAIN);
+  delayMicroseconds(5);
+  for (int i = 0; i < 9; i++) {
+    digitalWrite(SCL, LOW);
+    delayMicroseconds(5);
+    digitalWrite(SCL, HIGH);
+    delayMicroseconds(5);
+    if (digitalRead(SDA) == HIGH) break;   // slave released the line
+  }
+  // START then STOP with no address: every slave's state machine resets on
+  // the STOP. SDA falling while SCL is high is the START.
+  pinMode(SDA, OUTPUT_OPEN_DRAIN);
+  digitalWrite(SDA, LOW);
+  delayMicroseconds(5);
+  digitalWrite(SDA, HIGH);
+  delayMicroseconds(5);
+  Wire.begin();                              // variant SDA/SCL at the default clock, as in setup()
+}
+
+bool i2cReprobeDue(I2CDevice& d) {
+  if (*d.available || d.drops == 0) return false;
+  unsigned long now = millis();
+  if (now - d.lastProbe < (unsigned long)(I2C_REPROBE_MS << d.backoff)) return false;
+  d.lastProbe = now;
+  if (!checkAt(d.addr, d)) {
+    i2cBusRecover();
+    if (!checkAt(d.addr, d)) {
+      if (!d.altAddr || !checkAt(d.altAddr, d)) return false;
+      logPrintf("[I2C] %s answers at 0x%02X now, not 0x%02X\n", d.name, d.altAddr, d.addr);
+      uint8_t was = d.addr;
+      d.addr = d.altAddr;
+      d.altAddr = was;
+    }
+  }
+  logPrintf("[I2C] %s (0x%02X) answers again after %lu s (drop #%lu, %lu failed checks so far); re-initialising\n",
+            d.name, d.addr, (now - d.droppedAt) / 1000, (unsigned long)d.drops, (unsigned long)d.totalFails);
+  d.lastReturn = now;                        // a drop within I2C_FLAP_MS of this raises the back-off
+  return true;                               // the caller's init clears the history via i2cNoteReady() if it succeeds
+}
+
+void logResetReason() {
+  esp_reset_reason_t r = esp_reset_reason();
+  const char* s;
+  switch (r) {
+    case ESP_RST_POWERON:   s = "power-on"; break;
+    case ESP_RST_EXT:       s = "external pin"; break;
+    case ESP_RST_SW:        s = "software (esp_restart)"; break;
+    case ESP_RST_PANIC:     s = "PANIC -- the previous run crashed"; break;
+    case ESP_RST_INT_WDT:   s = "INTERRUPT WATCHDOG"; break;
+    case ESP_RST_TASK_WDT:  s = "TASK WATCHDOG -- loop() stalled"; break;
+    case ESP_RST_WDT:       s = "other watchdog"; break;
+    case ESP_RST_DEEPSLEEP: s = "deep-sleep wake"; break;
+    case ESP_RST_BROWNOUT:  s = "BROWNOUT -- supply dipped"; break;
+    case ESP_RST_SDIO:      s = "SDIO"; break;
+    case ESP_RST_USB:       s = "USB peripheral"; break;
+    case ESP_RST_JTAG:      s = "JTAG"; break;
+    case ESP_RST_EFUSE:     s = "efuse error"; break;
+    default:                s = "unknown"; break;
+  }
+  logPrintf("[BOOT] Reset reason: %s (%d)\n", s, (int)r);
+}