refactor(#270): E4 band 3a -- fram, settings and rtc leave src.ino as units

docs/god-object-inventory.md

@@ -283,6 +283,44 @@ python scripts/verify_extraction.py --before-nm nm-before.txt --before-graph gra
   --after-elf .pio/build/feather_s3/firmware.elf --moved <functions> --published <variables> [--resigned <functions>]
 ```
 
+## 10. Band 3a — fram, settings and rtc; five things the harness learned
+
+**Band 3 was split first.** #264 said it would be, once band 2 had landed and the harness was proven. The relocation graph after band 2 gave the composition: five children, [#270](https://github.com/Strycher/Field_Compass/issues/270)–[#274](https://github.com/Strycher/Field_Compass/issues/274), grouped by dependency layer — storage/settings/time, the four sensors, the two data domains that consume them, display and navigation state, and the web server last. **Not `webServer` first, as #264's text says**: the graph shows `webServer` referenced only by the 23 handlers, `initWebServer` and `loop`, and `initWiFi` calling `initWebServer`. A web unit before the domains its handlers read would need those handlers declared in a header while they stay in `src.ino` — the reach-back the ruling forbids. The cut set says which variables must be owned; the graph says the order they can be. #264's acceptance is met when its last child lands.
+
+**What moved** — three units, in this order because each includes the previous:
+
+| unit | owns | functions |
+|---|---|---|
+| `fram` | `fram`, `framHeader`, `framAvailable`; `FRAMHeader`, `FRAMBatteryEntry`, `FRAMSettings`; the 17 `FRAM_*` layout defines | `framReadHeader`, `framWriteHeader`, `framFormat`, `initFRAM` |
+| `settings` | the nine user and display settings, `tzPresets`/`TZPreset`, the four timeout preset tables and their counts, `settingsLoadedFromSD` | `framSettingsChecksum` (static), `saveSettingsToFRAM`, `loadSettingsFromFRAM`, `applyTimezone`, `mktimeUTC`, `formatTimeStr`, `loadSettings`, `saveSettings`, `factoryReset`, `findTimeoutIndex` |
+| `rtc` | `rtc`, `rtcAvailable` | `initRTC`, `syncRTCFromSystemTime` |
+
+Twelve of the 34 remaining cut-set variables left `src.ino`. `FRAM_CS` and `TFT_BL` joined `fc_config.h`, which `fram.cpp` and `factoryReset` need.
+
+**Calls made under the grant, for review:** the timeout preset tables and `findTimeoutIndex` are *settings* (they define what `tftSleepMs` may be), not display, so they went with the unit that persists them; `framFlushToSD` stayed, because it flushes the battery and weather buffers — two domains not yet published — and the weather child (#272) decides its home.
+
+**Five things the harness learned**, each found by this band and fixed in `extract_unit.py`: bare include names are wrapped (`"x.h"` for our files, `<x.h>` otherwise); a `--header-lines` block is emitted *before* the externs, since an `extern FRAMHeader framHeader;` needs the struct first; a table initialiser spanning lines (`tzPresets`, 16 of them) is carried whole; forward `extern` declarations of published variables are dropped like prototypes are (the timeout tables had six); and the comment line that introduced a removed declaration is swept when only a blank follows it (bands 2 and 3a had left eight).
+
+**Measured** — `scripts/verify_extraction.py`, PASS:
+
+```
+symbols   584 before -> 585 after, union of 8 objects
+          38 relocated: fram 7, rtc 4, settings 27 (incl. the shared_count inline only settings.cpp instantiates)
+           2 linkage changes, both declared: settingsLoadedFromSD b -> B, tzPresets r -> R
+           1 added: _GLOBAL__sub_I_fram (41 bytes) -- fram.cpp constructs the FRAM object;
+             src.ino's initialiser shrank by 34
+          removed: none
+deltas    26 symbols, net +22: 8 call a moved function, 5 touch published state,
+          2 are moved functions (loadSettings -4, findTimeoutIndex -2), 11 identical references
+          unexplained: none
+sections  .flash.text  +64 = function deltas +22, the new initialiser +41, alignment +1
+          .flash.rodata +56, .dram0.bss +16: no data symbol changed size; packing and string-literal
+                          residue across three new objects (rodata/bss bytes: fram 195/73,
+                          settings 1710/10, rtc 155/5)
+Flash     1,856,026 -> 1,856,146 (+120)   RAM 103,232 -> 103,248 (+16)
+src.ino   7,986 -> 7,483 lines           arduino-cli SUCCESS   58 tests pass
+```
+
 ---
 
 ## Appendix — the full tables

scripts/extract_unit.py

@@ -132,7 +132,8 @@ def is_static(lines: list[str], start: int) -> bool:
 
 def find_var(lines: list[str], name: str) -> int:
     pat = re.compile(VAR_RE_TMPL.format(name=re.escape(name)))
-    return one([i for i, l in enumerate(lines) if pat.match(l)], f"variable {name}")
+    return one([i for i, l in enumerate(lines)
+                if pat.match(l) and not l.lstrip().startswith("extern ")], f"variable {name}")
 
 
 def extern_decl(line: str) -> str:
@@ -185,6 +186,10 @@ def main() -> int:
             sys.exit(f"refusing: {x[4]} and {y[4]} overlap")
 
     var_lines = {n: find_var(lines, n) for n in mvars + pvars}
+    # a table initialiser (`static const TZPreset tzPresets[] = {` ... `};`) spans
+    # lines: carry the whole block, not just its first line (band 3a)
+    var_ends = {n: (block_end(lines, i) if "{" in lines[i] and "}" not in lines[i] else i)
+                for n, i in var_lines.items()}
     def_lines = {n: one([i for i, l in enumerate(lines)
                         if re.match(r"^\s*#define\s+" + re.escape(n) + r"\b", l)], f"#define {n}")
                  for n in defines}
@@ -211,10 +216,25 @@ def main() -> int:
             if i != d and pat.match(l):
                 protos[i] = (n, re.sub(r"\s*//.*$", "", l.rstrip("\r\n")).strip())
 
+    # forward `extern` declarations of published variables are redundant once
+    # the header declares them (band 3a: the timeout tables had six)
+    fwd = {}   # line index -> text
+    for n in pvars:
+        fpat = re.compile(r"^\s*extern\s+[^;=]*\b" + re.escape(n) + r"\s*(?:\[[^\]]*\])*\s*;")
+        for i, l in enumerate(lines):
+            if fpat.match(l):
+                fwd[i] = l.strip()
+
     # ---- build outputs ------------------------------------------------------
+    def inc_line(inc: str) -> str:
+        """A bare name becomes "name" if it is one of our files in src/, else <name>."""
+        if inc[0] in "<\"":
+            return f"#include {inc}" + nl
+        return (f'#include "{inc}"' if (SRC.parent / inc).exists() else f"#include <{inc}>") + nl
+
     h = ["#pragma once" + nl, f"// {unit}.h -- extracted from src.ino by scripts/extract_unit.py (E4)." + nl]
     for inc in split(a.header_includes):
-        h.append(f"#include {inc}" + nl)
+        h.append(inc_line(inc))
     if defines:
         h.append(nl)
         h.extend(lines[def_lines[n]] for n in defines)
@@ -223,6 +243,11 @@ def main() -> int:
             s, e = decl_ranges[n]
             h.append(nl)
             h.extend(lines[s:e + 1])
+    if hdr_range:
+        # verbatim block goes BEFORE the externs and prototypes: it is types and
+        # macros, and an extern of a struct type needs the struct first (band 3a)
+        h.append(nl)
+        h.extend(lines[hdr_range[0]:hdr_range[1] + 1])
     if pvars:
         h.append(nl)
         h.extend(extern_decl(lines[var_lines[n]]) + nl for n in pvars)
@@ -232,21 +257,19 @@ def main() -> int:
         for c, d, b, e, n, st in moves:
             if not st:
                 h.append(existing.get(n, prototype(lines, d, b)) + nl)
-    if hdr_range:
-        h.append(nl)
-        h.extend(lines[hdr_range[0]:hdr_range[1] + 1])
 
     cpp = []
     if not a.header_only:
         cpp.append(f"// {unit}.cpp -- extracted from src.ino by scripts/extract_unit.py (E4)." + nl)
         cpp.append(f'#include "{unit}.h"' + nl)
         for inc in split(a.cpp_includes):
-            cpp.append(f"#include {inc}" + nl)
+            cpp.append(inc_line(inc))
         if mvars or pvars:
             cpp.append(nl)
             for n in sorted(mvars + pvars, key=lambda x: var_lines[x]):
                 l = lines[var_lines[n]]
                 cpp.append(re.sub(r"^(\s*)static\s+", r"\1", l) if n in pvars else l)
+                cpp.extend(lines[var_lines[n] + 1:var_ends[n] + 1])
         for c, d, b, e, n, st in moves:
             cpp.append(nl)
             cpp.extend(lines[c:e + 1])
@@ -268,6 +291,8 @@ def main() -> int:
         print(f"  lines {hdr_range[0] + 1}-{hdr_range[1] + 1} -> header verbatim")
     for i in sorted(protos):
         print(f"  reuse and drop prototype of {protos[i][0]} at line {i + 1}: {protos[i][1]}")
+    for i in sorted(fwd):
+        print(f"  drop forward extern at line {i + 1}: {fwd[i]}")
     inc_idx = max(i for i, l in enumerate(lines[:200]) if l.startswith("#include"))
     print(f"  insert #include \"{unit}.h\" after line {inc_idx + 1}")
     if a.dry_run:
@@ -280,13 +305,35 @@ def main() -> int:
         remove.update(range(c, e + 1))
         if e + 1 < len(lines) and lines[e + 1].strip() == "":
             remove.add(e + 1)
-    remove.update(var_lines.values())
+    for n, i in var_lines.items():
+        remove.update(range(i, var_ends[n] + 1))
     remove.update(def_lines.values())
     for s, e in decl_ranges.values():
         remove.update(range(s, e + 1))
     if hdr_range:
         remove.update(range(hdr_range[0], hdr_range[1] + 1))
     remove.update(protos.keys())
+    remove.update(fwd.keys())
+
+    # a `// ...` line that introduced a removed declaration is orphaned when the
+    # next kept line is blank (or the file ends): `// User settings (#70)` over
+    # six variables that all left. Sweep it, and any comment lines stacked
+    # above it. Scoped to removal sites; free-standing comments are untouched.
+    def is_comment(i: int) -> bool:
+        t = lines[i].rstrip("\r\n")
+        return t.lstrip().startswith("//") and not BANNER_RE.match(t)
+
+    for i in sorted(remove):
+        if i == 0 or (i - 1) in remove or not is_comment(i - 1):
+            continue
+        j = i
+        while j < len(lines) and j in remove:
+            j += 1
+        if j >= len(lines) or lines[j].strip() == "":
+            k = i - 1
+            while k >= 0 and k not in remove and is_comment(k):
+                remove.add(k)
+                k -= 1
     out = [l for i, l in enumerate(lines) if i not in remove]
     inc_idx2 = max(i for i, l in enumerate(out[:200]) if l.startswith("#include"))
     out.insert(inc_idx2 + 1, f'#include "{unit}.h"' + nl)

src/fc_config.h

@@ -5,3 +5,5 @@
 #define SPI_SCK   36  // Default Feather SPI clock
 #define SPI_MOSI  35  // Default Feather SPI MOSI
 #define SPI_MISO  37  // Default Feather SPI MISO (unused for TFT)
+#define FRAM_CS   15  // A3 -> FRAM CS
+#define TFT_BL     8  // A5 -> LED pin on display module (PWM-dimmable backlight)

src/fram.cpp

@@ -0,0 +1,69 @@
+// fram.cpp -- extracted from src.ino by scripts/extract_unit.py (E4).
+#include "fram.h"
+#include "fc_config.h"
+#include "logging.h"
+
+Adafruit_FRAM_SPI fram = Adafruit_FRAM_SPI(FRAM_CS);  // SPI FRAM 256KB
+bool framAvailable = false;           // SPI FRAM 256KB
+FRAMHeader framHeader;
+
+// Read FRAM header into RAM (use sizeof to avoid overflowing the struct)
+void framReadHeader() {
+  uint8_t buf[FRAM_HEADER_SIZE];
+  for (int i = 0; i < FRAM_HEADER_SIZE; i++) {
+    buf[i] = fram.read8(FRAM_HEADER_ADDR + i);
+  }
+  memcpy(&framHeader, buf, sizeof(framHeader));  // only copy struct-sized bytes
+}
+
+// Write the RAM header back to FRAM (zero-pad to fill full 64-byte region)
+void framWriteHeader() {
+  uint8_t buf[FRAM_HEADER_SIZE];
+  memset(buf, 0, FRAM_HEADER_SIZE);              // zero-fill padding bytes
+  memcpy(buf, &framHeader, sizeof(framHeader));   // copy struct into buffer
+  for (int i = 0; i < FRAM_HEADER_SIZE; i++) {
+    fram.write8(FRAM_HEADER_ADDR + i, buf[i]);
+  }
+}
+
+// Format FRAM with clean header (zeroed ring buffers)
+void framFormat() {
+  logPrintln("  Formatting...");
+  memset(&framHeader, 0, sizeof(framHeader));
+  framHeader.magic = FRAM_MAGIC;
+  framHeader.version = FRAM_VERSION;
+  framHeader.battCapacity = FRAM_BATT_COUNT;
+  framHeader.wxCapacity = FRAM_WX_COUNT;
+  framWriteHeader();
+  logPrintln("  Format complete");
+}
+
+void initFRAM() {
+  logPrint("Initializing FRAM... ");
+  // Safety check: struct must fit within FRAM header region
+  static_assert(sizeof(FRAMHeader) <= FRAM_HEADER_SIZE, "FRAMHeader exceeds FRAM_HEADER_SIZE");
+
+  if (!fram.begin()) {
+    logPrintln("NOT FOUND (check wiring)");
+    return;
+  }
+
+  // Verify FRAM by reading manufacturer/product IDs
+  uint8_t mfgId;
+  uint16_t prodId;
+  fram.getDeviceID(&mfgId, &prodId);
+  framAvailable = true;
+  logPrintf("OK (mfg:0x%02X prod:0x%04X, 256KB, hdr=%d/%d bytes)\n",
+            mfgId, prodId, sizeof(FRAMHeader), FRAM_HEADER_SIZE);
+
+  // Read and validate FRAM header
+  framReadHeader();
+  if (framHeader.magic != FRAM_MAGIC || framHeader.version != FRAM_VERSION) {
+    logPrintln("  Header invalid - formatting");
+    framFormat();
+  } else {
+    logPrintf("  Batt:%d/%d Wx:%d/%d pending\n",
+              framHeader.battCount, framHeader.battCapacity,
+              framHeader.wxCount, framHeader.wxCapacity);
+  }
+}

src/fram.h

@@ -0,0 +1,80 @@
+#pragma once
+// fram.h -- extracted from src.ino by scripts/extract_unit.py (E4).
+#include <Adafruit_FRAM_SPI.h>
+
+#define FRAM_MAGIC          0x4652414D  // "FRAM" in ASCII
+#define FRAM_VERSION        1
+#define FRAM_HEADER_ADDR    0x00000
+#define FRAM_HEADER_SIZE    64
+#define FRAM_BSEC_ADDR     0x00040     // 512 bytes for BSEC state blob
+#define FRAM_BSEC_SIZE      512
+#define FRAM_BATT_ADDR     0x00240     // Battery ring buffer start
+#define FRAM_BATT_ENTRY     20         // Bytes per battery entry
+#define FRAM_BATT_COUNT     512        // Ring buffer capacity (~85 min at 10s)
+#define FRAM_WX_ADDR       0x02A40     // Weather ring buffer start
+#define FRAM_WX_ENTRY       24         // Bytes per weather entry
+#define FRAM_WX_COUNT       300        // Ring buffer capacity (25 hrs at 5 min)
+#define FRAM_FLUSH_INTERVAL 300000     // Flush to SD every 5 minutes (ms)
+#define FRAM_SETTINGS_ADDR  0x046C0    // User settings backup (after weather ring)
+#define FRAM_SETTINGS_SIZE  128        // Allocated block size
+#define FRAM_SETTINGS_MAGIC 0x53544E47 // "STNG" in ASCII
+#define FRAM_SETTINGS_VER   1
+
+// FRAM ring buffer header (64 bytes, stored at FRAM_HEADER_ADDR)
+struct FRAMHeader {
+  uint32_t magic;            // FRAM_MAGIC validates initialized state
+  uint8_t  version;          // Schema version
+  uint8_t  flags;            // Bit 0: dirty (unwritten data), Bit 1: BSEC valid
+  uint16_t reserved1;
+  // Battery ring
+  uint16_t battHead;         // Next write position (0 to FRAM_BATT_COUNT-1)
+  uint16_t battTail;         // Next flush position
+  uint16_t battCount;        // Entries pending flush
+  uint16_t battCapacity;     // FRAM_BATT_COUNT
+  // Weather ring
+  uint16_t wxHead;
+  uint16_t wxTail;
+  uint16_t wxCount;
+  uint16_t wxCapacity;       // FRAM_WX_COUNT
+  // BSEC metadata
+  uint32_t bsecTimestamp;    // millis() when last saved
+  uint8_t  bsecAccuracy;    // IAQ accuracy at save time
+  uint8_t  reserved2[27];   // Pad to 64 bytes total
+};
+
+struct FRAMBatteryEntry {
+  uint32_t timestamp;        // millis()
+  float    voltage;
+  float    percent;
+  float    rate;             // Charge rate (%/hr)
+  uint16_t flags;            // Reserved
+  uint16_t checksum;         // XOR checksum
+};
+// FRAMWeatherEntry: reuse existing WeatherReading struct (24 bytes, same layout)
+
+// FRAM settings backup struct (#118) — written as raw bytes to FRAM_SETTINGS_ADDR
+struct FRAMSettings {
+  uint32_t magic;              // FRAM_SETTINGS_MAGIC
+  uint8_t  version;            // FRAM_SETTINGS_VER
+  uint8_t  use12Hour;
+  uint8_t  useFahrenheit;
+  uint8_t  useMetricUnits;
+  char     posixTZ[48];
+  char     tzDisplayName[24];
+  int8_t   tzSelectedIndex;
+  uint8_t  tftBrightness;
+  uint8_t  _pad[2];            // Align to 4 bytes
+  uint32_t tftSleepMs;
+  uint32_t oledSleepMs;
+  uint32_t checksum;           // XOR-32 of all preceding bytes
+};
+static_assert(sizeof(FRAMSettings) <= FRAM_SETTINGS_SIZE, "FRAMSettings exceeds allocated block");
+
+extern Adafruit_FRAM_SPI fram;
+extern FRAMHeader framHeader;
+extern bool framAvailable;
+
+void framReadHeader();
+void framWriteHeader();
+void framFormat();
+void initFRAM();

src/rtc.cpp

@@ -0,0 +1,64 @@
+// rtc.cpp -- extracted from src.ino by scripts/extract_unit.py (E4).
+#include "rtc.h"
+#include "settings.h"
+#include "logging.h"
+
+RTC_PCF8523 rtc;
+bool rtcAvailable = false;        // Adalogger RTC
+
+void initRTC() {
+  logPrint("Initializing RTC (PCF8523)... ");
+
+  if (!rtc.begin()) {
+    logPrintln("NOT FOUND");
+    return;
+  }
+
+  rtcAvailable = true;
+
+  // Check if RTC lost power and is running with invalid time
+  if (!rtc.initialized() || rtc.lostPower()) {
+    logPrintln("OK (needs time sync)");
+    // Don't set a default time - wait for GPS or NTP to provide accurate time
+    return;
+  }
+
+  // RTC has valid time - use it to set system time
+  DateTime now = rtc.now();
+  struct tm timeinfo;
+  timeinfo.tm_year = now.year() - 1900;
+  timeinfo.tm_mon = now.month() - 1;
+  timeinfo.tm_mday = now.day();
+  timeinfo.tm_hour = now.hour();
+  timeinfo.tm_min = now.minute();
+  timeinfo.tm_sec = now.second();
+
+  time_t t = mktimeUTC(&timeinfo);  // RTC stores UTC — use UTC-aware mktime (#98 bugfix)
+  struct timeval tv = { .tv_sec = t, .tv_usec = 0 };
+  settimeofday(&tv, NULL);
+
+  logPrintf("OK (%04d-%02d-%02d %02d:%02d:%02d)\n",
+            now.year(), now.month(), now.day(),
+            now.hour(), now.minute(), now.second());
+}
+
+// Sync RTC from current system time (call after GPS or NTP sync)
+void syncRTCFromSystemTime(const char* source) {
+  if (!rtcAvailable) return;
+
+  struct tm timeinfo;
+  if (!getLocalTime(&timeinfo)) return;
+
+  // Adjust for timezone - RTC stores UTC
+  time_t now;
+  time(&now);
+  struct tm* utc = gmtime(&now);
+
+  rtc.adjust(DateTime(utc->tm_year + 1900, utc->tm_mon + 1, utc->tm_mday,
+                      utc->tm_hour, utc->tm_min, utc->tm_sec));
+
+  logPrintf("[RTC] Synced from %s: %04d-%02d-%02d %02d:%02d:%02d UTC\n",
+            source,
+            utc->tm_year + 1900, utc->tm_mon + 1, utc->tm_mday,
+            utc->tm_hour, utc->tm_min, utc->tm_sec);
+}