fix(memory): hot-layer entry markers — scaffold both, heal a missing BEGIN, migrate existing installs

LifeOS/install/LIFEOS/TOOLS/MemoryWriter.markers.test.ts

@@ -0,0 +1,131 @@
+#!/usr/bin/env bun
+/**
+ * Regression test for the hot-layer memory marker bug.
+ * Exercises the never-tested path: a fact written in one "session" must be
+ * readable in the next. Run with an isolated HOME so it hits the real public
+ * API (setEntries/read) without touching the live tree:
+ *   HOME=/tmp/mem-test bun test-memory-markers.ts
+ */
+import { mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const WT = new URL(".", import.meta.url).pathname;
+const HOME = homedir();
+const USER = join(HOME, ".claude/LIFEOS/USER/PRINCIPAL");
+const OBS = join(HOME, ".claude/LIFEOS/MEMORY/OBSERVABILITY");
+const FILE = join(USER, "PRINCIPAL_MEMORY.md");
+
+const FIXED_TEMPLATE = `---
+schema_version: 1
+cap_entries: 48
+cap_chars_per_entry: 256
+last_updated: 2026-01-01
+last_updated_by: bootstrap-template
+convention: pai-freshness-v1
+---
+
+# Principal Hot-Layer Memory
+
+> Auto-curated hot-layer memory about you.
+
+<!-- Entries are appended automatically by the memory curation loop. Empty on a fresh install — this is expected. -->
+<!-- BEGIN ENTRIES -->
+<!-- END ENTRIES -->
+`;
+
+// A file corrupted by the OLD bug: no BEGIN, stray ENDs, orphaned entries.
+const CORRUPTED = `---
+schema_version: 1
+cap_entries: 48
+cap_chars_per_entry: 256
+last_updated: 2026-01-01
+last_updated_by: bootstrap-template
+convention: pai-freshness-v1
+---
+
+# Principal Hot-Layer Memory
+
+<!-- Entries are appended automatically by the memory curation loop. Empty on a fresh install — this is expected. -->
+PREFERENCE: existing fact one ~explicit
+<!-- END ENTRIES -->
+PREFERENCE: existing fact two ~inferred
+<!-- END ENTRIES -->
+`;
+
+const BEGIN = "<!-- BEGIN ENTRIES -->";
+const END = "<!-- END ENTRIES -->";
+
+// Mirror of LoadMemory.hook.ts readMemory(): the consumer contract.
+function readerSees(raw: string): string[] {
+  const s = raw.indexOf(BEGIN), e = raw.indexOf(END);
+  if (s === -1 || e === -1 || e < s) return [];
+  return raw.slice(s + BEGIN.length, e).trim().split("\n").map(l => l.trim()).filter(Boolean);
+}
+
+let failures = 0;
+const check = (name: string, cond: boolean, detail = "") => {
+  console.log(`${cond ? "✅" : "❌"} ${name}${cond ? "" : "  — " + detail}`);
+  if (!cond) failures++;
+};
+
+mkdirSync(USER, { recursive: true });
+mkdirSync(OBS, { recursive: true });
+mkdirSync(join(HOME, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT"), { recursive: true });
+
+const { setEntries, read } = await import(join(WT, "MemoryWriter.ts"));
+
+// ── Scenario A: fresh (fixed) template — write in session 1, read in session 2 ──
+writeFileSync(FILE, FIXED_TEMPLATE);
+check("A0: fresh template starts empty to the reader", readerSees(readFileSync(FILE, "utf8")).length === 0);
+setEntries(FILE, ["PREFERENCE: session-1 wrote this ~explicit"]);
+const afterA = readFileSync(FILE, "utf8");
+check("A1: reader (LoadMemory contract) now sees the written fact", readerSees(afterA).includes("PREFERENCE: session-1 wrote this ~explicit"));
+check("A2: read() API returns 1 entry", (read(FILE) as any).count === 1);
+check("A3: exactly one BEGIN + one END", (afterA.match(/BEGIN ENTRIES/g)||[]).length === 1 && (afterA.match(/END ENTRIES/g)||[]).length === 1);
+
+// ── Scenario B: heal a file the OLD bug already corrupted ──
+// The real heal path: read() (parseFile) recovers orphaned entries, then
+// MemorySystem.add does read → append → setEntries (MemorySystem.ts:243).
+writeFileSync(FILE, CORRUPTED);
+check("B0: corrupted file reads as 0 to the consumer (the bug)", readerSees(CORRUPTED).length === 0);
+const recovered = (read(FILE) as any).entries as string[];
+check("B1: read()/parseFile recovers orphaned fact one", recovered.includes("PREFERENCE: existing fact one ~explicit"), recovered.join(" | "));
+check("B2: read()/parseFile recovers orphaned fact two", recovered.includes("PREFERENCE: existing fact two ~inferred"), recovered.join(" | "));
+// Simulate MemorySystem.add: merge current + new, then set-overwrite.
+setEntries(FILE, [...recovered, "PREFERENCE: session-N new fact ~explicit"]);
+const afterB = readFileSync(FILE, "utf8");
+const seenB = readerSees(afterB);
+check("B3: after add-flow, reader sees both orphaned facts + the new one", seenB.length === 3 && seenB.includes("PREFERENCE: session-N new fact ~explicit"), seenB.join(" | "));
+check("B4: self-healed to exactly one BEGIN + one END", (afterB.match(/BEGIN ENTRIES/g)||[]).length === 1 && (afterB.match(/END ENTRIES/g)||[]).length === 1, `B=${(afterB.match(/BEGIN ENTRIES/g)||[]).length} E=${(afterB.match(/END ENTRIES/g)||[]).length}`);
+
+// ── Scenario C: recovery must NOT adopt PREFIX lines inside a code fence / blockquote ──
+const FENCED = `---
+schema_version: 1
+cap_entries: 48
+cap_chars_per_entry: 256
+last_updated: 2026-01-01
+last_updated_by: bootstrap-template
+convention: pai-freshness-v1
+---
+
+# Principal Hot-Layer Memory
+
+> Auto-curated hot-layer memory about you.
+
+Example of the entry format used below:
+\`\`\`
+PREFERENCE: this is only a documentation example ~explicit
+\`\`\`
+> RULE: this blockquoted line is a citation, not a real entry ~explicit
+PREFERENCE: this one is a genuine entry ~explicit
+`;
+writeFileSync(FILE, FENCED);
+const recC = (read(FILE) as any).entries as string[];
+check("C1: fenced PREFERENCE example NOT adopted as an entry", !recC.some(e => e.includes("documentation example")), recC.join(" | "));
+check("C2: blockquoted RULE citation NOT adopted", !recC.some(e => e.includes("citation")), recC.join(" | "));
+check("C3: the one genuine entry IS recovered", recC.includes("PREFERENCE: this one is a genuine entry ~explicit"), recC.join(" | "));
+
+rmSync(join(HOME, ".claude"), { recursive: true, force: true });
+console.log(failures === 0 ? "\n🎉 ALL PASS" : `\n💥 ${failures} FAILURE(S)`);
+process.exit(failures === 0 ? 0 : 1);

LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts

@@ -222,12 +222,39 @@ function parseFile(content: string): ParsedFile {
   const endIdx = afterFm.indexOf(END_MARKER);
 
   if (beginIdx === -1 || endIdx === -1 || endIdx < beginIdx) {
-    // Markers missing or malformed — treat as empty entries with the whole
-    // body as preEntries; this preserves principal content if they edited.
+    // Markers missing or malformed. Happens on files scaffolded before the
+    // entry markers existed, and on files a prior write corrupted by appending
+    // after the body (leaving stray END markers and no BEGIN). Recover any
+    // existing entries from the body and drop stray markers, so serializeFile
+    // can re-emit a canonical BEGIN…END block. Without this, the whole body
+    // (including real entries) becomes preEntriesBody, the entries block stays
+    // empty, and readers that slice between the markers silently see nothing.
+    const recoveredEntries: string[] = [];
+    const preambleLines: string[] = [];
+    let sawEntry = false;
+    let inFence = false;
+    for (const line of afterFm.split("\n")) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
+        inFence = !inFence;
+        if (!sawEntry) preambleLines.push(line);
+        continue;
+      }
+      if (trimmed === BEGIN_MARKER || trimmed === END_MARKER) continue;
+      // Only treat a line as an entry when it is not inside a fenced code block
+      // or a blockquote — prose/examples there can legitimately look like an
+      // entry (e.g. a documented `PREFERENCE: …` sample) and must not be adopted.
+      if (!inFence && !trimmed.startsWith(">") && PREFIX_PATTERN.test(trimmed)) {
+        recoveredEntries.push(trimmed);
+        sawEntry = true;
+        continue;
+      }
+      if (!sawEntry) preambleLines.push(line);
+    }
     return {
       frontmatter,
-      preEntriesBody: afterFm,
-      entries: [],
+      preEntriesBody: preambleLines.join("\n").replace(/\s+$/, "") + "\n",
+      entries: recoveredEntries,
       postEntriesBody: "",
     };
   }
@@ -267,8 +294,14 @@ function serializeFile(parsed: ParsedFile, newEntries: string[], updatedBy: stri
   let fm = updateFrontmatterTimestamp(parsed.frontmatter);
   fm = updateFrontmatterUpdatedBy(fm, updatedBy);
 
-  // Ensure preEntriesBody ends just after BEGIN_MARKER, with newline before entries
+  // Ensure the entries block is opened by exactly one BEGIN_MARKER. Healthy
+  // files already carry it in preEntriesBody; a recovered/marker-less file does
+  // not, so add it here — the symmetric partner to the END_MARKER guard below.
+  // Without both guards a missing BEGIN silently swallows every entry.
   let pre = parsed.preEntriesBody;
+  if (!pre.includes(BEGIN_MARKER)) {
+    pre = pre.replace(/\s+$/, "") + "\n" + BEGIN_MARKER;
+  }
   if (!pre.endsWith("\n")) pre += "\n";
 
   const entriesBlock = newEntries.length === 0 ? "" : newEntries.join("\n") + "\n";

LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts

@@ -0,0 +1,54 @@
+#!/usr/bin/env bun
+// Run with an isolated HOME: HOME=<tmp> bun test-migration.ts
+import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const MIG = new URL("./MigrateMemoryMarkers.ts", import.meta.url).pathname;
+const H = homedir();
+const P = join(H, ".claude/LIFEOS/USER/PRINCIPAL/PRINCIPAL_MEMORY.md");
+const D = join(H, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT/DA_MEMORY.md");
+mkdirSync(join(H, ".claude/LIFEOS/USER/PRINCIPAL"), { recursive: true });
+mkdirSync(join(H, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT"), { recursive: true });
+mkdirSync(join(H, ".claude/LIFEOS/MEMORY/OBSERVABILITY"), { recursive: true });
+
+const FM = `---\nschema_version: 1\ncap_entries: 48\ncap_chars_per_entry: 256\nlast_updated: 2026-01-01\nlast_updated_by: bootstrap-template\nconvention: pai-freshness-v1\n---\n\n# Hot-Layer Memory\n\n`;
+// Corrupted principal: no BEGIN, two stray ENDs, two orphaned entries.
+writeFileSync(P, FM + `PREFERENCE: orphaned alpha ~explicit\n<!-- END ENTRIES -->\nPREFERENCE: orphaned beta ~inferred\n<!-- END ENTRIES -->\n`);
+// Canonical DA: must be left untouched.
+const CANON = FM + `<!-- BEGIN ENTRIES -->\nRULE: healthy entry ~explicit\n<!-- END ENTRIES -->\n`;
+writeFileSync(D, CANON);
+
+const BEGIN = "<!-- BEGIN ENTRIES -->", END = "<!-- END ENTRIES -->";
+const readerSees = (raw: string): string[] => {
+  const s = raw.indexOf(BEGIN), e = raw.indexOf(END);
+  if (s === -1 || e === -1 || e < s) return [];
+  return raw.slice(s + BEGIN.length, e).trim().split("\n").map(l => l.trim()).filter(Boolean);
+};
+const run = (dry: boolean) => Bun.spawnSync(["bun", MIG, ...(dry ? ["--dry-run"] : [])], { env: { ...process.env, HOME: H } });
+
+let fails = 0;
+const check = (n: string, c: boolean, d = "") => { console.log(`${c ? "✅" : "❌"} ${n}${c ? "" : "  — " + d}`); if (!c) fails++; };
+
+// 1) dry-run: reports but writes nothing
+const dry = run(true); const dryOut = dry.stdout.toString();
+check("dry-run flags principal as would-repair", /would-repair.*PRINCIPAL_MEMORY/s.test(dryOut) || /would-repair/.test(dryOut), dryOut);
+check("dry-run leaves canonical DA as skip-ok", /skip-ok.*DA_MEMORY|DA_MEMORY.*already/s.test(dryOut), dryOut);
+check("dry-run did NOT modify the corrupted file", readerSees(readFileSync(P, "utf8")).length === 0);
+
+// 2) real run: heals principal, leaves DA
+const real = run(false); const realOut = real.stdout.toString();
+check("real run reports principal repaired", /repaired.*PRINCIPAL_MEMORY|PRINCIPAL_MEMORY.*recovered/s.test(realOut), realOut);
+const healed = readFileSync(P, "utf8");
+const seen = readerSees(healed);
+check("healed principal: reader sees orphaned alpha", seen.includes("PREFERENCE: orphaned alpha ~explicit"), seen.join(" | "));
+check("healed principal: reader sees orphaned beta", seen.includes("PREFERENCE: orphaned beta ~inferred"), seen.join(" | "));
+check("healed principal: exactly one BEGIN + one END", (healed.match(/BEGIN ENTRIES/g)||[]).length === 1 && (healed.match(/END ENTRIES/g)||[]).length === 1);
+check("canonical DA left byte-identical", readFileSync(D, "utf8") === CANON);
+
+// 3) idempotent: second run repairs nothing
+const again = run(false).stdout.toString();
+check("second run is idempotent (0 repaired)", /^0 repaired|\n0 repaired/m.test(again) || again.includes("0 repaired"), again);
+
+console.log(fails === 0 ? "\n🎉 MIGRATION ALL PASS" : `\n💥 ${fails} FAILURE(S)`);
+process.exit(fails === 0 ? 0 : 1);

LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.ts

@@ -0,0 +1,79 @@
+#!/usr/bin/env bun
+/**
+ * MigrateMemoryMarkers — one-time repair for hot-layer memory files scaffolded
+ * before the entry markers existed, or corrupted by the pre-fix writer (which
+ * re-added END but never BEGIN, leaving stray END markers and no readable
+ * entry block). Existing installs would otherwise stay silently empty to the
+ * reader until the next reviewer write; this heals them immediately on update.
+ *
+ * Idempotent: files already in canonical <!-- BEGIN ENTRIES -->…<!-- END ENTRIES -->
+ * shape are left untouched. Reuses the fixed read()/setEntries() path, so entry
+ * recovery is exactly the behavior covered by MemoryWriter.markers.test.ts.
+ *
+ * Usage:
+ *   bun ~/.claude/LIFEOS/TOOLS/MigrateMemoryMarkers.ts
+ *   bun ~/.claude/LIFEOS/TOOLS/MigrateMemoryMarkers.ts --dry-run
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { PRINCIPAL_MEMORY_PATH, DA_MEMORY_PATH } from "./MemoryTypes";
+import { read, setEntries } from "./MemoryWriter";
+
+const BEGIN_MARKER = "<!-- BEGIN ENTRIES -->";
+const END_MARKER = "<!-- END ENTRIES -->";
+const TARGETS = [PRINCIPAL_MEMORY_PATH, DA_MEMORY_PATH];
+
+/** Canonical = exactly one BEGIN and one END, in order. Anything else needs repair. */
+function isCanonical(content: string): boolean {
+  const begins = (content.match(/<!-- BEGIN ENTRIES -->/g) || []).length;
+  const ends = (content.match(/<!-- END ENTRIES -->/g) || []).length;
+  return begins === 1 && ends === 1 && content.indexOf(BEGIN_MARKER) < content.indexOf(END_MARKER);
+}
+
+interface Result { path: string; action: string; detail: string; }
+
+function repair(path: string, dryRun: boolean): Result {
+  if (!existsSync(path)) {
+    return { path, action: "skip-missing", detail: "no file (a fresh install scaffolds it correctly)" };
+  }
+  const content = readFileSync(path, "utf8");
+  if (isCanonical(content)) {
+    return { path, action: "skip-ok", detail: "already has a canonical BEGIN…END block" };
+  }
+  // read() (post-fix) recovers orphaned entries from the broken body.
+  const r = read(path);
+  if (!("entries" in r)) {
+    return { path, action: "error", detail: `read failed: ${(r as any).message ?? "unknown"}` };
+  }
+  const recovered = r.entries;
+  if (dryRun) {
+    return { path, action: "would-repair", detail: `${recovered.length} entr${recovered.length === 1 ? "y" : "ies"} recoverable` };
+  }
+  // Back up the pre-repair file next to it (repo convention: Backups/).
+  const backupDir = join(dirname(path), "Backups");
+  if (!existsSync(backupDir)) mkdirSync(backupDir, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  writeFileSync(join(backupDir, `${basename(path, ".md")}-pre-marker-migration-${stamp}.md`), content);
+
+  // setEntries rewrites a canonical block (serializeFile re-emits BEGIN…END).
+  const res = setEntries(path, recovered, { updatedBy: "MigrateMemoryMarkers", allowDrastic: true });
+  if (!res.ok) {
+    return { path, action: "error", detail: `setEntries failed: ${res.code} ${res.message}` };
+  }
+  return { path, action: "repaired", detail: `recovered ${recovered.length} entr${recovered.length === 1 ? "y" : "ies"} into a canonical block` };
+}
+
+function main(): void {
+  const dryRun = process.argv.includes("--dry-run");
+  const results = TARGETS.map((p) => repair(p, dryRun));
+  console.log(`MigrateMemoryMarkers${dryRun ? " (dry-run)" : ""}`);
+  for (const res of results) {
+    console.log(`  ${res.action.padEnd(14)} ${basename(dirname(res.path))}/${basename(res.path)} — ${res.detail}`);
+  }
+  const repaired = results.filter((r) => r.action === "repaired" || r.action === "would-repair").length;
+  const errored = results.filter((r) => r.action === "error").length;
+  console.log(`\n${repaired} repaired, ${results.length - repaired - errored} already-ok/skipped, ${errored} error(s).`);
+  process.exit(errored > 0 ? 1 : 0);
+}
+
+main();

LifeOS/install/USER/DIGITAL_ASSISTANT/DA_MEMORY.md

@@ -12,3 +12,5 @@ convention: pai-freshness-v1
 > 🎯 SAMPLE TEMPLATE — auto-curated hot-layer memory your DA keeps about how it works with you. Starts empty; the memory reviewer populates it over time. Nothing to fill in manually. Pulse's memory panel reads this file.
 
 <!-- Entries are appended automatically by the memory curation loop. Empty on a fresh install — this is expected. -->
+<!-- BEGIN ENTRIES -->
+<!-- END ENTRIES -->

LifeOS/install/USER/PRINCIPAL/PRINCIPAL_MEMORY.md

@@ -12,3 +12,5 @@ convention: pai-freshness-v1
 > 🎯 SAMPLE TEMPLATE — auto-curated hot-layer memory about you. It starts empty and the memory reviewer writes durable facts here as you work with your DA. Nothing to fill in manually. Pulse's memory panel reads this file.
 
 <!-- Entries are appended automatically by the memory curation loop. Empty on a fresh install — this is expected. -->
+<!-- BEGIN ENTRIES -->
+<!-- END ENTRIES -->