From 904d944fed1a44800d6a2e579b3e45b8a79439ef Mon Sep 17 00:00:00 2001 From: rpriven Date: Sat, 4 Jul 2026 22:00:41 -0600 Subject: [PATCH] =?UTF-8?q?fix(memory):=20hot-layer=20entry=20markers=20?= =?UTF-8?q?=E2=80=94=20scaffold=20both,=20heal=20a=20missing=20BEGIN,=20mi?= =?UTF-8?q?grate=20existing=20installs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hot-layer memory never loads into a session on a stock install: the templates ship without / markers, and serializeFile re-adds a missing END but never a missing BEGIN — so every write to a marker-less file accretes stray END markers and the reader (LoadMemory) slices an empty block. The reviewer captures durable facts that are then never recalled. - templates: ship both markers (fixes fresh installs) - MemoryWriter.serializeFile: heal a missing BEGIN, symmetric to the END guard - MemoryWriter.parseFile: recover orphaned entries from an already-broken file (skipping code-fence/blockquote lookalikes) so existing data survives next write - MigrateMemoryMarkers.ts: idempotent one-time repair for existing installs - tests: cover the previously-untested write-then-recall-next-session path Co-Authored-By: Kai --- .../LIFEOS/TOOLS/MemoryWriter.markers.test.ts | 131 ++++++++++++++++++ LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts | 43 +++++- .../LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts | 54 ++++++++ .../LIFEOS/TOOLS/MigrateMemoryMarkers.ts | 79 +++++++++++ .../USER/DIGITAL_ASSISTANT/DA_MEMORY.md | 2 + .../USER/PRINCIPAL/PRINCIPAL_MEMORY.md | 2 + 6 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 LifeOS/install/LIFEOS/TOOLS/MemoryWriter.markers.test.ts create mode 100644 LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts create mode 100644 LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.ts diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.markers.test.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.markers.test.ts new file mode 100644 index 000000000..03dc78b07 --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.markers.test.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env bun +/** + * Regression test for the hot-layer memory marker bug. + * Exercises the never-tested path: a fact written in one "session" must be + * readable in the next. Run with an isolated HOME so it hits the real public + * API (setEntries/read) without touching the live tree: + * HOME=/tmp/mem-test bun test-memory-markers.ts + */ +import { mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const WT = new URL(".", import.meta.url).pathname; +const HOME = homedir(); +const USER = join(HOME, ".claude/LIFEOS/USER/PRINCIPAL"); +const OBS = join(HOME, ".claude/LIFEOS/MEMORY/OBSERVABILITY"); +const FILE = join(USER, "PRINCIPAL_MEMORY.md"); + +const FIXED_TEMPLATE = `--- +schema_version: 1 +cap_entries: 48 +cap_chars_per_entry: 256 +last_updated: 2026-01-01 +last_updated_by: bootstrap-template +convention: pai-freshness-v1 +--- + +# Principal Hot-Layer Memory + +> Auto-curated hot-layer memory about you. + + + + +`; + +// A file corrupted by the OLD bug: no BEGIN, stray ENDs, orphaned entries. +const CORRUPTED = `--- +schema_version: 1 +cap_entries: 48 +cap_chars_per_entry: 256 +last_updated: 2026-01-01 +last_updated_by: bootstrap-template +convention: pai-freshness-v1 +--- + +# Principal Hot-Layer Memory + + +PREFERENCE: existing fact one ~explicit + +PREFERENCE: existing fact two ~inferred + +`; + +const BEGIN = ""; +const END = ""; + +// Mirror of LoadMemory.hook.ts readMemory(): the consumer contract. +function readerSees(raw: string): string[] { + const s = raw.indexOf(BEGIN), e = raw.indexOf(END); + if (s === -1 || e === -1 || e < s) return []; + return raw.slice(s + BEGIN.length, e).trim().split("\n").map(l => l.trim()).filter(Boolean); +} + +let failures = 0; +const check = (name: string, cond: boolean, detail = "") => { + console.log(`${cond ? "✅" : "❌"} ${name}${cond ? "" : " — " + detail}`); + if (!cond) failures++; +}; + +mkdirSync(USER, { recursive: true }); +mkdirSync(OBS, { recursive: true }); +mkdirSync(join(HOME, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT"), { recursive: true }); + +const { setEntries, read } = await import(join(WT, "MemoryWriter.ts")); + +// ── Scenario A: fresh (fixed) template — write in session 1, read in session 2 ── +writeFileSync(FILE, FIXED_TEMPLATE); +check("A0: fresh template starts empty to the reader", readerSees(readFileSync(FILE, "utf8")).length === 0); +setEntries(FILE, ["PREFERENCE: session-1 wrote this ~explicit"]); +const afterA = readFileSync(FILE, "utf8"); +check("A1: reader (LoadMemory contract) now sees the written fact", readerSees(afterA).includes("PREFERENCE: session-1 wrote this ~explicit")); +check("A2: read() API returns 1 entry", (read(FILE) as any).count === 1); +check("A3: exactly one BEGIN + one END", (afterA.match(/BEGIN ENTRIES/g)||[]).length === 1 && (afterA.match(/END ENTRIES/g)||[]).length === 1); + +// ── Scenario B: heal a file the OLD bug already corrupted ── +// The real heal path: read() (parseFile) recovers orphaned entries, then +// MemorySystem.add does read → append → setEntries (MemorySystem.ts:243). +writeFileSync(FILE, CORRUPTED); +check("B0: corrupted file reads as 0 to the consumer (the bug)", readerSees(CORRUPTED).length === 0); +const recovered = (read(FILE) as any).entries as string[]; +check("B1: read()/parseFile recovers orphaned fact one", recovered.includes("PREFERENCE: existing fact one ~explicit"), recovered.join(" | ")); +check("B2: read()/parseFile recovers orphaned fact two", recovered.includes("PREFERENCE: existing fact two ~inferred"), recovered.join(" | ")); +// Simulate MemorySystem.add: merge current + new, then set-overwrite. +setEntries(FILE, [...recovered, "PREFERENCE: session-N new fact ~explicit"]); +const afterB = readFileSync(FILE, "utf8"); +const seenB = readerSees(afterB); +check("B3: after add-flow, reader sees both orphaned facts + the new one", seenB.length === 3 && seenB.includes("PREFERENCE: session-N new fact ~explicit"), seenB.join(" | ")); +check("B4: self-healed to exactly one BEGIN + one END", (afterB.match(/BEGIN ENTRIES/g)||[]).length === 1 && (afterB.match(/END ENTRIES/g)||[]).length === 1, `B=${(afterB.match(/BEGIN ENTRIES/g)||[]).length} E=${(afterB.match(/END ENTRIES/g)||[]).length}`); + +// ── Scenario C: recovery must NOT adopt PREFIX lines inside a code fence / blockquote ── +const FENCED = `--- +schema_version: 1 +cap_entries: 48 +cap_chars_per_entry: 256 +last_updated: 2026-01-01 +last_updated_by: bootstrap-template +convention: pai-freshness-v1 +--- + +# Principal Hot-Layer Memory + +> Auto-curated hot-layer memory about you. + +Example of the entry format used below: +\`\`\` +PREFERENCE: this is only a documentation example ~explicit +\`\`\` +> RULE: this blockquoted line is a citation, not a real entry ~explicit +PREFERENCE: this one is a genuine entry ~explicit +`; +writeFileSync(FILE, FENCED); +const recC = (read(FILE) as any).entries as string[]; +check("C1: fenced PREFERENCE example NOT adopted as an entry", !recC.some(e => e.includes("documentation example")), recC.join(" | ")); +check("C2: blockquoted RULE citation NOT adopted", !recC.some(e => e.includes("citation")), recC.join(" | ")); +check("C3: the one genuine entry IS recovered", recC.includes("PREFERENCE: this one is a genuine entry ~explicit"), recC.join(" | ")); + +rmSync(join(HOME, ".claude"), { recursive: true, force: true }); +console.log(failures === 0 ? "\n🎉 ALL PASS" : `\n💥 ${failures} FAILURE(S)`); +process.exit(failures === 0 ? 0 : 1); diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts index f33f5424f..cc4ddaddd 100755 --- a/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryWriter.ts @@ -222,12 +222,39 @@ function parseFile(content: string): ParsedFile { const endIdx = afterFm.indexOf(END_MARKER); if (beginIdx === -1 || endIdx === -1 || endIdx < beginIdx) { - // Markers missing or malformed — treat as empty entries with the whole - // body as preEntries; this preserves principal content if they edited. + // Markers missing or malformed. Happens on files scaffolded before the + // entry markers existed, and on files a prior write corrupted by appending + // after the body (leaving stray END markers and no BEGIN). Recover any + // existing entries from the body and drop stray markers, so serializeFile + // can re-emit a canonical BEGIN…END block. Without this, the whole body + // (including real entries) becomes preEntriesBody, the entries block stays + // empty, and readers that slice between the markers silently see nothing. + const recoveredEntries: string[] = []; + const preambleLines: string[] = []; + let sawEntry = false; + let inFence = false; + for (const line of afterFm.split("\n")) { + const trimmed = line.trim(); + if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) { + inFence = !inFence; + if (!sawEntry) preambleLines.push(line); + continue; + } + if (trimmed === BEGIN_MARKER || trimmed === END_MARKER) continue; + // Only treat a line as an entry when it is not inside a fenced code block + // or a blockquote — prose/examples there can legitimately look like an + // entry (e.g. a documented `PREFERENCE: …` sample) and must not be adopted. + if (!inFence && !trimmed.startsWith(">") && PREFIX_PATTERN.test(trimmed)) { + recoveredEntries.push(trimmed); + sawEntry = true; + continue; + } + if (!sawEntry) preambleLines.push(line); + } return { frontmatter, - preEntriesBody: afterFm, - entries: [], + preEntriesBody: preambleLines.join("\n").replace(/\s+$/, "") + "\n", + entries: recoveredEntries, postEntriesBody: "", }; } @@ -267,8 +294,14 @@ function serializeFile(parsed: ParsedFile, newEntries: string[], updatedBy: stri let fm = updateFrontmatterTimestamp(parsed.frontmatter); fm = updateFrontmatterUpdatedBy(fm, updatedBy); - // Ensure preEntriesBody ends just after BEGIN_MARKER, with newline before entries + // Ensure the entries block is opened by exactly one BEGIN_MARKER. Healthy + // files already carry it in preEntriesBody; a recovered/marker-less file does + // not, so add it here — the symmetric partner to the END_MARKER guard below. + // Without both guards a missing BEGIN silently swallows every entry. let pre = parsed.preEntriesBody; + if (!pre.includes(BEGIN_MARKER)) { + pre = pre.replace(/\s+$/, "") + "\n" + BEGIN_MARKER; + } if (!pre.endsWith("\n")) pre += "\n"; const entriesBlock = newEntries.length === 0 ? "" : newEntries.join("\n") + "\n"; diff --git a/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts b/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts new file mode 100644 index 000000000..2064fae4f --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.test.ts @@ -0,0 +1,54 @@ +#!/usr/bin/env bun +// Run with an isolated HOME: HOME= bun test-migration.ts +import { mkdirSync, writeFileSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const MIG = new URL("./MigrateMemoryMarkers.ts", import.meta.url).pathname; +const H = homedir(); +const P = join(H, ".claude/LIFEOS/USER/PRINCIPAL/PRINCIPAL_MEMORY.md"); +const D = join(H, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT/DA_MEMORY.md"); +mkdirSync(join(H, ".claude/LIFEOS/USER/PRINCIPAL"), { recursive: true }); +mkdirSync(join(H, ".claude/LIFEOS/USER/DIGITAL_ASSISTANT"), { recursive: true }); +mkdirSync(join(H, ".claude/LIFEOS/MEMORY/OBSERVABILITY"), { recursive: true }); + +const FM = `---\nschema_version: 1\ncap_entries: 48\ncap_chars_per_entry: 256\nlast_updated: 2026-01-01\nlast_updated_by: bootstrap-template\nconvention: pai-freshness-v1\n---\n\n# Hot-Layer Memory\n\n`; +// Corrupted principal: no BEGIN, two stray ENDs, two orphaned entries. +writeFileSync(P, FM + `PREFERENCE: orphaned alpha ~explicit\n\nPREFERENCE: orphaned beta ~inferred\n\n`); +// Canonical DA: must be left untouched. +const CANON = FM + `\nRULE: healthy entry ~explicit\n\n`; +writeFileSync(D, CANON); + +const BEGIN = "", END = ""; +const readerSees = (raw: string): string[] => { + const s = raw.indexOf(BEGIN), e = raw.indexOf(END); + if (s === -1 || e === -1 || e < s) return []; + return raw.slice(s + BEGIN.length, e).trim().split("\n").map(l => l.trim()).filter(Boolean); +}; +const run = (dry: boolean) => Bun.spawnSync(["bun", MIG, ...(dry ? ["--dry-run"] : [])], { env: { ...process.env, HOME: H } }); + +let fails = 0; +const check = (n: string, c: boolean, d = "") => { console.log(`${c ? "✅" : "❌"} ${n}${c ? "" : " — " + d}`); if (!c) fails++; }; + +// 1) dry-run: reports but writes nothing +const dry = run(true); const dryOut = dry.stdout.toString(); +check("dry-run flags principal as would-repair", /would-repair.*PRINCIPAL_MEMORY/s.test(dryOut) || /would-repair/.test(dryOut), dryOut); +check("dry-run leaves canonical DA as skip-ok", /skip-ok.*DA_MEMORY|DA_MEMORY.*already/s.test(dryOut), dryOut); +check("dry-run did NOT modify the corrupted file", readerSees(readFileSync(P, "utf8")).length === 0); + +// 2) real run: heals principal, leaves DA +const real = run(false); const realOut = real.stdout.toString(); +check("real run reports principal repaired", /repaired.*PRINCIPAL_MEMORY|PRINCIPAL_MEMORY.*recovered/s.test(realOut), realOut); +const healed = readFileSync(P, "utf8"); +const seen = readerSees(healed); +check("healed principal: reader sees orphaned alpha", seen.includes("PREFERENCE: orphaned alpha ~explicit"), seen.join(" | ")); +check("healed principal: reader sees orphaned beta", seen.includes("PREFERENCE: orphaned beta ~inferred"), seen.join(" | ")); +check("healed principal: exactly one BEGIN + one END", (healed.match(/BEGIN ENTRIES/g)||[]).length === 1 && (healed.match(/END ENTRIES/g)||[]).length === 1); +check("canonical DA left byte-identical", readFileSync(D, "utf8") === CANON); + +// 3) idempotent: second run repairs nothing +const again = run(false).stdout.toString(); +check("second run is idempotent (0 repaired)", /^0 repaired|\n0 repaired/m.test(again) || again.includes("0 repaired"), again); + +console.log(fails === 0 ? "\n🎉 MIGRATION ALL PASS" : `\n💥 ${fails} FAILURE(S)`); +process.exit(fails === 0 ? 0 : 1); diff --git a/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.ts b/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.ts new file mode 100644 index 000000000..f18f829e3 --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MigrateMemoryMarkers.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun +/** + * MigrateMemoryMarkers — one-time repair for hot-layer memory files scaffolded + * before the entry markers existed, or corrupted by the pre-fix writer (which + * re-added END but never BEGIN, leaving stray END markers and no readable + * entry block). Existing installs would otherwise stay silently empty to the + * reader until the next reviewer write; this heals them immediately on update. + * + * Idempotent: files already in canonical … + * shape are left untouched. Reuses the fixed read()/setEntries() path, so entry + * recovery is exactly the behavior covered by MemoryWriter.markers.test.ts. + * + * Usage: + * bun ~/.claude/LIFEOS/TOOLS/MigrateMemoryMarkers.ts + * bun ~/.claude/LIFEOS/TOOLS/MigrateMemoryMarkers.ts --dry-run + */ +import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { PRINCIPAL_MEMORY_PATH, DA_MEMORY_PATH } from "./MemoryTypes"; +import { read, setEntries } from "./MemoryWriter"; + +const BEGIN_MARKER = ""; +const END_MARKER = ""; +const TARGETS = [PRINCIPAL_MEMORY_PATH, DA_MEMORY_PATH]; + +/** Canonical = exactly one BEGIN and one END, in order. Anything else needs repair. */ +function isCanonical(content: string): boolean { + const begins = (content.match(//g) || []).length; + const ends = (content.match(//g) || []).length; + return begins === 1 && ends === 1 && content.indexOf(BEGIN_MARKER) < content.indexOf(END_MARKER); +} + +interface Result { path: string; action: string; detail: string; } + +function repair(path: string, dryRun: boolean): Result { + if (!existsSync(path)) { + return { path, action: "skip-missing", detail: "no file (a fresh install scaffolds it correctly)" }; + } + const content = readFileSync(path, "utf8"); + if (isCanonical(content)) { + return { path, action: "skip-ok", detail: "already has a canonical BEGIN…END block" }; + } + // read() (post-fix) recovers orphaned entries from the broken body. + const r = read(path); + if (!("entries" in r)) { + return { path, action: "error", detail: `read failed: ${(r as any).message ?? "unknown"}` }; + } + const recovered = r.entries; + if (dryRun) { + return { path, action: "would-repair", detail: `${recovered.length} entr${recovered.length === 1 ? "y" : "ies"} recoverable` }; + } + // Back up the pre-repair file next to it (repo convention: Backups/). + const backupDir = join(dirname(path), "Backups"); + if (!existsSync(backupDir)) mkdirSync(backupDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + writeFileSync(join(backupDir, `${basename(path, ".md")}-pre-marker-migration-${stamp}.md`), content); + + // setEntries rewrites a canonical block (serializeFile re-emits BEGIN…END). + const res = setEntries(path, recovered, { updatedBy: "MigrateMemoryMarkers", allowDrastic: true }); + if (!res.ok) { + return { path, action: "error", detail: `setEntries failed: ${res.code} ${res.message}` }; + } + return { path, action: "repaired", detail: `recovered ${recovered.length} entr${recovered.length === 1 ? "y" : "ies"} into a canonical block` }; +} + +function main(): void { + const dryRun = process.argv.includes("--dry-run"); + const results = TARGETS.map((p) => repair(p, dryRun)); + console.log(`MigrateMemoryMarkers${dryRun ? " (dry-run)" : ""}`); + for (const res of results) { + console.log(` ${res.action.padEnd(14)} ${basename(dirname(res.path))}/${basename(res.path)} — ${res.detail}`); + } + const repaired = results.filter((r) => r.action === "repaired" || r.action === "would-repair").length; + const errored = results.filter((r) => r.action === "error").length; + console.log(`\n${repaired} repaired, ${results.length - repaired - errored} already-ok/skipped, ${errored} error(s).`); + process.exit(errored > 0 ? 1 : 0); +} + +main(); diff --git a/LifeOS/install/USER/DIGITAL_ASSISTANT/DA_MEMORY.md b/LifeOS/install/USER/DIGITAL_ASSISTANT/DA_MEMORY.md index 0487c3f38..3151c4316 100644 --- a/LifeOS/install/USER/DIGITAL_ASSISTANT/DA_MEMORY.md +++ b/LifeOS/install/USER/DIGITAL_ASSISTANT/DA_MEMORY.md @@ -12,3 +12,5 @@ convention: pai-freshness-v1 > 🎯 SAMPLE TEMPLATE — auto-curated hot-layer memory your DA keeps about how it works with you. Starts empty; the memory reviewer populates it over time. Nothing to fill in manually. Pulse's memory panel reads this file. + + diff --git a/LifeOS/install/USER/PRINCIPAL/PRINCIPAL_MEMORY.md b/LifeOS/install/USER/PRINCIPAL/PRINCIPAL_MEMORY.md index 015e76935..1d4b27b40 100644 --- a/LifeOS/install/USER/PRINCIPAL/PRINCIPAL_MEMORY.md +++ b/LifeOS/install/USER/PRINCIPAL/PRINCIPAL_MEMORY.md @@ -12,3 +12,5 @@ convention: pai-freshness-v1 > 🎯 SAMPLE TEMPLATE — auto-curated hot-layer memory about you. It starts empty and the memory reviewer writes durable facts here as you work with your DA. Nothing to fill in manually. Pulse's memory panel reads this file. + +