fix: apply configurable retention to LLM traces

.ci/observe-workspace.patch

@@ -0,0 +1,57 @@
+--- a/pnpm-lock.yaml
++++ b/pnpm-lock.yaml
+@@ -58,6 +58,25 @@
+       vite:
+         specifier: ^6.0.7
+         version: 6.4.1(@types/node@25.9.1)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)
++
++  apps/observe:
++    dependencies:
++      '@neutron-build/cli':
++        specifier: workspace:*
++        version: link:../../packages/neutron-cli
++      '@neutron-build/core':
++        specifier: workspace:*
++        version: link:../../packages/neutron
++      preact:
++        specifier: 10.25.4
++        version: 10.25.4
++      uplot:
++        specifier: 1.6.31
++        version: 1.6.31
++    devDependencies:
++      typescript:
++        specifier: 5.7.2
++        version: 5.7.2
+
+   apps/playground:
+     dependencies:
+@@ -4860,6 +4879,9 @@
+     peerDependencies:
+       browserslist: '>= 4.21.0'
+
++  uplot@1.6.31:
++    resolution: {integrity: sha512-sQZqSwVCbJGnFB4IQjQYopzj5CoTZJ4Br1fG/xdONimqgHmsacvCjNesdGDypNKFbrhLGIeshYhy89FxPF+H+w==}
++
+   use-callback-ref@1.3.3:
+     resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==}
+     version: 1.3.3
+@@ -9722,6 +9744,8 @@
+       escalade: 3.2.0
+       picocolors: 1.1.1
+
++  uplot@1.6.31: {}
++
+   use-callback-ref@1.3.3(@types/react@19.2.14)(react-compat-shim@file:compat-matrix/shims/react):
+     dependencies:
+       react: react-compat-shim@file:compat-matrix/shims/react
+--- a/pnpm-workspace.yaml
++++ b/pnpm-workspace.yaml
+@@ -6,7 +6,6 @@
+   # workspace graph became machine-dependent: a snapshot written locally listed
+   # observe-dashboard and CI could never reproduce it. The workspace definition
+   # has to describe the repository, not whatever happens to be on disk.
+-  - "!apps/observe"
+   - "examples/*"
+   # Ecosystem compatibility harness (A-007). Never published.
+   - "compat-matrix"

.ci/prepare-ui-workspace.mjs

@@ -0,0 +1,22 @@
+// Assemble the external Observe app in a disposable, pinned Neutron checkout.
+// The patch adds only this importer and its missing dependency to the lockfile.
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+if (!process.argv[2]) throw new Error('usage: node .ci/prepare-ui-workspace.mjs <isolated-neutron-checkout>');
+const workspace = resolve(process.argv[2], 'typescript');
+const app = resolve(workspace, 'apps/observe');
+if (existsSync(app)) throw new Error(`Refusing to overwrite an existing app: ${app}`);
+execFileSync('patch', ['--dry-run', '-p1', '-i', resolve(root, '.ci/observe-workspace.patch')], { cwd: workspace, stdio: 'inherit' });
+execFileSync('patch', ['-p1', '-i', resolve(root, '.ci/observe-workspace.patch')], { cwd: workspace, stdio: 'inherit' });
+mkdirSync(app, { recursive: true });
+const pkg = JSON.parse(readFileSync(resolve(root, 'ui/package.json'), 'utf8'));
+for (const [oldName, newName] of [['neutron', '@neutron-build/core'], ['neutron-cli', '@neutron-build/cli']]) {
+  pkg.dependencies[newName] = pkg.dependencies[oldName];
+  delete pkg.dependencies[oldName];
+}
+writeFileSync(resolve(app, 'package.json'), JSON.stringify(pkg, null, 2) + '\n');
+writeFileSync(resolve(app, 'neutron.config.ts'), readFileSync(resolve(root, 'ui/neutron.config.ts'), 'utf8').replace('"neutron"', '"@neutron-build/core"'));

.github/workflows/ci.yml

@@ -151,12 +151,17 @@ jobs:
           corepack enable
           corepack prepare "pnpm@$version" --activate
           pnpm --version
+      - name: Add Observe to the isolated pinned workspace
+        run: node .ci/prepare-ui-workspace.mjs ../Neutron
       - name: Provision TS workspace
         run: pnpm -C ../Neutron/typescript install --frozen-lockfile
+      - name: Build framework and CLI from the pinned source
+        run: pnpm -C ../Neutron/typescript --filter '@neutron-build/cli...' run build
       - name: ui-sync (build + embed)
-        run: sh scripts/ui-sync.sh
+        run: bash scripts/ui-sync.sh
         env:
           OBSERVE_BIN: /tmp/observe-ui-sync
+          NEUTRON_ROOT: ${{ github.workspace }}/../Neutron
       - name: Embedded dist must be fresh
         run: git diff --exit-code -- cmd/observe/ui/dist
 
@@ -170,10 +175,9 @@ jobs:
   # 027) was fixed upstream on 2026-09-18 (Neutron 6286531a) and
   # repo-built engines pass the full migration ladder again — the
   # lease-capability job below builds from the pin and relies on it. This
-  # job keeps the published v0.1.8 fixture for compatibility coverage;
-  # its known flake (intermittent committed-upsert loss on accumulated
-  # data, also fixed in the tree) is why the capability job exists
-  # separately rather than replacing this one.
+  # Supported-release coverage now uses v1.1.1 on both architectures.
+  # v0.1.8 no longer satisfies the full suite (leases, parameterized paging,
+  # and committed upserts); its failing run is recorded in AUDIT_OPEN.md.
   # X01: the vendor/ tree is what actually compiles, but go.mod's replace
   # points at a LOCAL Neutron checkout — nothing proved vendor/ matches the
   # pinned submodule until this check. Drift here means the shipped build
@@ -244,7 +248,7 @@ jobs:
         run: |
           OBSERVE_NUCLEUS_URL='postgres://nucleus@127.0.0.1:55432/observe?sslmode=disable' \
           OBSERVE_ADMIN_PASSWORD=ci-migration-boot \
-          OBSERVE_JWT_SECRET=ci-migration-secret \
+          OBSERVE_JWT_SECRET=ci-only-migration-secret-at-least-32-characters \
           /tmp/observe &
           OBS_PID=$!
           for i in $(seq 1 90); do
@@ -277,7 +281,7 @@ jobs:
             -e NUCLEUS_ALLOW_NO_AUTH=1 \
             -e NUCLEUS_ALLOW_INSECURE_CLUSTER=1 \
             -e NUCLEUS_ALLOW_INSECURE_REPLICATION=1 \
-            ghcr.io/neutron-build/nucleus:v0.1.8 \
+            ghcr.io/neutron-build/nucleus:v1.1.1 \
             start --host 0.0.0.0 --port 5432 --cluster-port 5433 --data /data --max-memory 512
           for i in $(seq 1 60); do
             if (exec 3<>/dev/tcp/127.0.0.1/55432) 2>/dev/null; then exec 3>&-; exit 0; fi
@@ -292,7 +296,7 @@ jobs:
         run: |
           OBSERVE_NUCLEUS_URL='postgres://nucleus@127.0.0.1:55432/observe?sslmode=disable' \
           OBSERVE_ADMIN_PASSWORD=ci-migration-boot \
-          OBSERVE_JWT_SECRET=ci-migration-secret \
+          OBSERVE_JWT_SECRET=ci-only-migration-secret-at-least-32-characters \
           /tmp/observe &
           OBS_PID=$!
           for i in $(seq 1 90); do

AUDIT_OPEN.md

@@ -14,6 +14,17 @@ report lives outside the repo) — remediation record below. Round 4: audit
 passes 1-5) are closed history; their one surviving item is folded into F16
 below.
 
+## L10 (2026-09-27) — Fixed in source: LLM traces omitted from retention
+
+LLM traces, including prompt/completion payloads, were never expired by the
+scheduled cleanup. Added a validated OBSERVE_LLM_RETENTION_DAYS setting (30-day
+default) and included the table in the existing bounded cleanup path. Historical
+model-price catalog entries remain independent of trace retention. Regression
+coverage checks expiry and preservation against a real Nucleus fixture.
+This fixes unbounded logical trace retention; it does not claim SQL DELETE
+shrinks files or establish this table as the cause of a past host incident.
+Deployment and sustained storage measurements remain separate acceptance gates.
+
 ## L9 (2026-09-24) - P1 - Open: live llm_traces unreadable after migration 054; ALTER-ADD migrations
 
 Found deploying `d4bcbe6` (the zombie-reaping image fix) to infra-home, the
@@ -2361,3 +2372,24 @@ Open items this slice deliberately leaves (programme O11 tails):
 - Package publication prep is owner-controlled per the O11 spec: npm
   tarball dry-runs (browser, sentry-shim), PyPI sdist/wheel build,
   Go module tagging — none attempted from the lane.
+
+## 2026-09-27 — reproducible UI and supported-engine CI
+
+CI run 36336753676 proved the complete suite on Nucleus v1.1.1 arm64
+and the source-pinned snapshot-lease suite. Its v0.1.8 amd64 job failed
+lease expectations, parameterized paging, and committed-upsert assertions.
+That obsolete release is not a passing compatibility target. The amd64
+gate now uses v1.1.1, matching the existing arm64 gate; no tests are skipped
+or weakened. The failed historical run remains the compatibility receipt.
+
+The pinned Neutron repository deliberately excludes the external Observe app.
+CI now assembles it in its isolated checkout using a checked-in lockfile patch,
+then installs frozen dependencies and builds the framework/CLI before ui-sync.
+No developer-local canonical app or uncommitted shared Neutron changes are used.
+Embedded-asset freshness remains a required check.
+
+A clean rebuild matched every application asset but exposed upstream static
+adapter wall-clock timestamps. ui-sync normalizes those metadata timestamps
+and their compressed policy/size counts before embedding; all application
+assets remain byte-checked. Upstream report: private UPSTREAM_BUGS.md entry
+2026-09-27. Two local builds and normalization idempotence verified.

README.md

@@ -384,6 +384,7 @@ observeErrors.addBreadcrumb({ type: "user", category: "click", message: "Button"
 | `OBSERVE_REQUIRE_WAL` | (unset) | Set to `true` (or `1`) to refuse to start when WAL-backed ingestion durability is unavailable, instead of degrading to memory-only. |
 | `OBSERVE_RAW_RETENTION_DAYS` | `30` | Raw event retention. Also the window over which visitor counts are exact from raw events; past it they are counted from the `sessions` table (90 days), and past both the dashboard says which window the figure covers. |
 | `OBSERVE_HOURLY_RETENTION_DAYS` | `365` | Hourly rollup retention. |
+| `OBSERVE_LLM_RETENTION_DAYS` | `30` | LLM trace retention, including stored prompts/completions. Must be at least 1. Cleanup runs daily; historical model-price catalog entries are retained. Logical expiry does not guarantee database files immediately shrink. |
 | `OBSERVE_LOG_ROUTES` | `0` | Set to `1` to print route table at boot. |
 | `OBSERVE_SMTP_HOST` | | SMTP server for email reports. |
 | `OBSERVE_SMTP_PORT` | `587` | SMTP port. |

cmd/observe/main.go

@@ -372,7 +372,7 @@ func main() {
 	// same numbers rather than a constant of its own. The ledger policies
 	// (O01 slice 5) ride the same job: dedupe ledgers expire with the data
 	// they dedupe, processed outbox intents prune, dead letters never do.
-	retentionPolicies := append(jobs.DefaultPolicies(cfg.RawRetentionDays, cfg.HourlyRetentionDays),
+	retentionPolicies := append(jobs.DefaultPolicies(cfg.RawRetentionDays, cfg.HourlyRetentionDays, cfg.LLMRetentionDays),
 		jobs.DefaultLedgerPolicies(cfg.ErrorInboxRetentionDays, cfg.ReplayBatchesRetentionDays, cfg.DerivedOutboxRetentionDays)...)
 
 	// Stats service

internal/config/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	// Retention
 	RawRetentionDays    int
 	HourlyRetentionDays int
+	LLMRetentionDays    int
 	// Ledger retention (O01 ADR 5.8, decided 2026-09-23). Expiry means a
 	// retried record is processed as new — no silent infinite exactly-once
 	// promise beyond the window.
@@ -95,6 +96,7 @@ func Load() Config {
 	set("OBSERVE_FLUSH_SIZE", 500, &c.FlushSize)
 	set("OBSERVE_RAW_RETENTION_DAYS", 30, &c.RawRetentionDays)
 	set("OBSERVE_HOURLY_RETENTION_DAYS", 365, &c.HourlyRetentionDays)
+	set("OBSERVE_LLM_RETENTION_DAYS", 30, &c.LLMRetentionDays)
 	// O01 slice 5 ledger retention (decided defaults 2026-09-23): the
 	// error_inbox and replay_batches ledgers must not outlive the data they
 	// dedupe (error_events 180d, replay_sessions 14d); derived_outbox
@@ -168,6 +170,9 @@ func (c Config) Validate() error {
 	if c.HourlyRetentionDays < 1 {
 		return fmt.Errorf("OBSERVE_HOURLY_RETENTION_DAYS must be >= 1, got %d", c.HourlyRetentionDays)
 	}
+	if c.LLMRetentionDays < 1 {
+		return fmt.Errorf("OBSERVE_LLM_RETENTION_DAYS must be >= 1, got %d", c.LLMRetentionDays)
+	}
 	if c.ErrorInboxRetentionDays < 1 {
 		return fmt.Errorf("OBSERVE_ERROR_INBOX_RETENTION_DAYS must be >= 1, got %d", c.ErrorInboxRetentionDays)
 	}

internal/config/config_test.go

@@ -24,6 +24,7 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) {
 		{"negative interval", func(c *Config) { c.FlushInterval = -time.Second }, "OBSERVE_FLUSH_INTERVAL_MS"},
 		{"huge interval", func(c *Config) { c.FlushInterval = 10 * time.Minute }, "OBSERVE_FLUSH_INTERVAL_MS"},
 		{"zero rate limit", func(c *Config) { c.RateLimit = 0 }, "OBSERVE_RATE_LIMIT"},
+		{"zero LLM retention", func(c *Config) { c.LLMRetentionDays = 0 }, "OBSERVE_LLM_RETENTION_DAYS"},
 		{"zero retention", func(c *Config) { c.RawRetentionDays = 0 }, "OBSERVE_RAW_RETENTION_DAYS"},
 		{"malformed integer", func(c *Config) {
 			c.parseErr = fmt.Errorf("OBSERVE_BUFFER_SIZE must be an integer, got %q", "big")
@@ -32,7 +33,7 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) {
 	for _, tc := range cases {
 		c := Config{
 			BufferSize: 1000, FlushSize: 100, FlushInterval: time.Second,
-			RateLimit: 100, RawRetentionDays: 30, HourlyRetentionDays: 365,
+			RateLimit: 100, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30,
 			ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7,
 		}
 		tc.mut(&c)
@@ -46,10 +47,23 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) {
 func TestValidateAcceptsDefaults(t *testing.T) {
 	c := Config{
 		BufferSize: 100_000, FlushSize: 500, FlushInterval: 2 * time.Second,
-		RateLimit: 1000, RawRetentionDays: 30, HourlyRetentionDays: 365,
+		RateLimit: 1000, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30,
 		ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7,
 	}
 	if err := c.Validate(); err != nil {
 		t.Fatalf("default-shaped config must validate: %v", err)
 	}
 }
+
+func TestLoadLLMRetentionWindow(t *testing.T) {
+	t.Setenv("OBSERVE_LLM_RETENTION_DAYS", "7")
+	c := Load()
+	if c.LLMRetentionDays != 7 {
+		t.Fatalf("LLM retention=%d, want 7", c.LLMRetentionDays)
+	}
+	t.Setenv("OBSERVE_LLM_RETENTION_DAYS", "invalid")
+	c = Load()
+	if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "OBSERVE_LLM_RETENTION_DAYS") {
+		t.Fatalf("invalid LLM duration must fail validation, got %v", err)
+	}
+}

internal/ingest/buffer_test.go

@@ -195,14 +195,15 @@ func TestBuffer_FlushFailureSurvivesRetryAndRestart(t *testing.T) {
 // wakeup is one bounded attempt, so Stop completes even with a dead DB.
 func TestBuffer_FlushFailureDoesNotLivelock(t *testing.T) {
 	buf := NewBuffer(nil, 8, 2, time.Hour, slog.New(slog.NewTextHandler(io.Discard, nil)))
-	buf.Start()
-	defer buf.Stop()
-
+	// Seed before starting: after its first flush panics, the worker correctly
+	// refuses admission. Racing those later pushes tests scheduling, not Stop.
 	for i := 0; i < 4; i++ {
 		if err := buf.Push(ev(fmt.Sprintf("e%d", i))); err != nil {
 			t.Fatalf("push %d refused: %v", i, err)
 		}
 	}
+	buf.Start()
+	defer buf.Stop()
 	// nil db -> insertBatch panics inside Flush, recovered by the worker's
 	// panic guard, leaving the events requeued... in fact the panic aborts
 	// the whole worker goroutine (recover logs and exits the goroutine),

internal/jobs/retention.go

@@ -47,17 +47,17 @@ type RetentionService struct {
 	policies []RetentionPolicy
 }
 
-// DefaultPolicies returns the out-of-box retention policies. Called with the two
-// legacy env-configured durations for raw events + hourly rollups so existing
-// deployments keep their behavior.
-func DefaultPolicies(rawDays, hourlyDays int) []RetentionPolicy {
+// DefaultPolicies returns the out-of-box retention policies, using configured
+// durations for raw events, hourly rollups and LLM traces.
+func DefaultPolicies(rawDays, hourlyDays, llmDays int) []RetentionPolicy {
 	return []RetentionPolicy{
 		{Table: "events", Column: "timestamp", Days: rawDays},
 		{Table: "events_recent", Column: "timestamp", Days: 7},
 		{Table: "stats_hourly", Column: "ts_bucket", Days: hourlyDays},
 		{Table: "sessions", Column: "last_ts", Days: 90},
 		{Table: "error_events", Column: "timestamp", Days: 180},
 		{Table: "logs", Column: "timestamp", Days: 30},
+		{Table: "llm_traces", Column: "timestamp", Days: llmDays},
 		{Table: "spans", Column: "start_time", Days: 14},
 		{Table: "service_stats", Column: "ts_bucket", Days: 30},
 		{Table: "replay_sessions", Column: "start_time", Days: 14},
@@ -111,7 +111,7 @@ func PolicyDays(policies []RetentionPolicy, table string) int {
 
 // NewRetentionService keeps the old two-arg constructor for backwards compat.
 func NewRetentionService(db *nucleus.Client, logger *slog.Logger, rawDays, hourlyDays int) *RetentionService {
-	return NewRetentionServiceWithPolicies(db, logger, DefaultPolicies(rawDays, hourlyDays))
+	return NewRetentionServiceWithPolicies(db, logger, DefaultPolicies(rawDays, hourlyDays, 30))
 }
 
 // NewRetentionServiceWithPolicies allows callers to supply a fully custom policy set.