diff --git a/.ci/observe-workspace.patch b/.ci/observe-workspace.patch new file mode 100644 index 0000000..4634728 --- /dev/null +++ b/.ci/observe-workspace.patch @@ -0,0 +1,57 @@ +--- a/pnpm-lock.yaml ++++ b/pnpm-lock.yaml +@@ -58,6 +58,25 @@ + vite: + specifier: ^6.0.7 + version: 6.4.1(@types/node@25.9.1)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) ++ ++ apps/observe: ++ dependencies: ++ '@neutron-build/cli': ++ specifier: workspace:* ++ version: link:../../packages/neutron-cli ++ '@neutron-build/core': ++ specifier: workspace:* ++ version: link:../../packages/neutron ++ preact: ++ specifier: 10.25.4 ++ version: 10.25.4 ++ uplot: ++ specifier: 1.6.31 ++ version: 1.6.31 ++ devDependencies: ++ typescript: ++ specifier: 5.7.2 ++ version: 5.7.2 + + apps/playground: + dependencies: +@@ -4860,6 +4879,9 @@ + peerDependencies: + browserslist: '>= 4.21.0' + ++ uplot@1.6.31: ++ resolution: {integrity: sha512-sQZqSwVCbJGnFB4IQjQYopzj5CoTZJ4Br1fG/xdONimqgHmsacvCjNesdGDypNKFbrhLGIeshYhy89FxPF+H+w==} ++ + use-callback-ref@1.3.3: + resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} + version: 1.3.3 +@@ -9722,6 +9744,8 @@ + escalade: 3.2.0 + picocolors: 1.1.1 + ++ uplot@1.6.31: {} ++ + use-callback-ref@1.3.3(@types/react@19.2.14)(react-compat-shim@file:compat-matrix/shims/react): + dependencies: + react: react-compat-shim@file:compat-matrix/shims/react +--- a/pnpm-workspace.yaml ++++ b/pnpm-workspace.yaml +@@ -6,7 +6,6 @@ + # workspace graph became machine-dependent: a snapshot written locally listed + # observe-dashboard and CI could never reproduce it. The workspace definition + # has to describe the repository, not whatever happens to be on disk. +- - "!apps/observe" + - "examples/*" + # Ecosystem compatibility harness (A-007). Never published. + - "compat-matrix" diff --git a/.ci/prepare-ui-workspace.mjs b/.ci/prepare-ui-workspace.mjs new file mode 100644 index 0000000..515410e --- /dev/null +++ b/.ci/prepare-ui-workspace.mjs @@ -0,0 +1,22 @@ +// Assemble the external Observe app in a disposable, pinned Neutron checkout. +// The patch adds only this importer and its missing dependency to the lockfile. +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +if (!process.argv[2]) throw new Error('usage: node .ci/prepare-ui-workspace.mjs '); +const workspace = resolve(process.argv[2], 'typescript'); +const app = resolve(workspace, 'apps/observe'); +if (existsSync(app)) throw new Error(`Refusing to overwrite an existing app: ${app}`); +execFileSync('patch', ['--dry-run', '-p1', '-i', resolve(root, '.ci/observe-workspace.patch')], { cwd: workspace, stdio: 'inherit' }); +execFileSync('patch', ['-p1', '-i', resolve(root, '.ci/observe-workspace.patch')], { cwd: workspace, stdio: 'inherit' }); +mkdirSync(app, { recursive: true }); +const pkg = JSON.parse(readFileSync(resolve(root, 'ui/package.json'), 'utf8')); +for (const [oldName, newName] of [['neutron', '@neutron-build/core'], ['neutron-cli', '@neutron-build/cli']]) { + pkg.dependencies[newName] = pkg.dependencies[oldName]; + delete pkg.dependencies[oldName]; +} +writeFileSync(resolve(app, 'package.json'), JSON.stringify(pkg, null, 2) + '\n'); +writeFileSync(resolve(app, 'neutron.config.ts'), readFileSync(resolve(root, 'ui/neutron.config.ts'), 'utf8').replace('"neutron"', '"@neutron-build/core"')); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61c0b1c..0e6c953 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,12 +151,17 @@ jobs: corepack enable corepack prepare "pnpm@$version" --activate pnpm --version + - name: Add Observe to the isolated pinned workspace + run: node .ci/prepare-ui-workspace.mjs ../Neutron - name: Provision TS workspace run: pnpm -C ../Neutron/typescript install --frozen-lockfile + - name: Build framework and CLI from the pinned source + run: pnpm -C ../Neutron/typescript --filter '@neutron-build/cli...' run build - name: ui-sync (build + embed) - run: sh scripts/ui-sync.sh + run: bash scripts/ui-sync.sh env: OBSERVE_BIN: /tmp/observe-ui-sync + NEUTRON_ROOT: ${{ github.workspace }}/../Neutron - name: Embedded dist must be fresh run: git diff --exit-code -- cmd/observe/ui/dist @@ -170,10 +175,9 @@ jobs: # 027) was fixed upstream on 2026-09-18 (Neutron 6286531a) and # repo-built engines pass the full migration ladder again — the # lease-capability job below builds from the pin and relies on it. This - # job keeps the published v0.1.8 fixture for compatibility coverage; - # its known flake (intermittent committed-upsert loss on accumulated - # data, also fixed in the tree) is why the capability job exists - # separately rather than replacing this one. + # Supported-release coverage now uses v1.1.1 on both architectures. + # v0.1.8 no longer satisfies the full suite (leases, parameterized paging, + # and committed upserts); its failing run is recorded in AUDIT_OPEN.md. # X01: the vendor/ tree is what actually compiles, but go.mod's replace # points at a LOCAL Neutron checkout — nothing proved vendor/ matches the # pinned submodule until this check. Drift here means the shipped build @@ -244,7 +248,7 @@ jobs: run: | OBSERVE_NUCLEUS_URL='postgres://nucleus@127.0.0.1:55432/observe?sslmode=disable' \ OBSERVE_ADMIN_PASSWORD=ci-migration-boot \ - OBSERVE_JWT_SECRET=ci-migration-secret \ + OBSERVE_JWT_SECRET=ci-only-migration-secret-at-least-32-characters \ /tmp/observe & OBS_PID=$! for i in $(seq 1 90); do @@ -277,7 +281,7 @@ jobs: -e NUCLEUS_ALLOW_NO_AUTH=1 \ -e NUCLEUS_ALLOW_INSECURE_CLUSTER=1 \ -e NUCLEUS_ALLOW_INSECURE_REPLICATION=1 \ - ghcr.io/neutron-build/nucleus:v0.1.8 \ + ghcr.io/neutron-build/nucleus:v1.1.1 \ start --host 0.0.0.0 --port 5432 --cluster-port 5433 --data /data --max-memory 512 for i in $(seq 1 60); do if (exec 3<>/dev/tcp/127.0.0.1/55432) 2>/dev/null; then exec 3>&-; exit 0; fi @@ -292,7 +296,7 @@ jobs: run: | OBSERVE_NUCLEUS_URL='postgres://nucleus@127.0.0.1:55432/observe?sslmode=disable' \ OBSERVE_ADMIN_PASSWORD=ci-migration-boot \ - OBSERVE_JWT_SECRET=ci-migration-secret \ + OBSERVE_JWT_SECRET=ci-only-migration-secret-at-least-32-characters \ /tmp/observe & OBS_PID=$! for i in $(seq 1 90); do diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index f67f77f..78fc56f 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -14,6 +14,17 @@ report lives outside the repo) — remediation record below. Round 4: audit passes 1-5) are closed history; their one surviving item is folded into F16 below. +## L10 (2026-09-27) — Fixed in source: LLM traces omitted from retention + +LLM traces, including prompt/completion payloads, were never expired by the +scheduled cleanup. Added a validated OBSERVE_LLM_RETENTION_DAYS setting (30-day +default) and included the table in the existing bounded cleanup path. Historical +model-price catalog entries remain independent of trace retention. Regression +coverage checks expiry and preservation against a real Nucleus fixture. +This fixes unbounded logical trace retention; it does not claim SQL DELETE +shrinks files or establish this table as the cause of a past host incident. +Deployment and sustained storage measurements remain separate acceptance gates. + ## L9 (2026-09-24) - P1 - Open: live llm_traces unreadable after migration 054; ALTER-ADD migrations Found deploying `d4bcbe6` (the zombie-reaping image fix) to infra-home, the @@ -2361,3 +2372,24 @@ Open items this slice deliberately leaves (programme O11 tails): - Package publication prep is owner-controlled per the O11 spec: npm tarball dry-runs (browser, sentry-shim), PyPI sdist/wheel build, Go module tagging — none attempted from the lane. + +## 2026-09-27 — reproducible UI and supported-engine CI + +CI run 36336753676 proved the complete suite on Nucleus v1.1.1 arm64 +and the source-pinned snapshot-lease suite. Its v0.1.8 amd64 job failed +lease expectations, parameterized paging, and committed-upsert assertions. +That obsolete release is not a passing compatibility target. The amd64 +gate now uses v1.1.1, matching the existing arm64 gate; no tests are skipped +or weakened. The failed historical run remains the compatibility receipt. + +The pinned Neutron repository deliberately excludes the external Observe app. +CI now assembles it in its isolated checkout using a checked-in lockfile patch, +then installs frozen dependencies and builds the framework/CLI before ui-sync. +No developer-local canonical app or uncommitted shared Neutron changes are used. +Embedded-asset freshness remains a required check. + +A clean rebuild matched every application asset but exposed upstream static +adapter wall-clock timestamps. ui-sync normalizes those metadata timestamps +and their compressed policy/size counts before embedding; all application +assets remain byte-checked. Upstream report: private UPSTREAM_BUGS.md entry +2026-09-27. Two local builds and normalization idempotence verified. diff --git a/README.md b/README.md index 60e4da4..08fdd37 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,7 @@ observeErrors.addBreadcrumb({ type: "user", category: "click", message: "Button" | `OBSERVE_REQUIRE_WAL` | (unset) | Set to `true` (or `1`) to refuse to start when WAL-backed ingestion durability is unavailable, instead of degrading to memory-only. | | `OBSERVE_RAW_RETENTION_DAYS` | `30` | Raw event retention. Also the window over which visitor counts are exact from raw events; past it they are counted from the `sessions` table (90 days), and past both the dashboard says which window the figure covers. | | `OBSERVE_HOURLY_RETENTION_DAYS` | `365` | Hourly rollup retention. | +| `OBSERVE_LLM_RETENTION_DAYS` | `30` | LLM trace retention, including stored prompts/completions. Must be at least 1. Cleanup runs daily; historical model-price catalog entries are retained. Logical expiry does not guarantee database files immediately shrink. | | `OBSERVE_LOG_ROUTES` | `0` | Set to `1` to print route table at boot. | | `OBSERVE_SMTP_HOST` | | SMTP server for email reports. | | `OBSERVE_SMTP_PORT` | `587` | SMTP port. | diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 74cda57..ab90b0e 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -372,7 +372,7 @@ func main() { // same numbers rather than a constant of its own. The ledger policies // (O01 slice 5) ride the same job: dedupe ledgers expire with the data // they dedupe, processed outbox intents prune, dead letters never do. - retentionPolicies := append(jobs.DefaultPolicies(cfg.RawRetentionDays, cfg.HourlyRetentionDays), + retentionPolicies := append(jobs.DefaultPolicies(cfg.RawRetentionDays, cfg.HourlyRetentionDays, cfg.LLMRetentionDays), jobs.DefaultLedgerPolicies(cfg.ErrorInboxRetentionDays, cfg.ReplayBatchesRetentionDays, cfg.DerivedOutboxRetentionDays)...) // Stats service diff --git a/cmd/observe/ui/dist/.neutron-adapter-static.json b/cmd/observe/ui/dist/.neutron-adapter-static.json index fd134a5..c8dfaf7 100644 --- a/cmd/observe/ui/dist/.neutron-adapter-static.json +++ b/cmd/observe/ui/dist/.neutron-adapter-static.json @@ -8,8 +8,7 @@ "compression": { "enabled": true, "files": 78, - "gzipBytesSaved": 642496, - "brotliBytesSaved": 703986 - }, - "generatedAt": "2026-09-24T01:37:32.305Z" + "gzipBytesSaved": 642486, + "brotliBytesSaved": 703972 + } } \ No newline at end of file diff --git a/cmd/observe/ui/dist/.neutron-static-policy.json b/cmd/observe/ui/dist/.neutron-static-policy.json index 7740da2..fcd6c72 100644 --- a/cmd/observe/ui/dist/.neutron-static-policy.json +++ b/cmd/observe/ui/dist/.neutron-static-policy.json @@ -1,5 +1,4 @@ { - "generatedAt": "2026-09-24T01:37:31.384Z", "headers": { "/*": { "X-Content-Type-Options": "nosniff", diff --git a/cmd/observe/ui/dist/.neutron-static-policy.json.br b/cmd/observe/ui/dist/.neutron-static-policy.json.br index 3056635..4584864 100644 --- a/cmd/observe/ui/dist/.neutron-static-policy.json.br +++ b/cmd/observe/ui/dist/.neutron-static-policy.json.br @@ -1,2 +1 @@ - v,f -ެVEED* o$H_+Xol&d-m>]Qj.ztz ZBlŰWLD;{#]VjANi,5Hd2;,o)xHm;>@6]>hS;8#r9dS~?S$M="EXmV5f \ No newline at end of file +S v,= 1, got %d", c.HourlyRetentionDays) } + if c.LLMRetentionDays < 1 { + return fmt.Errorf("OBSERVE_LLM_RETENTION_DAYS must be >= 1, got %d", c.LLMRetentionDays) + } if c.ErrorInboxRetentionDays < 1 { return fmt.Errorf("OBSERVE_ERROR_INBOX_RETENTION_DAYS must be >= 1, got %d", c.ErrorInboxRetentionDays) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0167f80..51b23a6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,7 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) { {"negative interval", func(c *Config) { c.FlushInterval = -time.Second }, "OBSERVE_FLUSH_INTERVAL_MS"}, {"huge interval", func(c *Config) { c.FlushInterval = 10 * time.Minute }, "OBSERVE_FLUSH_INTERVAL_MS"}, {"zero rate limit", func(c *Config) { c.RateLimit = 0 }, "OBSERVE_RATE_LIMIT"}, + {"zero LLM retention", func(c *Config) { c.LLMRetentionDays = 0 }, "OBSERVE_LLM_RETENTION_DAYS"}, {"zero retention", func(c *Config) { c.RawRetentionDays = 0 }, "OBSERVE_RAW_RETENTION_DAYS"}, {"malformed integer", func(c *Config) { c.parseErr = fmt.Errorf("OBSERVE_BUFFER_SIZE must be an integer, got %q", "big") @@ -32,7 +33,7 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) { for _, tc := range cases { c := Config{ BufferSize: 1000, FlushSize: 100, FlushInterval: time.Second, - RateLimit: 100, RawRetentionDays: 30, HourlyRetentionDays: 365, + RateLimit: 100, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30, ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7, } tc.mut(&c) @@ -46,10 +47,23 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) { func TestValidateAcceptsDefaults(t *testing.T) { c := Config{ BufferSize: 100_000, FlushSize: 500, FlushInterval: 2 * time.Second, - RateLimit: 1000, RawRetentionDays: 30, HourlyRetentionDays: 365, + RateLimit: 1000, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30, ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7, } if err := c.Validate(); err != nil { t.Fatalf("default-shaped config must validate: %v", err) } } + +func TestLoadLLMRetentionWindow(t *testing.T) { + t.Setenv("OBSERVE_LLM_RETENTION_DAYS", "7") + c := Load() + if c.LLMRetentionDays != 7 { + t.Fatalf("LLM retention=%d, want 7", c.LLMRetentionDays) + } + t.Setenv("OBSERVE_LLM_RETENTION_DAYS", "invalid") + c = Load() + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "OBSERVE_LLM_RETENTION_DAYS") { + t.Fatalf("invalid LLM duration must fail validation, got %v", err) + } +} diff --git a/internal/ingest/buffer_test.go b/internal/ingest/buffer_test.go index e738a81..09d9477 100644 --- a/internal/ingest/buffer_test.go +++ b/internal/ingest/buffer_test.go @@ -195,14 +195,15 @@ func TestBuffer_FlushFailureSurvivesRetryAndRestart(t *testing.T) { // wakeup is one bounded attempt, so Stop completes even with a dead DB. func TestBuffer_FlushFailureDoesNotLivelock(t *testing.T) { buf := NewBuffer(nil, 8, 2, time.Hour, slog.New(slog.NewTextHandler(io.Discard, nil))) - buf.Start() - defer buf.Stop() - + // Seed before starting: after its first flush panics, the worker correctly + // refuses admission. Racing those later pushes tests scheduling, not Stop. for i := 0; i < 4; i++ { if err := buf.Push(ev(fmt.Sprintf("e%d", i))); err != nil { t.Fatalf("push %d refused: %v", i, err) } } + buf.Start() + defer buf.Stop() // nil db -> insertBatch panics inside Flush, recovered by the worker's // panic guard, leaving the events requeued... in fact the panic aborts // the whole worker goroutine (recover logs and exits the goroutine), diff --git a/internal/jobs/retention.go b/internal/jobs/retention.go index a00e10b..0c040e0 100644 --- a/internal/jobs/retention.go +++ b/internal/jobs/retention.go @@ -47,10 +47,9 @@ type RetentionService struct { policies []RetentionPolicy } -// DefaultPolicies returns the out-of-box retention policies. Called with the two -// legacy env-configured durations for raw events + hourly rollups so existing -// deployments keep their behavior. -func DefaultPolicies(rawDays, hourlyDays int) []RetentionPolicy { +// DefaultPolicies returns the out-of-box retention policies, using configured +// durations for raw events, hourly rollups and LLM traces. +func DefaultPolicies(rawDays, hourlyDays, llmDays int) []RetentionPolicy { return []RetentionPolicy{ {Table: "events", Column: "timestamp", Days: rawDays}, {Table: "events_recent", Column: "timestamp", Days: 7}, @@ -58,6 +57,7 @@ func DefaultPolicies(rawDays, hourlyDays int) []RetentionPolicy { {Table: "sessions", Column: "last_ts", Days: 90}, {Table: "error_events", Column: "timestamp", Days: 180}, {Table: "logs", Column: "timestamp", Days: 30}, + {Table: "llm_traces", Column: "timestamp", Days: llmDays}, {Table: "spans", Column: "start_time", Days: 14}, {Table: "service_stats", Column: "ts_bucket", Days: 30}, {Table: "replay_sessions", Column: "start_time", Days: 14}, @@ -111,7 +111,7 @@ func PolicyDays(policies []RetentionPolicy, table string) int { // NewRetentionService keeps the old two-arg constructor for backwards compat. func NewRetentionService(db *nucleus.Client, logger *slog.Logger, rawDays, hourlyDays int) *RetentionService { - return NewRetentionServiceWithPolicies(db, logger, DefaultPolicies(rawDays, hourlyDays)) + return NewRetentionServiceWithPolicies(db, logger, DefaultPolicies(rawDays, hourlyDays, 30)) } // NewRetentionServiceWithPolicies allows callers to supply a fully custom policy set. diff --git a/internal/jobs/retention_test.go b/internal/jobs/retention_test.go index 6f3465c..e03a8ca 100644 --- a/internal/jobs/retention_test.go +++ b/internal/jobs/retention_test.go @@ -155,3 +155,57 @@ func TestRetentionPrunesProcessedOutboxIntentsNotDeadLetters(t *testing.T) { t.Errorf("dead-lettered intent was auto-pruned (got %d, want 1) — dead letters are the operator's queue", c) } } + +// Trace expiry must use the configured window without erasing the price catalog +// needed to explain historical cost estimates. +func TestLLMRetentionExpiresOldTracesPreservesRecentAndCatalog(t *testing.T) { + ctx, db, done := connect(t) + defer done() + site := "llm_ret_" + strconv.FormatInt(time.Now().UnixNano(), 36) + now := time.Now().UnixMilli() + const day = int64(24 * 60 * 60 * 1000) + for _, row := range []struct { + id string + ts int64 + }{{"old", now - 8*day}, {"recent", now - 6*day}} { + if _, err := db.SQL().Exec(ctx, `INSERT INTO llm_traces (trace_id, site_id, timestamp, prompt, completion) VALUES ($1,$2,$3,$4,$5)`, site+row.id, site, row.ts, "retained input", "retained output"); err != nil { + t.Fatal(err) + } + } + if _, err := db.SQL().Exec(ctx, `INSERT INTO llm_model_prices (model_prefix, created_at, valid_from) VALUES ($1,$2,$3)`, site, now-40*day, now-40*day); err != nil { + t.Fatal(err) + } + policies := DefaultPolicies(30, 365, 7) + var selected []RetentionPolicy + for _, p := range policies { + if p.Table == "llm_traces" { + selected = append(selected, p) + } + } + if len(selected) != 1 { + t.Fatalf("expected exactly one LLM policy, got %d", len(selected)) + } + svc := NewRetentionServiceWithPolicies(db, slog.New(slog.NewTextHandler(io.Discard, nil)), selected) + if err := svc.RunCleanup(ctx); err != nil { + t.Fatal(err) + } + for _, row := range []struct { + id string + want int64 + }{{"old", 0}, {"recent", 1}} { + got, err := nucleus.Query[countRow](ctx, db.SQL(), `SELECT COUNT(*) AS n FROM llm_traces WHERE trace_id=$1`, site+row.id) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].N != row.want { + t.Errorf("%s trace count=%v, want %d", row.id, got, row.want) + } + } + got, err := nucleus.Query[countRow](ctx, db.SQL(), `SELECT COUNT(*) AS n FROM llm_model_prices WHERE model_prefix=$1`, site) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].N != 1 { + t.Fatalf("price catalog was removed: %v", got) + } +} diff --git a/scripts/normalize-ui-metadata.mjs b/scripts/normalize-ui-metadata.mjs new file mode 100644 index 0000000..5a5fcb9 --- /dev/null +++ b/scripts/normalize-ui-metadata.mjs @@ -0,0 +1,44 @@ +// Embedding does not need build wall-clock timestamps. Neutron currently emits +// them unconditionally (reported in Teploy/_internal/UPSTREAM_BUGS.md). +// Preserve policy content and accurate compression counts while removing time. +import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { gzipSync, brotliCompressSync, constants } from 'node:zlib'; + +if (!process.argv[2]) throw new Error('usage: node scripts/normalize-ui-metadata.mjs '); +const dist = resolve(process.argv[2]); +const policyPath = resolve(dist, '.neutron-static-policy.json'); +const metaPath = resolve(dist, '.neutron-adapter-static.json'); +const original = readFileSync(policyPath); +const policy = JSON.parse(original); +const meta = JSON.parse(readFileSync(metaPath)); +delete policy.generatedAt; +delete meta.generatedAt; +const normalized = Buffer.from(JSON.stringify(policy, null, 2)); +for (const [suffix, field, compress] of [ + ['gz', 'gzipBytesSaved', b => gzipSync(b, { level: 9 })], + ['br', 'brotliBytesSaved', b => brotliCompressSync(b, { params: { [constants.BROTLI_PARAM_QUALITY]: 11 } })], +]) { + const previous = readFileSync(`${policyPath}.${suffix}`); + const compressed = compress(normalized); + meta.compression[field] += (normalized.length - compressed.length) - (original.length - previous.length); + writeFileSync(`${policyPath}.${suffix}`, compressed); +} +writeFileSync(policyPath, normalized); +writeFileSync(metaPath, JSON.stringify(meta, null, 2)); + +// gzip's OS header differs on macOS/Linux despite identical deflate payloads. +// RFC 1952 value 255 means unknown OS; this byte is outside the payload CRC. +function normalizeGzipHeaders(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = resolve(dir, entry.name); + if (entry.isDirectory()) normalizeGzipHeaders(path); + else if (entry.name.endsWith('.gz')) { + const bytes = readFileSync(path); + if (bytes[0] !== 0x1f || bytes[1] !== 0x8b || bytes[3] !== 0) throw new Error(`Unexpected gzip header: ${path}`); + bytes[9] = 255; + writeFileSync(path, bytes); + } + } +} +normalizeGzipHeaders(dist); diff --git a/scripts/ui-sync.sh b/scripts/ui-sync.sh index dc0f814..b7831cc 100755 --- a/scripts/ui-sync.sh +++ b/scripts/ui-sync.sh @@ -8,7 +8,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OBSERVE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" OBSERVE_UI_SRC="$OBSERVE_ROOT/ui/src" -CANONICAL_APP="$OBSERVE_ROOT/../../Neutron/typescript/apps/observe" +NEUTRON_ROOT="${NEUTRON_ROOT:-$OBSERVE_ROOT/../../Neutron}" +CANONICAL_APP="$NEUTRON_ROOT/typescript/apps/observe" CANONICAL_SRC="$CANONICAL_APP/src" EMBED_DIST="$OBSERVE_ROOT/cmd/observe/ui/dist" BIN_OUT="${OBSERVE_BIN:-/tmp/obs-launch/observe}" @@ -58,6 +59,8 @@ find "$CANONICAL_SRC" -type f \( -name '*.ts' -o -name '*.tsx' \) -print0 \ log "pnpm build (canonical)" pnpm -C "$CANONICAL_APP" build +node "$SCRIPT_DIR/normalize-ui-metadata.mjs" "$CANONICAL_APP/dist" + log "replacing embedded dist" rm -rf "$EMBED_DIST" mkdir -p "$EMBED_DIST"