diff --git a/docs/STORAGE_BACKEND_MATRIX.md b/docs/STORAGE_BACKEND_MATRIX.md new file mode 100644 index 0000000..30870ec --- /dev/null +++ b/docs/STORAGE_BACKEND_MATRIX.md @@ -0,0 +1,63 @@ +# Storage & DB backend matrix + +A selection of storage / DB backends and their properties — the estate's own backends first, then the +external field for context. The dimensions are **the SocioProphet storage-standard properties**, not a +generic feature list: content-addressing, provenance, integrity-at-rest, fault tolerance, and +sovereignty are what the standard load-bears on. `(✓)` = supported via an adapter / optional dependency. + +> Measure it or don't claim it. Perf numbers for any row belong in a reproducible **EvaluationRecord** +> (SocioProphet/socioprophet-standards-storage `evaluation-record-standard.v1`), produced by +> `ts/src/storage-bench.ts` — never inline in this table. + +## Estate backends + +| Backend | Model | Content-addressed | Provenance-stamped | Integrity-at-rest | Fault tolerance | Sovereign / BYOS | In-memory | Distributed | Fail-closed verify | +|---|---|---|---|---|---|---|---|---|---| +| `InMemoryObjectBackend` | blob KV | ✓ (sha256 key) | (via store) | (via codex) | – | – | ✓ | – | ✓ (bench) | +| `CanonicalObjectStore` | content-addressed object store | ✓ | ✓ (`ObjectProvenance`) | ✓ (**codex-seal** at ingest) | (backend) | (backend) | (backend) | (backend) | ✓ (`verify → Syndrome`) | +| `S3ObjectBackend` (BYOS) | object store on S3/MinIO | ✓ | (via store) | (via codex) | ✓ (S3/MinIO) | ✓ (customer holds bytes) | – | ✓ | ✓ (bench) | +| `RocksDBBackend` | embedded LSM KV (AtomSpace WAL) | ✓ (atom handle) | ✓ (log entry) | ✓ (replay) | ✓ (WAL) | ✓ (local-first) | (cache) | – | ✓ | +| `HypercoreBackend` | signed append-only log | ✓ | ✓ (signed) | ✓ (merkle) | ✓ (replicated) | ✓ (per-writer keys) | – | ✓ (Autobase) | ✓ | +| `AtomSpace` (metagraph) | typed hypergraph, PLN truth | ✓ (structural hash) | ✓ (Values) | ✓ (codex sealer) | (backend) | ✓ | ✓ | (super-peer) | ✓ | +| `FederatedAtomSpace` | Autobase-merged sovereign logs | ✓ | ✓ (per-op writer+seq) | ✓ (causal-cut proof) | ✓ (multi-writer) | ✓ (federation) | – | ✓ | ✓ (proof withholds) | + +## External field (context) + +| Backend | Model | Content-addressed | Provenance | Fault tolerance | In-memory | Distributed | Language | +|---|---|---|---|---|---|---|---| +| RocksDB | embedded LSM KV | – | – | ✓ (WAL) | (cache) | – | C++ | +| Neo4j | property graph | – | – | ✓ | (cache) | ✓ | Java | +| Kùzu | embedded graph | – | – | ✓ | ✓ | – | C++ | +| Redis | in-memory KV | – | – | (✓) | ✓ | ✓ | C | +| S3 / MinIO | object store | (by key) | – | ✓ | – | ✓ | Go / — | +| Postgres | relational | – | – | ✓ | (cache) | (✓) | C | +| IPFS | content-addressed | ✓ | – | ✓ | – | ✓ | Go | + +## What the estate adds over the field + +The standard-aligned columns are exactly where the estate backends differ from the external field: +**content-addressing + provenance + integrity-at-rest (codex-seal) + causal-cut proof** are first-class, +not bolt-ons. A backend that returns the wrong bytes fast is a *failure* here — see the fail-closed +integrity check in `storage-bench.ts` (a corrupting/losing backend fails the run, proven both ways in +`storage-bench.test.ts`). + +## Running the benchmark + +```ts +import { InMemoryObjectBackend } from './object-store.js' +import { benchObjectBackend, toEvaluationRecord } from './storage-bench.js' + +const result = await benchObjectBackend(new InMemoryObjectBackend(), + { backend_id: 'in-memory', n_ops: 10_000, payload_bytes: 4096, seed: 42 }) + +// store as evidence — a number is only a claim if it is backed by a reproducible record. +const record = toEvaluationRecord(result, { + subject_ref: 'hellgraph:CanonicalObjectStore', + evaluation_track_ref: 'track:storage-backend-perf.v1', +}) +``` + +The `EvaluationRecord` conforms to SocioProphet/socioprophet-standards-storage +`standards/evaluation-record-standard.v1` (`subject_type: platform_capability`, +`attempt_mode: benchmark_run`), carrying the raw measurements as evidence so results are storable, +reviewable, and regression-checkable across epochs. diff --git a/ts/dist/index.d.mts b/ts/dist/index.d.mts index 3daa451..73307aa 100644 Binary files a/ts/dist/index.d.mts and b/ts/dist/index.d.mts differ diff --git a/ts/dist/index.d.ts b/ts/dist/index.d.ts index 3daa451..73307aa 100644 Binary files a/ts/dist/index.d.ts and b/ts/dist/index.d.ts differ diff --git a/ts/dist/index.js b/ts/dist/index.js index e258c09..b97460c 100644 Binary files a/ts/dist/index.js and b/ts/dist/index.js differ diff --git a/ts/dist/index.mjs b/ts/dist/index.mjs index c6fb5c2..8b13c56 100644 Binary files a/ts/dist/index.mjs and b/ts/dist/index.mjs differ diff --git a/ts/src/index.ts b/ts/src/index.ts index 388e1c8..19639b7 100644 --- a/ts/src/index.ts +++ b/ts/src/index.ts @@ -77,3 +77,4 @@ export * from './nlq' export * from './effect-request-data' export * from './vendor-graph' export * from './agent-graph-ingest' +export * from './storage-bench' diff --git a/ts/src/storage-bench.test.ts b/ts/src/storage-bench.test.ts new file mode 100644 index 0000000..335ea80 --- /dev/null +++ b/ts/src/storage-bench.test.ts @@ -0,0 +1,66 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { InMemoryObjectBackend, type ObjectBackend } from './object-store.js' +import { benchObjectBackend, toEvaluationRecord } from './storage-bench.js' + +const CFG = { backend_id: 'in-memory', n_ops: 64, payload_bytes: 256, seed: 42 } + +test('a correct backend passes with full integrity + populated latency/throughput', async () => { + const r = await benchObjectBackend(new InMemoryObjectBackend(), CFG) + assert.equal(r.result, 'pass') + assert.equal(r.integrity_verified, CFG.n_ops) + assert.equal(r.integrity_failures, 0) + const put = r.ops.find((o) => o.op === 'put')! + assert.equal(put.count, CFG.n_ops) + assert.ok(put.latency.p95_ms >= put.latency.p50_ms) // percentiles ordered + assert.ok(put.throughput_ops_s > 0 && put.throughput_mb_s > 0) +}) + +test('the run is deterministic for a fixed seed (reproducible — the standard requires it)', async () => { + const a = await benchObjectBackend(new InMemoryObjectBackend(), CFG) + const b = await benchObjectBackend(new InMemoryObjectBackend(), CFG) + assert.equal(a.integrity_verified, b.integrity_verified) + assert.equal(a.result, b.result) +}) + +// a fast liar: stores nothing and returns wrong bytes. It MUST fail the bench. +class CorruptingBackend implements ObjectBackend { + async put(): Promise {} + async get(): Promise { return Buffer.from('not the bytes you stored') } +} + +test('a backend that returns the wrong bytes FAILS (integrity is first-class, teeth both ways)', async () => { + const r = await benchObjectBackend(new CorruptingBackend(), CFG) + assert.equal(r.result, 'fail') + assert.ok(r.integrity_failures > 0) + assert.equal(r.integrity_verified, 0) +}) + +// a backend that loses data (get returns undefined) is also a failure, counted as a get failure. +class LosingBackend implements ObjectBackend { + async put(): Promise {} + async get(): Promise { return undefined } +} + +test('a backend that loses data FAILS (missing blob is not a silent pass)', async () => { + const r = await benchObjectBackend(new LosingBackend(), CFG) + assert.equal(r.result, 'fail') + const get = r.ops.find((o) => o.op === 'get')! + assert.equal(get.failures, CFG.n_ops) +}) + +test('toEvaluationRecord conforms to the storage standard shape', async () => { + const r = await benchObjectBackend(new InMemoryObjectBackend(), CFG) + const rec = toEvaluationRecord(r, { + subject_ref: 'hellgraph:InMemoryObjectBackend', + evaluation_track_ref: 'track:storage-backend-perf.v1', + }) + assert.equal(rec.subject_type, 'platform_capability') + assert.equal(rec.attempt_refs[0]!.attempt_mode, 'benchmark_run') + assert.ok(['pass', 'fail'].includes(rec.result)) + assert.equal(rec.review_state, 'draft') + for (const k of ['id', 'evaluation_track_ref', 'subject_ref', 'evidence_bundle_ref', 'created_at', 'updated_at']) + assert.ok((rec as Record)[k], `missing required field ${k}`) + // the raw, reproducible measurements are carried as evidence (no bare numbers). + assert.equal(rec.attempt_refs[0]!.measurements.integrity_verified, CFG.n_ops) +}) diff --git a/ts/src/storage-bench.ts b/ts/src/storage-bench.ts new file mode 100644 index 0000000..e31c317 --- /dev/null +++ b/ts/src/storage-bench.ts @@ -0,0 +1,195 @@ +/** + * storage-bench — a performance + integrity benchmark harness for storage / DB backends, + * aligned to the SocioProphet storage standard (evidence-first: a number is only a claim if it is + * backed by a reproducible record). + * + * It measures any `ObjectBackend` (InMemory / S3-BYOS / RocksDB / …) on the operations a + * content-addressed store actually performs — put, get, and verify (get + re-hash) — reporting + * latency percentiles and throughput. Integrity is a FIRST-CLASS result, not a footnote: if a + * round-tripped blob does not byte-match, or its content hash does not re-derive, the run is a + * FAILURE. A fast backend that returns the wrong bytes fails here; that is the point. + * + * `toEvaluationRecord` emits a record conforming to + * SocioProphet/socioprophet-standards-storage `standards/evaluation-record-standard.v1.md` + * (subject_type=platform_capability, attempt_mode=benchmark_run) so a result is storable, + * reviewable, and regression-checkable — never a bare number. + */ +import { createHash } from 'node:crypto' +import { performance } from 'node:perf_hooks' +import type { ObjectBackend } from './object-store.js' + +export interface StorageBenchConfig { + backend_id: string + n_ops: number + payload_bytes: number + /** deterministic seed so a run is reproducible (the storage standard requires it) */ + seed?: number +} + +export interface LatencyStats { + p50_ms: number + p95_ms: number + p99_ms: number + max_ms: number + mean_ms: number +} + +export interface OpResult { + op: 'put' | 'get' | 'verify' + count: number + failures: number + latency: LatencyStats + throughput_ops_s: number + throughput_mb_s: number +} + +export interface StorageBenchResult { + backend_id: string + config: StorageBenchConfig + ops: OpResult[] + integrity_verified: number + integrity_failures: number + /** fail-closed: any integrity failure (or op failure) makes the whole run a failure */ + result: 'pass' | 'fail' + started_at: string + finished_at: string +} + +const sha256 = (b: Buffer): string => createHash('sha256').update(b).digest('hex') + +/** Deterministic pseudo-random bytes from a seed, so a run reproduces exactly. */ +function seededPayload(seed: number, i: number, size: number): Buffer { + const out = Buffer.allocUnsafe(size) + let x = (seed ^ (i * 2654435761)) >>> 0 + for (let j = 0; j < size; j++) { + x ^= x << 13; x ^= x >>> 17; x ^= x << 5; x >>>= 0 + out[j] = x & 0xff + } + return out +} + +function stats(samples: number[]): LatencyStats { + if (!samples.length) return { p50_ms: 0, p95_ms: 0, p99_ms: 0, max_ms: 0, mean_ms: 0 } + const s = [...samples].sort((a, b) => a - b) + const q = (p: number) => s[Math.min(s.length - 1, Math.floor(p * (s.length - 1)))] + return { + p50_ms: q(0.5), p95_ms: q(0.95), p99_ms: q(0.99), max_ms: s[s.length - 1], + mean_ms: samples.reduce((a, b) => a + b, 0) / samples.length, + } +} + +function opResult(op: OpResult['op'], lat: number[], failures: number, bytesEach: number): OpResult { + const totalMs = lat.reduce((a, b) => a + b, 0) || 1 + return { + op, + count: lat.length, + failures, + latency: stats(lat), + throughput_ops_s: (lat.length / totalMs) * 1000, + throughput_mb_s: ((lat.length * bytesEach) / (1024 * 1024) / totalMs) * 1000, + } +} + +/** + * Benchmark an ObjectBackend over `n_ops` content-addressed blobs: put, then get, then verify + * (re-hash the retrieved bytes). Deterministic given `seed`. Fail-closed on any integrity mismatch. + */ +export async function benchObjectBackend( + backend: ObjectBackend, + config: StorageBenchConfig, +): Promise { + const seed = config.seed ?? 0x5010 + const started_at = new Date().toISOString() + const hashes: string[] = [] + const putLat: number[] = [] + const getLat: number[] = [] + const verLat: number[] = [] + let putFail = 0, getFail = 0 + let integrity_verified = 0, integrity_failures = 0 + + for (let i = 0; i < config.n_ops; i++) { + const bytes = seededPayload(seed, i, config.payload_bytes) + const hash = sha256(bytes) + hashes.push(hash) + const t0 = performance.now() + try { await backend.put(hash, bytes) } catch { putFail++ } + putLat.push(performance.now() - t0) + } + + for (let i = 0; i < hashes.length; i++) { + const expected = seededPayload(seed, i, config.payload_bytes) + const t0 = performance.now() + let got: Buffer | undefined + try { got = await backend.get(hashes[i]!) } catch { got = undefined } + getLat.push(performance.now() - t0) + if (!got) { getFail++; continue } + + // verify: bytes round-trip AND the content hash re-derives (content-addressing holds). + const tv = performance.now() + const rehash = sha256(got) + verLat.push(performance.now() - tv) + if (rehash === hashes[i] && got.equals(expected)) integrity_verified++ + else integrity_failures++ + } + + const opFailures = putFail + getFail + const finished_at = new Date().toISOString() + return { + backend_id: config.backend_id, + config: { ...config, seed }, + ops: [ + opResult('put', putLat, putFail, config.payload_bytes), + opResult('get', getLat, getFail, config.payload_bytes), + opResult('verify', verLat, 0, config.payload_bytes), + ], + integrity_verified, + integrity_failures, + result: integrity_failures === 0 && opFailures === 0 && integrity_verified === config.n_ops ? 'pass' : 'fail', + started_at, + finished_at, + } +} + +// ── SocioProphet storage-standard EvaluationRecord (evaluation-record-standard.v1) ────────────── + +export interface EvaluationRecord { + id: string + evaluation_track_ref: string + subject_ref: string + subject_type: 'platform_capability' + attempt_refs: { id: string; attempt_mode: 'benchmark_run'; measurements: StorageBenchResult }[] + metric_refs: string[] + result_summary: string + result: 'pass' | 'pass_with_findings' | 'remediation_required' | 'fail' | 'blocked' | 'unknown' + evidence_bundle_ref: string + review_state: 'draft' + created_at: string + updated_at: string +} + +/** Wrap a bench result as a storage-standard EvaluationRecord (evidence-first, regression-checkable). */ +export function toEvaluationRecord( + r: StorageBenchResult, + opts: { subject_ref: string; evaluation_track_ref: string }, +): EvaluationRecord { + const put = r.ops.find((o) => o.op === 'put')! + const get = r.ops.find((o) => o.op === 'get')! + const now = new Date().toISOString() + return { + id: `evalrec:storage-bench:${r.backend_id}:${r.started_at}`, + evaluation_track_ref: opts.evaluation_track_ref, + subject_ref: opts.subject_ref, + subject_type: 'platform_capability', + attempt_refs: [{ id: `attempt:${r.backend_id}:${r.started_at}`, attempt_mode: 'benchmark_run', measurements: r }], + metric_refs: ['put.p95_ms', 'get.p95_ms', 'put.throughput_mb_s', 'integrity_verified'], + result_summary: + `${r.backend_id}: ${r.config.n_ops}×${r.config.payload_bytes}B — ` + + `put p95 ${put.latency.p95_ms.toFixed(3)}ms, get p95 ${get.latency.p95_ms.toFixed(3)}ms, ` + + `integrity ${r.integrity_verified}/${r.config.n_ops}`, + result: r.result === 'pass' ? 'pass' : 'fail', + evidence_bundle_ref: `evidence:storage-bench:${r.backend_id}:${r.started_at}`, + review_state: 'draft', + created_at: r.started_at, + updated_at: now, + } +}