From c9250c2ee99528ddbc2c7c5f204ff091c95abea7 Mon Sep 17 00:00:00 2001 From: Aanish Bhirud <47579874+baanish@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:14:33 -0400 Subject: [PATCH] Stage frozen ARX6 native-frame codec and conformance harness Import the evaluated browser-safe codec without changing the viewer or legacy transport. Add standalone/Vitest checks, historical benchmark qualifications, and an unexecuted full-production-auto comparison runner. Keep application, Worker, and live transport integration as explicit draft deployment gates. --- docs/arx6-research.md | 103 ++++++ experiments/arx6/README.md | 78 +++++ experiments/arx6/bench.mts | 80 +++++ experiments/arx6/check.d.mts | 4 + experiments/arx6/check.mjs | 157 +++++++++ experiments/arx6/results/research-freeze.json | 9 + .../results/research-holdout-summary.json | 104 ++++++ experiments/arx6/runtime.d.mts | 8 + experiments/arx6/runtime.mjs | 58 ++++ experiments/arx6/src/arx6-core.d.mts | 13 + experiments/arx6/src/arx6-core.mjs | 46 +++ experiments/arx6/src/cm6.mjs | 305 ++++++++++++++++++ experiments/arx6/src/native-frame.mjs | 11 + tests/arx6-core.test.ts | 11 + 14 files changed, 987 insertions(+) create mode 100644 docs/arx6-research.md create mode 100644 experiments/arx6/README.md create mode 100644 experiments/arx6/bench.mts create mode 100644 experiments/arx6/check.d.mts create mode 100644 experiments/arx6/check.mjs create mode 100644 experiments/arx6/results/research-freeze.json create mode 100644 experiments/arx6/results/research-holdout-summary.json create mode 100644 experiments/arx6/runtime.d.mts create mode 100644 experiments/arx6/runtime.mjs create mode 100644 experiments/arx6/src/arx6-core.d.mts create mode 100644 experiments/arx6/src/arx6-core.mjs create mode 100644 experiments/arx6/src/cm6.mjs create mode 100644 experiments/arx6/src/native-frame.mjs create mode 100644 tests/arx6-core.test.ts diff --git a/docs/arx6-research.md b/docs/arx6-research.md new file mode 100644 index 0000000..4d49195 --- /dev/null +++ b/docs/arx6-research.md @@ -0,0 +1,103 @@ +# ARX6 research import and deployment gates + +Base revision: `72fb152e0cf92a11ff658c3b0dd3916b387f3e98`. + +## Scope of this draft + +This PR imports the executable, frozen general ARX6 codec, tests, and benchmark +glue under `experiments/arx6/`. **It does not implement viewer integration.** +The application protocol, default auto pool, UI, existing assets, dependencies, +and legacy decoders are unchanged. Experimental links are not yet shareable +through the deployed viewer. There are no server uploads or per-artifact fetches. + +## Mechanism + +The existing tuple pipeline JSON-escapes body newlines before the context mixer +sees them; substitutions can also create control bytes unrelated to source rows. +ARX6 codes a native tuple with raw WTF-8 strings instead. That preserves exact JS +strings, including lone surrogates, while exposing body rows to the existing +column expert. It adds causal previous-word / character-class contexts, a +syntax-conditioned residual mixer, and four-byte rather than seven-byte match +discovery. Direct low-order tables offset part of the additional allocation. + +The wire is `#g1L`. All coding decisions use deterministic +integer arithmetic. It reuses the existing three 16 KiB curated priors, verifies +their SHA-256 identities, and copies caller-owned buffers at installation. A +CRC32 over the native frame is included in measured link lengths. No target +artifact is added to the priors. + +## Historical measurements, not new production-baseline results + +The original development set had 40 artifacts from 18 source groups. The model +was then frozen before evaluation on 93 artifacts from 47 disjoint source groups. +The source groups and artifact kinds are software-text proxies, not a measured +sample of actual agent traffic; samples within groups are correlated. + +The historical baseline was the shorter of exact ARX5 entropy/base64url and +ARX2 with **Node Brotli q11**, not the complete production auto pool and not a +byte-identical validation of brotli-wasm. The new benchmark runner is designed to +close that gap; it has not yet supplied replacement numbers. + +| Kind | Samples | Aggregate full-link saving with fallback | +|---|---:|---:| +| Markdown | 20 | 5.76% | +| Code | 20 | 6.03% | +| Diff | 20 | 5.91% | +| CSV | 13 | 12.36% | +| JSON | 20 | 8.52% | +| Overall | 93 | 7.41% | + +Standalone ARX6 saved 7.36% aggregate, with 3 regressions and a worst regression +of 2.77%. The portfolio won 90/93 comparisons and kept the legacy wire on the +other three. Entire artifacts fitting 2,000 characters went from 75 to 76. +These scores count `[View](https://agent-render.com/#...)`, not visible glyphs. + +The existing report fixture was a **separate diagnostic, not holdout evidence**. +Its full link fell from 2,616 to 2,301 characters, still over the limit. Exhaustive +complete-line prefix testing increased the largest fitting prefix from 5,867 to +7,158 source characters at the same 1,996-character link size. That 22% capacity +gain applies to this fixture, not arbitrary inputs. + +The historical summaries and freeze record are retained under +`experiments/arx6/results/`. Their filenames and timings refer to the original +laboratory archive. Third-party corpus text and its large license bundle are not +silently republished as MIT application fixtures in this PR. Supply a permitted, +independent corpus to the new runner, and report its results separately. + +The original environment measured median encode times of 105 ms for ARX6 versus +56 ms for ARX5 and approximately 37.87 MiB versus 33.51 MiB of typed arrays. These +are neither peak-RSS measurements nor production/browser latency guarantees. +Async installation does not make the compression loop non-blocking. + +## Second-pass research disposition + +Dependency-aware numeric reconstruction programs and compression-chosen decode +order produced large synthetic specialist wins, but only 0.24% aggregate saving +on a fresh general-software set over ARX6. A separate 12-input natural CSV set +improved 4.21% with fallback, while the table method alone regressed 3.84% on the +ten inputs where it activated. A later root-only numeric ablation beat the graph +on some natural inputs. These results do not justify including that machinery in +this general codec PR. No universal breakthrough or scientific-first claim is +made here. + +## Required before viewer integration / default enablement + +- [ ] Reuse the canonical tuple helpers; validate reconstructed envelopes through + the normal schema/normalization path and reconcile decoded/fragment budgets. +- [ ] Introduce versioned async encode/decode routing and a compact tag; keep all + old tags and decoder bytes unchanged. Missing/skewed assets must be explicit, + retryable decode failures, never guessed priors. +- [ ] Run CPU-bound encoding/decoding in a bounded Worker with cancellation and + deliberate error handling; measure maximum-payload latency and peak memory. +- [ ] Compare the new candidate against the **complete** existing auto result, + retaining legacy bytes on ties, losses, and candidate unavailability. Keep + emission opt-in until independent real-agent data supports default enablement. +- [ ] Complete application/UI, asset-loading, TypeScript, lint, build, Chromium, + WebKit, and live Discord/WhatsApp paste/click verification. Update the app docs, + examples, codec pickers, and agent skill together when the protocol is wired in. + +Current local evidence: the original prototype suite passed 20/20 groups; the +new repository-asset conformance runner passed 16/16 checks. The full application +suite, the new production-baseline benchmark, and browser integration were not +run in the authoring environment. A draft PR is intentional, not a claim of +production readiness. diff --git a/experiments/arx6/README.md b/experiments/arx6/README.md new file mode 100644 index 0000000..68df5a4 --- /dev/null +++ b/experiments/arx6/README.md @@ -0,0 +1,78 @@ +# ARX6: frozen native-frame codec experiment + +**Draft research import, not viewer support.** Nothing here registers `arx6` in the +application schema, adds a picker option, changes automatic selection, or changes +any historical decoder. The laboratory `#g1L...` wire does **not** open in the +current viewer. Do not distribute it as a supported share link. + +This is the general ARX6 candidate from the first research pass. The second-pass +reconstruction-program / dependency-order experiments are deliberately excluded: +they did not earn a general-purpose default. + +## Run the conformance checks + +From the repository root, with Node 22: + +```sh +node experiments/arx6/check.mjs +``` + +This command needs no installed packages and makes no network requests. It reads +the existing three dictionary/prior assets from `public/`; no new corpus is +trained or downloaded. The same 16 cases are registered with the normal Vitest +suite in `tests/arx6-core.test.ts`: + +```sh +npx vitest run tests/arx6-core.test.ts +``` + +The tests cover implementation hashes, CRC32, all UTF-16 code units, lone +surrogates, exact whitespace/control preservation, bounded framing, all three +prior identities, immutable prior copies, bundles/diff metadata, seeded random +round trips, truncation/corruption, decoded limits, and byte-preserving fallback. + +## Compare against the actual production baseline + +After the usual `npm ci`, supply a JSON array of `{ "id": "...", "envelope": ... }` +samples. Each envelope must satisfy the existing application schema. + +```sh +node --import tsx experiments/arx6/bench.mts corpus.json > arx6-comparison.json +``` + +The benchmark loads the existing pinned assets, normalizes each envelope, calls +**the complete production `encodeEnvelopeAsync` auto pool** with transport +budgeting, and uses the real Markdown-link formatter. It verifies both the +baseline envelope and experimental tuple round trips. Every header, checksum, +base URL, and Markdown character is counted. Losses and oversized candidates +retain the original baseline wire, byte for byte. It emits measurements and +hashes, not unsupported viewer links. + +This production-baseline runner was added for this PR but has **not been executed +in the authoring environment**, which could not clone the repository or install +its npm dependencies. Do not substitute the historical research scores for its +results. The dependency-free conformance command was executed: **16/16 passed**. +The original, larger prototype suite was also rerun separately: **20/20 passed**. + +## What is frozen + +`src/cm6.mjs`, `src/native-frame.mjs`, `src/arx6-core.mjs`, and its `.d.mts` are copied +byte-for-byte from the evaluated prototype. Keep the wire-affecting code frozen; +a model/representation/prior change needs a new version and compatibility plan, +not a casual golden update. The source-hash test intentionally detects even +formatting changes. If a later cleanup changes formatting only, document it and +also prove unchanged wire vectors before replacing these source pins. + +`runtime.mjs` is Node-only laboratory glue. It reproduces the ARX2 tuple mapping +without modifying or importing private production helpers. Application +integration should expose/reuse the canonical production tuple helpers instead +of maintaining this duplicate mapping indefinitely. + +The browser-safe core returns **unknown tuples**, not validated application +envelopes. It is not an alternative to schema validation. CRC32 detects accidental +corruption, not malicious tampering or secret disclosure. Its 8,192-character +budget includes `#`; the current application budgets fragment bodies, so the +one-character boundary must be reconciled during integration. + +See [research notes](../../docs/arx6-research.md) for the qualified measurements +and the uncompleted deployment gates. diff --git a/experiments/arx6/bench.mts b/experiments/arx6/bench.mts new file mode 100644 index 0000000..933a5fb --- /dev/null +++ b/experiments/arx6/bench.mts @@ -0,0 +1,80 @@ +/** Compare ARX6 with the complete production auto pool; run with node --import tsx. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { performance } from "node:perf_hooks"; +import { createLabCodec, envelopeToTuple, priorForEnvelope } from "./runtime.mjs"; +import { formatMarkdownLink } from "@/lib/markdown-link"; +import { normalizeEnvelope } from "@/lib/payload/envelope"; +import { encodeEnvelopeAsync, decodeFragmentAsync } from "@/lib/payload/fragment"; +import { isPayloadEnvelope, MAX_DECODED_PAYLOAD_LENGTH } from "@/lib/payload/schema"; +import { loadArxDictionarySync, loadArx2OverlayDictionarySync } from "@/lib/payload/arx-codec"; +import { loadArx4PriorsSync } from "@/lib/payload/arx4-codec"; +import base from "../../public/arx-dictionary.json"; +import overlay from "../../public/arx2-dictionary.json"; +import priors from "../../public/arx4-priors.json"; + +const inputPath = process.argv[2]; +if (!inputPath) throw new Error("Usage: node --import tsx experiments/arx6/bench.mts corpus.json"); +const corpus: unknown = JSON.parse(readFileSync(inputPath, "utf8")); +if (!Array.isArray(corpus) || corpus.length === 0) throw new Error("Expected a nonempty array of {id, envelope} samples."); +loadArxDictionarySync(base); +loadArx2OverlayDictionarySync(overlay); +if (loadArx4PriorsSync(priors) !== 1) throw new Error("Production priors did not match their pins."); +const { codec } = await createLabCodec(); + +function link(fragmentBody: string): string { + const url = new URL("https://agent-render.com/"); + url.hash = fragmentBody; + return formatMarkdownLink("View", url.href); +} + +const rows = []; +for (const [index, sample] of corpus.entries()) { + if (sample === null || typeof sample !== "object" || !isPayloadEnvelope(sample.envelope)) { + throw new Error(`Invalid sample envelope at index ${index}.`); + } + const normalized = normalizeEnvelope(sample.envelope); + if (!normalized.ok) throw new Error(`Sample ${index}: ${normalized.message}`); + const envelope = normalized.envelope; + if (JSON.stringify(envelope).length > MAX_DECODED_PAYLOAD_LENGTH) throw new Error(`Sample ${index} exceeds the decoded budget.`); + const baselineStart = performance.now(); + const legacy = await encodeEnvelopeAsync(envelope, { budgetByTransport: true }); + const baselineMs = performance.now() - baselineStart; + const decoded = await decodeFragmentAsync(`#${legacy}`, { skipFragmentBudget: true }); + assert.ok(decoded.ok, `Production baseline failed to decode sample ${index}.`); + assert.deepEqual({ ...decoded.envelope, codec: "plain" }, { ...envelope, codec: "plain" }); + const value = envelopeToTuple(envelope); + const candidateStart = performance.now(); + const candidate = codec.encode(value, priorForEnvelope(envelope)); + const candidateMs = performance.now() - candidateStart; + if (candidate !== null) assert.deepEqual(codec.decode(candidate), value); + const baselineChars = link(legacy).length; + const candidateChars = candidate === null ? null : link(candidate.slice(1)).length; + const wins = candidateChars !== null && candidateChars < baselineChars; + const selected = wins ? candidate!.slice(1) : legacy; + // A tie/loss preserves the previous wire exactly; never wrap it in an ARX6 header. + if (!wins) assert.equal(selected, legacy); + rows.push({ + id: typeof sample.id === "string" ? sample.id : `sample-${index}`, + kind: envelope.artifacts[0].kind, + baselineTag: legacy.charAt(0), baselineChars, candidateChars, + selectedChars: wins ? candidateChars : baselineChars, wins, + baselineMs, candidateMs, + candidateSha256: candidate === null ? null : createHash("sha256").update(candidate).digest("hex"), + }); +} +const baselineTotal = rows.reduce((n, row) => n + row.baselineChars, 0); +const selectedTotal = rows.reduce((n, row) => n + (row.selectedChars ?? row.baselineChars), 0); +console.log(JSON.stringify({ + baseline: "complete production encodeEnvelopeAsync auto pool, budgetByTransport=true", + framing: "[View](https://agent-render.com/#...); includes every header and checksum character", + note: "Experimental ARX6 wires are not registered in the viewer; no shareable links are emitted.", + summary: { + samples: rows.length, wins: rows.filter(row => row.wins).length, + unavailableCandidates: rows.filter(row => row.candidateChars === null).length, + baselineTotal, selectedTotal, savedPercent: 100 * (1 - selectedTotal / baselineTotal), + baselineFits2000: rows.filter(row => row.baselineChars <= 2000).length, + selectedFits2000: rows.filter(row => (row.selectedChars ?? row.baselineChars) <= 2000).length, + }, rows, +}, null, 2)); diff --git a/experiments/arx6/check.d.mts b/experiments/arx6/check.d.mts new file mode 100644 index 0000000..495f20e --- /dev/null +++ b/experiments/arx6/check.d.mts @@ -0,0 +1,4 @@ +/** A dependency-free assertion case shared with the repository's Vitest runner. */ +export interface Arx6Check { name: string; run(): void | Promise; } +/** Install the pinned corpus and return the frozen-core conformance cases. */ +export function buildChecks(): Promise; diff --git a/experiments/arx6/check.mjs b/experiments/arx6/check.mjs new file mode 100644 index 0000000..b3a2a3c --- /dev/null +++ b/experiments/arx6/check.mjs @@ -0,0 +1,157 @@ +/** Dependency-free conformance cases, shared by the CLI and Vitest. No network requests. */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; +import { createLabCodec, envelopeToTuple } from './runtime.mjs'; +import { encodeNative, decodeNative, encodeString, decodeString } from './src/native-frame.mjs'; +import { createArx6Codec, crc32, formattedLink, selectShorterFragment } from './src/arx6-core.mjs'; + +const tuple = content => [3, ['m', 'doc', content, 'A title', 'notes.md'], 'Bundle title']; +const frozen = { + 'arx6-core.mjs': '6cd1d5b3bed631227adc564b057e23203fe52863399df88a9a15e131da4f5807', + 'cm6.mjs': '060bfda2b2207edf0ec2b415d2b8591188635b1d09b89f592d7dc36a9aa95e4b', + 'native-frame.mjs': '81074bf2360059c891233bde4034748cbe151958b7a5171c75baa542229c9392', +}; + +/** Build independent checks around a single immutable, hash-verified codec installation. */ +export async function buildChecks() { + const { codec, corpora } = await createLabCodec(); + function encode(value, prior = 'm') { + const fragment = codec.encode(value, prior); + assert.equal(typeof fragment, 'string', 'fixture must fit the experimental fragment budget'); + return fragment; + } + return [ + { name: 'frozen v1 implementation hashes', run() { + for (const [file, expected] of Object.entries(frozen)) { + const bytes = readFileSync(new URL(`./src/${file}`, import.meta.url)); + assert.equal(createHash('sha256').update(bytes).digest('hex'), expected, file); + } + } }, + { name: 'CRC32 standard check value', run() { + assert.equal(crc32(new TextEncoder().encode('123456789')), 0xcbf43926); + } }, + { name: 'every UTF-16 code unit survives native framing', run() { + const text = Array.from({ length: 65536 }, (_, c) => String.fromCharCode(c)).join(''); + assert.equal(decodeString(encodeString(text)), text); + assert.deepEqual(decodeNative(encodeNative(tuple(text))), tuple(text)); + } }, + { name: 'native frames reject truncation, trailing bytes, and overlong integers', run() { + const bytes = encodeNative(tuple('quoted " text\n\r\t\\n')); + for (let n = 0; n < bytes.length; n++) assert.throws(() => decodeNative(bytes.subarray(0, n))); + for (const invalid of [[...bytes, 0], [9], [2, 128, 0]]) { + assert.throws(() => decodeNative(Uint8Array.from(invalid))); + } + } }, + { name: 'invalid UTF-8 is rejected, not silently replaced', run() { + for (const bytes of [[255], [192, 128], [224, 128, 128], [244, 144, 128, 128], [226, 130], [128]]) { + assert.throws(() => decodeString(Uint8Array.from(bytes))); + } + } }, + { name: 'source strings round-trip without lexical normalization', run() { + for (const text of ['', '\0\x01\x1f\x7f\r\n\t\\u007f\\n\\\\n"', + '日本語の表\n한글 Ελληνικά मराठी 🧪 🚀 e\u0301', '\ud800\nx\udfff\ud800\udc00', + 'a,b,c\r\n1,"2,3",4\r\n', '--- a/a.ts\n+++ b/a.ts\n@@ -1 +1 @@\n-old\n+new\n']) { + const value = tuple(text), fragment = encode(value); + assert.match(fragment, /^#g1Lm[A-Za-z0-9_-]+$/); + assert.deepEqual(codec.decode(fragment), value); + assert.equal(new URL(`https://agent-render.com/${fragment}`).hash, fragment); + } + } }, + { name: 'bundles, active index, diff modes, and metadata survive', run() { + const value = [2, [['m', 'a', '# heading', null, 'README.md'], + ['c', 'b', 'const π = 3;', 'ts', 'Code'], + ['d', 'c', null, 'a\r\n', 'b\r\n', 'ts', 'split', 'Review', 'a.ts'], + ['s', 'd', 'x,y\n1,2'], ['j', 'e', '{"__proto__": true}\n']], 'Bundle', 3]; + assert.deepEqual(codec.decode(encode(value)), value); + } }, + { name: '24 seeded random tuples round-trip across all three priors', run() { + let seed = 0x6a09e667; + const rand = () => { seed ^= seed << 13; seed ^= seed >>> 17; seed ^= seed << 5; return seed >>> 0; }; + const chunks = [' ', '\n', '\r\n', '😀', '\ud800', '\udfff', '\\n', '"', '\0', + 'const ', 'return ', '|', '123.456', 'héllo', '中国']; + for (let i = 0; i < 24; i++) { + let text = ''; + const length = 20 + rand() % 120; + for (let j = 0; j < length; j++) text += chunks[rand() % chunks.length]; + const value = tuple(text); + assert.deepEqual(codec.decode(encode(value, ['m', 'c', 'j'][i % 3])), value); + } + } }, + { name: 'wire truncation, suffixes, and checksum corruption fail closed', run() { + const fragment = encode(tuple('A realistically short report.\n'.repeat(10))); + for (let n = 0; n < fragment.length; n += Math.max(1, Math.floor(fragment.length / 12))) { + assert.throws(() => codec.decode(fragment.slice(0, n))); + } + assert.throws(() => codec.decode(`${fragment}AAAA`)); + assert.throws(() => codec.decode(`${fragment}=`)); + const body = Buffer.from(fragment.slice(5), 'base64url'); + for (let i = 0; i < Math.min(20, body.length); i++) { + const changed = Buffer.from(body); changed[i] ^= 128; + assert.throws(() => codec.decode(fragment.slice(0, 5) + changed.toString('base64url'))); + } + } }, + { name: 'hostile decoded length fails before model allocation', run() { + const body = Buffer.from([0, 0, 0, 0, 255, 255, 255, 127, 0, 0, 0, 0]); + assert.throws(() => codec.decode(`#g1Lm${body.toString('base64url')}`), /decoded length/); + } }, + { name: 'payload budgets and unknown versions are enforced', run() { + let seed = 0x31415926, entropy = ''; + for (let i = 0; i < 20000; i++) { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + entropy += String.fromCharCode(33 + seed % 94); + } + assert.equal(codec.encode(tuple(entropy), 'm'), null); + assert.throws(() => codec.encode(tuple('x'.repeat(200001)), 'm'), /200000/); + assert.throws(() => codec.decode(`#g1Lm${'A'.repeat(8192)}`), /oversized/); + assert.throws(() => codec.decode('#g2LmAAAA'), /unsupported/); + assert.throws(() => codec.decode('#g1LnAAAA'), /unsupported/); + } }, + { name: 'missing and altered priors fail identity checks', async run() { + const altered = { ...corpora, m: new Uint8Array(corpora.m) }; altered.m[0] ^= 1; + await assert.rejects(createArx6Codec(altered), /digest/); + await assert.rejects(createArx6Codec({}), /missing/); + } }, + { name: 'installed prior bytes are immutable copies', async run() { + const supplied = Object.fromEntries(Object.entries(corpora).map(([id, bytes]) => [id, new Uint8Array(bytes)])); + const installed = await createArx6Codec(supplied); + const value = tuple('Prior ownership must not affect shared links.'); + const expected = installed.encode(value, 'm'); supplied.m.fill(0); + assert.equal(installed.encode(value, 'm'), expected); + assert.deepEqual(installed.decode(expected), value); + } }, + { name: 'fallback preserves legacy bytes on ties, losses, and null candidates', run() { + const legacy = '#fmb64'; + assert.equal(selectShorterFragment(legacy, '#g1Lmabcdef'), legacy); + assert.equal(selectShorterFragment(legacy, legacy), legacy); + assert.equal(selectShorterFragment(legacy, null), legacy); + assert.equal(formattedLink(legacy), '[View](https://agent-render.com/#fmb64)'); + } }, + { name: 'near-limit compressible content remains lossless', run() { + const value = tuple(' return value;\n'.repeat(10000)); + assert.deepEqual(codec.decode(encode(value, 'c')), value); + } }, + { name: 'laboratory envelope mapping preserves optional holes and active index', run() { + const envelope = { title: 'Bundle', activeArtifactId: 'review', artifacts: [ + { id: 'notes', kind: 'markdown', content: '# Notes', filename: 'notes.md' }, + { id: 'review', kind: 'diff', oldContent: 'a\r\n', newContent: 'b\r\n', view: 'split' }, + ] }; + assert.deepEqual(envelopeToTuple(envelope), [2, [ + ['m', 'notes', '# Notes', null, 'notes.md'], + ['d', 'review', null, 'a\r\n', 'b\r\n', null, 'split'], + ], 'Bundle', 1]); + } }, + ]; +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + const checks = await buildChecks(); + let failed = 0; + for (const check of checks) { + try { await check.run(); console.log(`PASS ${check.name}`); } + catch (error) { failed++; console.error(`FAIL ${check.name}`, error); } + } + console.log(`${checks.length - failed}/${checks.length} checks passed.`); + process.exitCode = failed ? 1 : 0; +} diff --git a/experiments/arx6/results/research-freeze.json b/experiments/arx6/results/research-freeze.json new file mode 100644 index 0000000..32c4155 --- /dev/null +++ b/experiments/arx6/results/research-freeze.json @@ -0,0 +1,9 @@ +{ + "note": "These compression model/representation files were frozen before the first holdout benchmark. Decoder-only hardening after holdout must not change encoder output.", + "files": { + "src/cm6.mjs": "060bfda2b2207edf0ec2b415d2b8591188635b1d09b89f592d7dc36a9aa95e4b", + "src/native-frame.mjs": "81074bf2360059c891233bde4034748cbe151958b7a5171c75baa542229c9392", + "src/arx6-core.mjs": "6cd1d5b3bed631227adc564b057e23203fe52863399df88a9a15e131da4f5807" + }, + "holdoutSha256": "d3e010666fb5b19f6e8dfacf651d7ea0657230609be04cfaf29b4b84b5185681" +} \ No newline at end of file diff --git a/experiments/arx6/results/research-holdout-summary.json b/experiments/arx6/results/research-holdout-summary.json new file mode 100644 index 0000000..ee57d1e --- /dev/null +++ b/experiments/arx6/results/research-holdout-summary.json @@ -0,0 +1,104 @@ +{ + "split": "holdout", + "n": 93, + "node": "v22.16.0", + "baseCommit": "72fb152e0cf92a11ff658c3b0dd3916b387f3e98", + "byKind": { + "all": { + "n": 93, + "weightedSavingPct": 7.41368510261985, + "medianSavingPct": 6.539509536784738, + "standaloneWeightedSavingPct": 7.364309665057666, + "wins": 90, + "ties": 3, + "standaloneRegressions": 3, + "baselineFits": 75, + "selectedFits": 76, + "newFits": 1, + "medianEncodeMs": 105.08930099999998, + "medianDecodeMs": 107.59362800000008, + "medianArx5Ms": 56.32993399999941, + "worstStandalonePct": -2.7692307692307683 + }, + "markdown": { + "n": 20, + "weightedSavingPct": 5.7575413281826915, + "medianSavingPct": 5.875302096379359, + "standaloneWeightedSavingPct": 5.7575413281826915, + "wins": 20, + "ties": 0, + "standaloneRegressions": 0, + "baselineFits": 14, + "selectedFits": 14, + "newFits": 0, + "medianEncodeMs": 115.93000299999949, + "medianDecodeMs": 125.57231599999932, + "medianArx5Ms": 59.93482949999998, + "worstStandalonePct": 1.6662327518875308 + }, + "code": { + "n": 20, + "weightedSavingPct": 6.0296468081105665, + "medianSavingPct": 6.095753122703895, + "standaloneWeightedSavingPct": 6.0296468081105665, + "wins": 20, + "ties": 0, + "standaloneRegressions": 0, + "baselineFits": 15, + "selectedFits": 16, + "newFits": 1, + "medianEncodeMs": 103.3278500000003, + "medianDecodeMs": 103.01537649999977, + "medianArx5Ms": 53.63199200000008, + "worstStandalonePct": 2.4582104228121904 + }, + "diff": { + "n": 20, + "weightedSavingPct": 5.910283078592748, + "medianSavingPct": 5.703107493316518, + "standaloneWeightedSavingPct": 5.905761056344394, + "wins": 19, + "ties": 1, + "standaloneRegressions": 1, + "baselineFits": 19, + "selectedFits": 19, + "newFits": 0, + "medianEncodeMs": 99.4444429999985, + "medianDecodeMs": 97.9721884999999, + "medianArx5Ms": 50.59004450000066, + "worstStandalonePct": -0.29850746268655914 + }, + "csv": { + "n": 13, + "weightedSavingPct": 12.36123748862602, + "medianSavingPct": 13.057224154283908, + "standaloneWeightedSavingPct": 12.36123748862602, + "wins": 13, + "ties": 0, + "standaloneRegressions": 0, + "baselineFits": 9, + "selectedFits": 9, + "newFits": 0, + "medianEncodeMs": 105.08930099999998, + "medianDecodeMs": 109.01029199999994, + "medianArx5Ms": 59.89777200000026, + "worstStandalonePct": 1.6921837228041858 + }, + "json": { + "n": 20, + "weightedSavingPct": 8.515069835824551, + "medianSavingPct": 10.28880613362541, + "standaloneWeightedSavingPct": 8.245528056848816, + "wins": 18, + "ties": 2, + "standaloneRegressions": 2, + "baselineFits": 18, + "selectedFits": 18, + "newFits": 0, + "medianEncodeMs": 95.96260600000005, + "medianDecodeMs": 96.99351950000073, + "medianArx5Ms": 57.010628, + "worstStandalonePct": -2.7692307692307683 + } + } +} \ No newline at end of file diff --git a/experiments/arx6/runtime.d.mts b/experiments/arx6/runtime.d.mts new file mode 100644 index 0000000..4a8e501 --- /dev/null +++ b/experiments/arx6/runtime.d.mts @@ -0,0 +1,8 @@ +import type { PayloadEnvelope } from '../../src/lib/payload/schema'; +import type { Arx6Codec, Arx6Corpora, Arx6PriorId } from './src/arx6-core.mjs'; +/** Load only the existing version-1 assets and verify their hashes. */ +export function createLabCodec(): Promise<{ codec: Readonly; corpora: Arx6Corpora }>; +/** Convert a previously validated envelope to the unchanged experimental tuple mapping. */ +export function envelopeToTuple(envelope: PayloadEnvelope): unknown; +/** Select the frozen kind-dependent prior. */ +export function priorForEnvelope(envelope: PayloadEnvelope): Arx6PriorId; diff --git a/experiments/arx6/runtime.mjs b/experiments/arx6/runtime.mjs new file mode 100644 index 0000000..cbad429 --- /dev/null +++ b/experiments/arx6/runtime.mjs @@ -0,0 +1,58 @@ +/** Node-only laboratory glue. Never imported by the deployed viewer. */ +import { readFileSync } from 'node:fs'; +import { createArx6Codec } from './src/arx6-core.mjs'; + +function readAsset(name) { + return JSON.parse(readFileSync(new URL(`../../public/${name}`, import.meta.url), 'utf8')); +} + +/** Rebuild the three existing priors; the codec checks their bytes against SHA-256 pins. */ +export async function createLabCodec() { + const base = readAsset('arx-dictionary.json'); + const overlay = readAsset('arx2-dictionary.json'); + const asset = readAsset('arx4-priors.json'); + if (base.version !== 1 || overlay.version !== 1 || asset.version !== 1) { + throw new Error('ARX6 v1 requires the version-1 base, overlay, and curated assets.'); + } + const common = [ + ...base.singleByteSlots, ...base.extendedSlots, + ...overlay.singleByteSlots, ...overlay.extendedSlots, + ].join('\n'); + const corpora = Object.fromEntries(Object.entries({ m: 'markdown', c: 'code', j: 'json' }) + .map(([id, kind]) => [id, new TextEncoder().encode(`${common}\n${asset.kinds[kind]}`)])); + return { codec: await createArx6Codec(corpora), corpora }; +} + +function trim(fields) { + let end = fields.length; + while (end > 0 && fields[end - 1] === undefined) end--; + return fields.slice(0, end).map(value => value === undefined ? null : value); +} + +function artifactTuple(artifact) { + switch (artifact.kind) { + case 'markdown': return trim(['m', artifact.id, artifact.content, artifact.title, artifact.filename]); + case 'code': return trim(['c', artifact.id, artifact.content, artifact.language, artifact.title, artifact.filename]); + case 'diff': return trim(['d', artifact.id, artifact.patch, artifact.oldContent, artifact.newContent, + artifact.language, artifact.view, artifact.title, artifact.filename]); + case 'csv': return trim(['s', artifact.id, artifact.content, artifact.title, artifact.filename]); + case 'json': return trim(['j', artifact.id, artifact.content, artifact.title, artifact.filename]); + default: throw new Error('Unsupported artifact kind.'); + } +} + +/** Laboratory copy of the frozen ARX2 tuple mapping; callers validate envelopes first. */ +export function envelopeToTuple(envelope) { + const artifacts = envelope.artifacts.map(artifactTuple); + const activeIndex = envelope.artifacts.findIndex(artifact => artifact.id === envelope.activeArtifactId); + return artifacts.length === 1 + ? trim([3, artifacts[0], envelope.title]) + : trim([2, artifacts, envelope.title, activeIndex > 0 ? activeIndex : undefined]); +} + +/** Select the same kind-dependent prior used in the frozen experiment. */ +export function priorForEnvelope(envelope) { + const id = { markdown: 'm', code: 'c', diff: 'c', csv: 'j', json: 'j' }[envelope.artifacts[0]?.kind]; + if (!id) throw new Error('An envelope must contain a supported artifact.'); + return id; +} diff --git a/experiments/arx6/src/arx6-core.d.mts b/experiments/arx6/src/arx6-core.d.mts new file mode 100644 index 0000000..da12da3 --- /dev/null +++ b/experiments/arx6/src/arx6-core.d.mts @@ -0,0 +1,13 @@ +/** Byte corpora are pinned to the already-shipped ARX4/5 version-1 assets. */ +export type Arx6PriorId = 'm' | 'c' | 'j'; +export type Arx6Corpora = Readonly>; +/** Native tuples are validated by the existing application envelope/schema layer after decoding. */ +export interface Arx6Codec { + encode(tuple: unknown, priorId: Arx6PriorId): string | null; + decode(fragment: string): unknown; +} +export interface LinkOptions { label?: string; baseUrl?: string; } +export function createArx6Codec(corpora: Arx6Corpora): Promise>; +export function crc32(bytes: Uint8Array): number; +export function formattedLink(fragment: string, options?: LinkOptions): string; +export function selectShorterFragment(legacy: string, candidate: string | null, options?: LinkOptions): string; diff --git a/experiments/arx6/src/arx6-core.mjs b/experiments/arx6/src/arx6-core.mjs new file mode 100644 index 0000000..505d951 --- /dev/null +++ b/experiments/arx6/src/arx6-core.mjs @@ -0,0 +1,46 @@ +/** + * ARX6 experimental wire v1. This is not deployed on agent-render.com. + * Shape: #g1L. + * L = native tuple + lexical/syntax context mixer. All five header characters count. + * Browser-safe: no Node dependencies. Hash-pinned priors must be installed before use. + */ +import {encodeCm,decodeCm} from './cm6.mjs'; +import {encodeNative,decodeNative} from './native-frame.mjs'; +const PINNED={ + m:'90da74cfa7a7394099aefd7d8f3ba9ed2acc40237b23d58048f4b8b4dd596c9c', + c:'3596c70d73b7d3f95e5f978a0c3bcb4ae1d4aa8711d563f4a22f39d0123aa6af', + j:'37e1cfa8f8885afda7e560d63616b4e84e891a1c3a63d2ac3a139ebe6558fb18', +}; +const MAX_FRAGMENT=8192,MAX_DECODED_CHARS=200000; +const crcTable=Uint32Array.from({length:256},(_,i)=>{let c=i;for(let k=0;k<8;k++)c=(c>>>1)^((c&1)?0xedb88320:0);return c>>>0;}); +/** CRC32 detects accidental corruption. It is NOT authentication or encryption. */ +export function crc32(bytes){let c=0xffffffff;for(const b of bytes)c=(c>>>8)^crcTable[(c^b)&255];return(c^0xffffffff)>>>0;} +function toBase64(a){let s='';for(let i=0;ic.charCodeAt(0));if(toBase64(a)!==s)throw Error('noncanonical base64url');return a;} +function budgetTuple(tuple){const s=JSON.stringify(tuple);if(s===undefined||s.length>MAX_DECODED_CHARS)throw Error('decoded tuple exceeds 200000 characters');} +/** + * Install immutable copies of the existing curated priors. Reject altered, missing or future data. + * Encoders/decoders cannot pass arbitrary "helpful" corpus text and silently contaminate a stream. + */ +export async function createArx6Codec(corpora){const pinned={};for(const id of Object.keys(PINNED)){ + const p=corpora[id];if(!(p instanceof Uint8Array)||p.length!==16384)throw Error('missing or invalid prior '+id); + const copy=new Uint8Array(p),digest=new Uint8Array(await crypto.subtle.digest('SHA-256',copy)); + const hex=Array.from(digest,b=>b.toString(16).padStart(2,'0')).join('');if(hex!==PINNED[id])throw Error('prior digest mismatch '+id);pinned[id]=copy; + } + return Object.freeze({ + /** Emit an experimental ASCII fragment. Returns null rather than violating the fragment budget. */ + encode(tuple,id){if(!Object.hasOwn(pinned,id))throw Error('unknown prior');budgetTuple(tuple);const raw=encodeNative(tuple),cm=encodeCm(raw,pinned[id]).bytes,crc=crc32(raw),body=new Uint8Array(cm.length+4);new DataView(body.buffer).setUint32(0,crc,false);body.set(cm,4);const fragment='#g1L'+id+toBase64(body);return fragment.length<=MAX_FRAGMENT?fragment:null;}, + /** Decode only this version; validate framing and checksum before exposing the resulting tuple. */ + decode(fragment){if(typeof fragment!=='string'||fragment.length>MAX_FRAGMENT||!/^#g1L[mcj]/.test(fragment))throw Error('unsupported or oversized ARX6 fragment');const id=fragment[4],body=fromBase64(fragment.slice(5));if(body.length<9)throw Error('short ARX6 payload');const expected=new DataView(body.buffer,body.byteOffset,body.byteLength).getUint32(0,false),raw=decodeCm(body.subarray(4),pinned[id]);if(crc32(raw)!==expected)throw Error('ARX6 checksum mismatch');const tuple=decodeNative(raw);budgetTuple(tuple);return tuple;} + }); +} +/** Count the actual wire in a complete markdown link; no rendered/visible-character shortcut. */ +export function formattedLink(fragment,{label='View',baseUrl='https://agent-render.com/'}={}){ + if(!/^[A-Za-z0-9 _-]*$/.test(label))throw Error('benchmark labels must not require markdown escaping'); + if(!fragment.startsWith('#')||/[^\x21-\x7e]/.test(fragment))throw Error('expected ASCII fragment'); + // The deployed application should use its own established URL/markdown serialization helper. + const url=new URL(baseUrl);if(url.hash||url.search)throw Error('base URL must not include payload or search');url.hash=fragment; + return `[${label}](${url.href})`; +} +/** Keep a byte-for-byte legacy link when the new candidate does not win after actual serialization. */ +export function selectShorterFragment(legacy,candidate,options){if(candidate===null)return legacy;return formattedLink(candidate,options).length=0) return Math.floor((n+Math.floor(d/2))/d); + return -Math.floor((-n+Math.floor(d/2))/d); +} +function log2Q16(value) { + const integerPart=31-Math.clz32(value); + let normalized=BigInt(value)<=0;bit--) { + normalized=(normalized*normalized)>>31n; + if(normalized>=1n<<32n) { normalized>>=1n; fraction|=1<>>1; if(stretchTable[middle]>>12)&0x3f; + const target=bit===1?4096:0; + const numerator=count===MAX_MODEL_COUNT?target-probability:2*(target-probability); + const denominator=count===MAX_MODEL_COUNT?32:2*count+3; + const nextProbability=Math.max(1,Math.min(4095,probability+divideRound(numerator,denominator))); + return nextProbability|(Math.min(MAX_MODEL_COUNT,count+1)<<12); +} +function taggedSlotProbability(packed,tag) { return packed===0||((packed>>>18)&0xff)!==tag?2048:packed&0xfff; } +function isWordByte(b) { return (b>=48&&b<=57)||(b>=65&&b<=90)||b===95||(b>=97&&b<=122); } +export class ContextMixModel { + constructor() { + this.tables=Array.from({length:MODEL_COUNT},(_,i)=>new Uint32Array(i===0?256:i===1?65536:TABLE_SIZE)); + this.weights=new Int32Array(256*MIXER_INPUT_COUNT);this.residualWeights=new Int32Array(2048*MIXER_INPUT_COUNT);this.residualOffset=0; + this.initializedMixers=new Uint8Array(256); + this.historyHashes=new Uint32Array(MODEL_COUNT); + this.matchTable=new Uint32Array(MATCH_TABLE_SIZE); + this.matchBuffer=new Uint8Array(MATCH_BUFFER_SIZE); + this.columnTable=new Uint32Array(COLUMN_TABLE_SIZE); + this.cachedIndexes=new Uint32Array(MODEL_COUNT); + this.cachedTags=new Uint8Array(MODEL_COUNT); + this.cachedStretches=new Int16Array(MIXER_INPUT_COUNT); + this.history=[];this.byteCount=0;this.matchPosition=-1;this.matchLength=0; + this.wordHash=0;this.wordLength=0;this.previousWord=0;this.previousWord2=0;this.classHash=0;this.c0=1;this.bitShift=7; + this.cachedMixerOffset=0;this.cachedRawProbability=2048;this.cachedMatchBit=-1; + this.columnLineStats=[];this.columnPreviousLineBytes=[];this.columnCurrentLineBytes=[]; + this.columnCurrentLength=0;this.columnCurrentPipes=0;this.columnCurrentCommas=0; + this.columnRowActive=false;this.columnDelimiter=-1;this.columnFieldIndex=0; + this.columnFieldOffset=0;this.columnPreviousFields=null;this.columnSlotIndex=-1; + this.columnTag=0;this.columnHash=0; + // Observational counters: not used by predictions or coding decisions. + this.stats={bytes:0,newlines:0,columnBytes:0}; + this.prepareByteContexts(); + } + hashHistory(order) { + let hash=(FNV_OFFSET_BASIS^order^(this.history.length<<24))>>>0; + for(let index=0;index>>0; + } + return hash; + } + prepareByteContexts() { + const word=(this.wordLength===0?0x9e3779b9:this.wordHash^Math.imul(this.wordLength,0x85ebca6b))>>>0; + for(let i=0;i0)this.historyHashes[i]=this.hashHistory(order); + else if(order===-1)this.historyHashes[i]=word; + else if(order===-2)this.historyHashes[i]=(word^Math.imul(this.previousWord,0x85ebca6b))>>>0; + else if(order===-3)this.historyHashes[i]=(word^Math.imul(this.previousWord,0x85ebca6b)^Math.imul(this.previousWord2,0xc2b2ae35))>>>0; + else if(order===-4)this.historyHashes[i]=(Math.imul(this.classHash,0x9e3779b1)^Math.imul((this.history[0]??0)+1,0x85ebca6b))>>>0; + } + } + matchByteAt(p) { return this.matchBuffer[p&(MATCH_BUFFER_SIZE-1)]; } + isReadableMatchPosition(p) { return p>=0&&p=0;offset--) hash=Math.imul(hash^this.matchByteAt(end-offset),FNV_PRIME)>>>0; + return hash>>>(32-MATCH_TABLE_BITS); + } + matchContextsEqual(left,right) { + for(let offset=0;offset=0; + if(matchedWholeByte) { this.matchPosition++;this.matchLength=Math.min(255,this.matchLength+1); } + const currentPosition=this.byteCount; + this.matchBuffer[currentPosition&(MATCH_BUFFER_SIZE-1)]=byte; + this.byteCount++; + if(this.byteCount=MATCH_HASH_BYTES-1&&this.isReadableMatchPosition(candidateNext)&&this.matchContextsEqual(previousEnd,currentPosition)) { + this.matchPosition=candidateNext;this.matchLength=MATCH_HASH_BYTES; + } else { this.matchPosition=-1;this.matchLength=0; } + } + beginColumnRow() { + this.columnRowActive=false;this.columnDelimiter=-1;this.columnFieldIndex=0; + this.columnFieldOffset=0;this.columnPreviousFields=null; + const previous=this.columnLineStats[0],older=this.columnLineStats[1]; + if(previous===undefined||older===undefined||previous.length===0||older.length===0) return; + if(previous.pipes===older.pipes&&previous.pipes>=COLUMN_MIN_DELIMITERS) this.columnDelimiter=PIPE_BYTE; + else if(previous.commas===older.commas&&previous.commas>=COLUMN_MIN_DELIMITERS) this.columnDelimiter=COMMA_BYTE; + else return; + this.columnRowActive=true; + this.columnPreviousFields=this.splitColumnFields(this.columnPreviousLineBytes); + } + splitColumnFields(lineBytes) { + const fields=[];let field=[]; + for(let index=0;indexCOLUMN_MAX_FIELD_INDEX) return fields; + field=[];continue; + } + if(field.length<=COLUMN_MAX_FIELD_OFFSET) field.push(byte); + } + fields.push(field);return fields; + } + updateColumnState(byte) { + if(byte===NEWLINE_BYTE) { + this.columnLineStats.unshift({length:this.columnCurrentLength,pipes:this.columnCurrentPipes,commas:this.columnCurrentCommas}); + if(this.columnLineStats.length>2) this.columnLineStats.length=2; + this.columnPreviousLineBytes=this.columnCurrentLineBytes;this.columnCurrentLineBytes=[]; + this.columnCurrentLength=0;this.columnCurrentPipes=0;this.columnCurrentCommas=0; + this.beginColumnRow(); + } else { + if(this.columnCurrentLineBytes.length>>0; + hash=Math.imul(hash^(fieldIndex+1),FNV_PRIME)>>>0; + hash=Math.imul(hash^(fieldOffset+1),FNV_PRIME)>>>0; + hash=Math.imul(hash^(above+1),FNV_PRIME)>>>0; + return hash===0?1:hash; + } + predict() { + const mixerContext=this.c0&0xff, mixerOffset=mixerContext*MIXER_INPUT_COUNT; + if(this.initializedMixers[mixerContext]===0) { + for(let index=0;index>>0; + const slotIndex=index===0?this.c0:index===1?(((this.history[0]??0)<<8)|this.c0):slotHash>>>TABLE_SHIFT,tag=slotHash&0xff; + const stretched=stretchTable[taggedSlotProbability(this.tables[index][slotIndex],tag)]; + this.cachedIndexes[index]=slotIndex;this.cachedTags[index]=tag;this.cachedStretches[index]=stretched; + mixedStretch+=this.weights[mixerOffset+index]*stretched; + } + this.cachedMatchBit=-1;this.cachedStretches[MODEL_COUNT]=0;this.cachedStretches[MODEL_COUNT+1]=0; + if(this.isReadableMatchPosition(this.matchPosition)) { + const matchBit=(this.matchByteAt(this.matchPosition)>>>this.bitShift)&1; + const direction=matchBit===1?1:-1,cappedMatchLength=Math.min(this.matchLength,32); + this.cachedMatchBit=matchBit; + this.cachedStretches[MODEL_COUNT]=direction*cappedMatchLength*64; + this.cachedStretches[MODEL_COUNT+1]=direction*Math.max(0,cappedMatchLength-11)*64; + mixedStretch+=this.weights[mixerOffset+MODEL_COUNT]*this.cachedStretches[MODEL_COUNT]+this.weights[mixerOffset+MODEL_COUNT+1]*this.cachedStretches[MODEL_COUNT+1]; + } + this.columnSlotIndex=-1;this.cachedStretches[COLUMN_INPUT_INDEX]=0; + if(this.columnHash!==0) { + const slotHash=(Math.imul(this.columnHash,0x9e3779b1)^Math.imul(this.c0,0x85ebca6b))>>>0; + const slotIndex=slotHash>>>TABLE_SHIFT,tag=slotHash&0xff; + const stretched=stretchTable[taggedSlotProbability(this.columnTable[slotIndex],tag)]; + this.columnSlotIndex=slotIndex;this.columnTag=tag;this.cachedStretches[COLUMN_INPUT_INDEX]=stretched; + mixedStretch+=this.weights[mixerOffset+COLUMN_INPUT_INDEX]*stretched; + } + const prev=this.history[0]??0; + const cls=prev>=97&&prev<=122?1:prev>=65&&prev<=90?2:prev>=48&&prev<=57?3:prev===32?4:prev===10?5:prev<32?6:7; + this.residualOffset=((cls<<8)|(this.c0&255))*MIXER_INPUT_COUNT; + for(let i=0;i>>18)&0xff)===tag?packed:0; + this.tables[index][si]=updatePackedState(state,bit)|(tag<<18); + } + if(this.columnSlotIndex>=0) { + const packed=this.columnTable[this.columnSlotIndex]; + const state=packed!==0&&((packed>>>18)&0xff)===this.columnTag?packed:0; + this.columnTable[this.columnSlotIndex]=updatePackedState(state,bit)|(this.columnTag<<18); + } + if(this.cachedMatchBit>=0&&bit!==this.cachedMatchBit) { this.matchPosition=-1;this.matchLength=0; } + this.c0=(this.c0<<1)|bit;this.bitShift--; + } + finishByte(byte) { + this.stats.bytes++;if(byte===10)this.stats.newlines++;if(this.columnRowActive)this.stats.columnBytes++; + this.updateMatch(byte);this.history.unshift(byte);if(this.history.length>HISTORY_BYTES)this.history.length=HISTORY_BYTES; + if(isWordByte(byte)) { + if(this.wordLength===0)this.wordHash=FNV_OFFSET_BASIS; + this.wordHash=Math.imul(this.wordHash^byte,FNV_PRIME)>>>0; + this.wordLength=Math.min(255,this.wordLength+1); + } else {if(this.wordLength){this.previousWord2=this.previousWord;this.previousWord=this.wordHash;}this.wordHash=0;this.wordLength=0;} + const cls=byte>=97&&byte<=122?1:byte>=65&&byte<=90?2:byte>=48&&byte<=57?3:byte===32?4:byte===10?5:byte<32?6:7; + this.classHash=((this.classHash<<3)|cls)&0x7fff; + this.c0=1;this.bitShift=7;this.prepareByteContexts();this.updateColumnState(byte); + } + processKnownByte(byte,consumePrediction) { + for(let shift=7;shift>=0;shift--) { + const p=this.predict(),bit=(byte>>>shift)&1; + consumePrediction(p,bit);this.update(bit); + } + this.finishByte(byte); + } + processDecodedByte(readBit) { + let byte=0; + for(let shift=7;shift>=0;shift--) { + const p=this.predict(),bit=readBit(p);byte|=bit<this.x2)throw Error('midpoint'); + if(bit===1)this.x2=mid>>>0;else this.x1=(mid+1)>>>0; + while(((this.x1^this.x2)&0xff000000)===0){this.output.push(this.x2>>>24);this.x1=(this.x1<<8)>>>0;this.x2=((this.x2<<8)|255)>>>0;} + } + finish(){this.output.push(this.x1>>>24,(this.x1>>>16)&255,(this.x1>>>8)&255,this.x1&255);return Uint8Array.from(this.output);} +} +export class BinaryArithmeticDecoder { + constructor(input){if(input.length<4)throw Error('short arithmetic payload');this.input=input;this.offset=4;this.x1=0;this.x2=0xffffffff;this.x=(input[0]*0x1000000+input[1]*0x10000+input[2]*0x100+input[3])>>>0;} + readByte(){if(this.offset>=this.input.length)throw Error('arithmetic decoder exhausted');return this.input[this.offset++];} + readBit(p){ + const mid=this.x1+Math.floor((this.x2-this.x1)/4096)*p; + if(midthis.x2)throw Error('midpoint'); + const bit=this.x<=mid?1:0; + if(bit===1)this.x2=mid>>>0;else this.x1=(mid+1)>>>0; + while(((this.x1^this.x2)&0xff000000)===0){this.x1=(this.x1<<8)>>>0;this.x2=((this.x2<<8)|255)>>>0;this.x=((this.x<<8)|this.readByte())>>>0;} + return bit; + } +} +export function encodeVarint(v){const bytes=[];do{let byte=v%128;v=Math.floor(v/128);if(v>0)byte|=128;bytes.push(byte);}while(v>0);return bytes;} +export function decodeVarint(input){let value=0,multiplier=1;for(let offset=0;offset{}); + model.stats={bytes:0,newlines:0,columnBytes:0}; + const coder=new BinaryArithmeticEncoder(); + for(const byte of input)model.processKnownByte(byte,(p,b)=>coder.writeBit(b,p)); + const prefix=encodeVarint(input.length),coded=coder.finish(),out=new Uint8Array(prefix.length+coded.length); + out.set(prefix);out.set(coded,prefix.length);return{bytes:out,stats:model.stats}; +} +export function decodeCm(input,prime=null,Model=ContextMixModel){ + const{value,bytesRead}=decodeVarint(input); + if(!Number.isSafeInteger(value)||value<0||value>800000)throw Error('decoded length exceeds limit'); + const model=new Model();if(prime)for(const byte of prime)model.processKnownByte(byte,()=>{}); + const coder=new BinaryArithmeticDecoder(input.subarray(bytesRead)),out=new Uint8Array(value); + for(let i=0;icoder.readBit(p));if(coder.offset!==coder.input.length)throw Error("trailing arithmetic bytes");return out; +} diff --git a/experiments/arx6/src/native-frame.mjs b/experiments/arx6/src/native-frame.mjs new file mode 100644 index 0000000..3b9876e --- /dev/null +++ b/experiments/arx6/src/native-frame.mjs @@ -0,0 +1,11 @@ +/** Lossless tuple framing: raw WTF-8 strings, small integers, arrays, and null. No JSON escaping. */ +const MAX=800000; +function pushVar(out,n){if(!Number.isSafeInteger(n)||n<0||n>MAX)throw Error('invalid frame integer');do{let b=n%128;n=Math.floor(n/128);out.push(b|(n?128:0));}while(n);} +/** WTF-8 retains JavaScript lone surrogates instead of replacing them with U+FFFD. */ +export function encodeString(s){const a=[];for(let i=0;i65535)i++;if(c<128)a.push(c);else if(c<2048)a.push(192|(c>>6),128|(c&63));else if(c<65536)a.push(224|(c>>12),128|((c>>6)&63),128|(c&63));else a.push(240|(c>>18),128|((c>>12)&63),128|((c>>6)&63),128|(c&63));}return Uint8Array.from(a);} +/** Rejects malformed and overlong byte sequences; allows a three-byte lone surrogate. */ +export function decodeString(a){const chunks=[];let chars=[];for(let i=0;i=194&&b<=223){n=1;c=b&31;min=128;}else if(b>=224&&b<=239){n=2;c=b&15;min=2048;}else if(b>=240&&b<=244){n=3;c=b&7;min=65536;}else throw Error('invalid WTF-8 lead');for(let j=0;j=a.length||(a[i]&192)!==128)throw Error('truncated WTF-8');c=c*64+(a[i++]&63);}if(c0x10ffff)throw Error('noncanonical WTF-8');if(c>65535){c-=65536;chars.push(0xd800+(c>>10),0xdc00+(c&1023));}else chars.push(c);if(chars.length>4096){chunks.push(String.fromCharCode(...chars));chars=[];}}chunks.push(String.fromCharCode(...chars));return chunks.join('');} +/** Encode the existing canonical ARX2 tuple without interpreting or rewriting artifact bodies. */ +export function encodeNative(tuple){const out=[];function walk(v,depth){if(depth>8)throw Error('frame nesting');if(v===null)out.push(0);else if(Array.isArray(v)){out.push(1);pushVar(out,v.length);for(const x of v)walk(x,depth+1);}else if(typeof v==='string'){const b=encodeString(v);out.push(2);pushVar(out,b.length);for(const x of b)out.push(x);}else if(typeof v==='number'){out.push(3);pushVar(out,v);}else throw Error('unsupported native type');if(out.length>MAX)throw Error('frame too large');}walk(tuple,0);return Uint8Array.from(out);} +/** Bounded decoder for native tuple frames; rejects junk suffixes, overlong integers and nesting. */ +export function decodeNative(bytes){if(bytes.length>MAX)throw Error('frame too large');let p=0,nodes=0;function number(){let n=0,m=1;for(let j=0;j<4;j++){if(p>=bytes.length)throw Error('truncated frame');const b=bytes[p++];n+=(b&127)*m;if(n>MAX)throw Error('oversized integer');if(b<128){if(j&&b===0)throw Error('overlong integer');return n;}m*=128;}throw Error('oversized integer');}function walk(d){if(++nodes>100000||d>8||p>=bytes.length)throw Error('invalid frame');switch(bytes[p++]){case 0:return null;case 1:{const n=number();if(n>bytes.length-p)throw Error('impossible array size');return Array.from({length:n},()=>walk(d+1));}case 2:{const n=number();if(n>bytes.length-p)throw Error('truncated string');const s=decodeString(bytes.subarray(p,p+n));p+=n;return s;}case 3:return number();default:throw Error('unknown frame tag');}}const result=walk(0);if(p!==bytes.length)throw Error('trailing frame data');return result;} diff --git a/tests/arx6-core.test.ts b/tests/arx6-core.test.ts new file mode 100644 index 0000000..712490b --- /dev/null +++ b/tests/arx6-core.test.ts @@ -0,0 +1,11 @@ +// @vitest-environment node +import { describe, it } from "vitest"; +import { buildChecks } from "../experiments/arx6/check.mjs"; + +const checks = await buildChecks(); + +describe("experimental ARX6 frozen core (not registered in the viewer)", () => { + for (const check of checks) { + it(check.name, () => check.run(), 180_000); + } +});