diff --git a/.gitignore b/.gitignore index 5e5848d..b1efc58 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,6 @@ final-light*.png # Playwright MCP snapshots .playwright-mcp/ + +# Self-hosted server runtime database (local default ./data/agent-render.db; see selfhosted/db.ts) +/data/*.db* diff --git a/AGENTS.md b/AGENTS.md index f549b0c..18870e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,10 +23,10 @@ Core product traits right now: Treat these as core constraints unless the owner explicitly changes the product direction. - The app is a single exported client-side shell, not a backend product. -- Artifact payloads live in the URL fragment, using the compact `#` form where the single tag char identifies the codec: `p` plain, `l` lz, `d` deflate, `a` arx, `b` arx2. Legacy `#agent-render=v1...` links still decode but are no longer emitted. +- Artifact payloads live in the URL fragment, using the compact `#` form where the single tag char identifies the codec: `p` plain, `l` lz, `d` deflate, `a` arx, `b` arx2, `c` arx3. Legacy `#agent-render=v1..` links (arx-family carry an extra `.` segment) still decode but are no longer emitted. - The deployed host should not receive artifact contents as part of the initial page request. - Supported artifact kinds are `markdown`, `code`, `diff`, `csv`, and `json`. -- Supported codecs are `plain`, `lz`, `deflate`, `arx`, and `arx2`. +- Supported codecs are `plain`, `lz`, `deflate`, `arx`, `arx2`, and `arx3`. - The product is zero-retention by host design, not secret-safe in an absolute sense. - Links may still leak through browser history, copied URLs, screenshots, and any future client-side analytics. @@ -84,9 +84,9 @@ The fragment transport is part of the product surface, not an implementation det Current rules: - fragment key: `agent-render` (legacy decode path only; the compact form has no key) -- emitted format: compact `#`, where the single tag char identifies the codec (`p` plain, `l` lz, `d` deflate, `a` arx, `b` arx2); the tag also encodes the active dictionary version for `arx`/`arx2` -- legacy format (still decodable, no longer emitted): `agent-render=v1..` for `plain|lz|deflate`, `agent-render=v1.arx..` for `arx`, and `agent-render=v1.arx2..` for `arx2` -- codecs: `plain`, `lz`, `deflate`, `arx`, and `arx2` +- emitted format: compact `#`, where the single tag char identifies the codec (`p` plain, `l` lz, `d` deflate, `a` arx, `b` arx2, `c` arx3); the compact tag does not carry a dictionary version — arx-family tags imply the build's current dictionary (the build pins the newest supported version and refuses to decode a newer one) +- legacy format (still decodable, no longer emitted): `agent-render=v1..` for `plain|lz|deflate`, `agent-render=v1.arx..` for `arx`, `agent-render=v1.arx2..` for `arx2`, and `agent-render=v1.arx3..` for `arx3` +- codecs: `plain`, `lz`, `deflate`, `arx`, `arx2`, and `arx3` - fragment size budget: `8192` characters - decoded payload budget: `200000` characters - packed wire transport (`p: 1`) is allowed and must decode back to the standard envelope @@ -120,12 +120,13 @@ If you change the payload contract, update the code, docs, examples, and the Ope ### Payload and protocol - `src/lib/payload/schema.ts` - type surface, limits, fragment key, supported kinds/codecs - `src/lib/payload/fragment.ts` - encode/decode logic and transport behavior -- `src/lib/payload/arx-codec.ts` - arx/arx2 codecs: dictionary substitution, tuple overlay, brotli, base76/base1k/baseBMP/base64url encoding +- `src/lib/payload/arx-codec.ts` - arx/arx2/arx3 codecs: dictionary substitution, tuple overlay, brotli, base76/base1k/baseBMP/base64url encoding - `public/arx-dictionary.json` - shared substitution dictionary for the arx codec (served as a static endpoint) - `public/arx-dictionary.json.br` - pre-compressed brotli variant of the dictionary - `public/arx2-dictionary.json` - overlay substitution dictionary for arx2 tuple-envelope transport +- `public/arx2-dictionary.json.br` - pre-compressed brotli variant of the arx2 overlay dictionary - `scripts/compress-dictionary.mjs` - minifies and brotli-compresses the dictionary file -- `scripts/bench-codecs.mjs` - arx/arx2 benchmark gate used by `npm run bench:codecs` +- `scripts/bench-codecs.mjs` - arx/arx2/arx3 benchmark gate used by `npm run bench:codecs` - `scripts/bench-baseline.json` - committed codec benchmark baseline - `src/lib/payload/envelope.ts` - bundle normalization and validation - `src/lib/payload/link-creator.ts` - draft-to-link generation helpers diff --git a/docs/architecture.md b/docs/architecture.md index efb6cd2..b672bae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -96,8 +96,8 @@ The fragment protocol keeps the JSON envelope stable and treats compression stri - `plain` stores base64url-encoded JSON for compatibility and debugging - `lz` stores compressed JSON via `lz-string` when it produces a smaller fragment - `deflate` stores deflate-compressed UTF-8 JSON bytes when it outperforms other codecs -- `arx` applies domain-dictionary substitution, brotli compression (quality 11), and binary-to-text encoding for best-in-class compression. Four wire shapes are candidates: base76 (ASCII, 77 fragment-safe chars), base64url (RFC 4648 `A-Za-z0-9-_` with a `B.` prefix for detection), base1k (Unicode, 1774 chars from U+00A1–U+07FF), and baseBMP (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). The async encoder tries all four and picks the shortest **transport** length (percent-encoded UTF-8 length for non-ASCII), so base64url can win over Unicode encodings on chat-style surfaces. baseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76 for the same compressed bytes, achieving ~70% smaller fragments than deflate on typical payloads (~6.1x compression ratio for 8k markdown). Full pipeline timing is on the order of ~8–14ms for 8k payloads depending on the wire encoding. The substitution dictionary is served as a static file at `/arx-dictionary.json` so agents can fetch it for local compression; a pre-compressed `/arx-dictionary.json.br` variant is also available. The viewer tries the pre-compressed dictionary first on default ARX-family loads, falls back to the JSON file, and only loads external dictionaries when an ARX/ARX2/ARX3 encode or decode path needs them. -- `arx2` keeps the arx compression stack but replaces the JSON envelope with a compact tuple envelope and applies `/arx2-dictionary.json` as an overlay before the shared arx dictionary. The viewer tries `/arx2-dictionary.json.br` first for default overlay loads and falls back to JSON. It is emitted with the compact `b` tag (which encodes the dictionary version) and decodes back to the standard envelope before validation/rendering. +- `arx` applies domain-dictionary substitution, brotli compression (quality 11), and binary-to-text encoding for best-in-class compression. Four wire shapes are candidates: base76 (ASCII, 77 fragment-safe chars), base64url (RFC 4648 `A-Za-z0-9-_` with a `B.` prefix for detection), base1k (Unicode, 1774 chars from U+00A1–U+07FF), and baseBMP (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). The async encoder tries all four and picks the shortest **transport** length (percent-encoded UTF-8 length for non-ASCII), so base64url can win over Unicode encodings on chat-style surfaces. baseBMP produces ~32% fewer characters than base1k and ~60% fewer than base76 for the same compressed bytes, achieving ~70% smaller fragments than deflate on typical payloads (~6.1x compression ratio for 8k markdown). Full pipeline timing is on the order of ~8–14ms for 8k payloads depending on the wire encoding. The substitution dictionary is served as a static file at `/arx-dictionary.json` so agents can fetch it for local compression; a pre-compressed `/arx-dictionary.json.br` variant is also available. The viewer tries the pre-compressed dictionary first on default ARX-family loads, falls back to the JSON file, and only loads external dictionaries when an ARX/ARX2/ARX3 encode or decode path needs them. +- `arx2` keeps the arx compression stack but replaces the JSON envelope with a compact tuple envelope and applies `/arx2-dictionary.json` as an overlay before the shared arx dictionary. The viewer tries `/arx2-dictionary.json.br` first for default overlay loads and falls back to JSON. It is emitted with the compact `b` tag (which identifies the codec but does not carry a dictionary version — it implies the current pinned dictionary) and decodes back to the standard envelope before validation/rendering. - `arx3` uses the same tuple envelope, overlay dictionary, shared arx dictionary, and brotli bytes as arx2, then allows the dense baseBMP wire to win by decoded visible character length. This deliberately optimizes copyable visible URL length for trusted Unicode-preserving surfaces; it is not a stronger compressed-byte format than arx2. - packed wire mode (`p: 1`) shortens transport keys before compression, then unpacks back to the standard envelope during decode - automatic async codec selection tries `arx3 -> arx2 -> arx -> deflate -> lz -> plain`; arx compares packed + non-packed candidates, while arx2/arx3 use tuple envelopes diff --git a/docs/payload-format.md b/docs/payload-format.md index 8b0ab3b..173be54 100644 --- a/docs/payload-format.md +++ b/docs/payload-format.md @@ -17,15 +17,15 @@ Payload contents are untrusted user content. Viewers, agents, and automations sh #c (arx3) ``` -The compact fragment is a single codec tag char followed by the payload. The tag encodes the codec (and, for `arx`/`arx2`/`arx3`, the active dictionary version) so unsupported formats fail cleanly. The legacy `#agent-render=v1...` form still decodes for back-compatibility but is no longer emitted. Fragment URLs can look long because they carry the artifact payload in the browser-only fragment instead of sending it to the host during the page request. +The compact fragment is a single codec tag char followed by the payload. The tag encodes the codec so unsupported formats fail cleanly; the compact tag does not carry a dictionary version — arx-family tags imply the build's current dictionary (the build pins the newest supported version and rejects a newer one). The legacy `#agent-render=v1..` form (arx-family carry an extra `.` segment) still decodes for back-compatibility but is no longer emitted. Fragment URLs can look long because they carry the artifact payload in the browser-only fragment instead of sending it to the host during the page request. Supported codecs: - `plain` - base64url-encoded JSON - `lz` - `lz-string` compressed JSON encoded for URL-safe transport - `deflate` - deflate-compressed UTF-8 JSON bytes encoded as base64url -- `arx` - domain-dictionary substitution + brotli (quality 11) + binary-to-text encoding. The compact `a` tag encodes the active dictionary version so links stay portable across dictionary updates. Four wire shapes are tried and the shortest **transport** size wins (see `computeTransportLength` in `fragment.ts` — non-ASCII Unicode may count longer after percent-encoding): **base76** (ASCII-only, 77 fragment-safe chars), **base64url** (standard RFC 4648 alphabet `A-Za-z0-9-_`, no padding, prefixed with `B.` for detection), **base1k** (Unicode, 1774 chars from U+00A1–U+07FF), and **baseBMP** (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). BaseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76 for the same compressed bytes. BaseBMP payloads are prefixed with a U+FFF0 marker for detection. The viewer’s `arxDecompress` auto-detects the wire shape (including the rare case where a base76 length prefix is also `B.` — it tries base64url first and falls back to base76 if Brotli fails). The substitution dictionary is served at `/arx-dictionary.json` with a pre-compressed `/arx-dictionary.json.br` variant; the viewer tries the `.br` file first on default loads and falls back to JSON. The arx2 overlay dictionary follows the same `.br`-then-JSON default load pattern. -- `arx2` - tuple-envelope transport + arx2 overlay substitution + the shared arx dictionary + brotli (quality 11) + the same four binary-to-text wire shapes. The compact `b` tag encodes the shared arx dictionary version. Existing `arx` links remain valid; async auto-selection keeps arx2 as the conservative transport-measured tuple codec. +- `arx` - domain-dictionary substitution + brotli (quality 11) + binary-to-text encoding. The compact `a` tag identifies the arx codec but does not carry a dictionary version — it implies the build's current pinned dictionary (the build refuses to decode a forward-incompatible newer dictionary). Four wire shapes are tried and the shortest **transport** size wins (see `computeTransportLength` in `fragment.ts` — non-ASCII Unicode may count longer after percent-encoding): **base76** (ASCII-only, 77 fragment-safe chars), **base64url** (standard RFC 4648 alphabet `A-Za-z0-9-_`, no padding, prefixed with `B.` for detection), **base1k** (Unicode, 1774 chars from U+00A1–U+07FF), and **baseBMP** (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). BaseBMP produces ~32% fewer characters than base1k and ~60% fewer than base76 for the same compressed bytes. BaseBMP payloads are prefixed with a U+FFF0 marker for detection. The viewer’s `arxDecompress` auto-detects the wire shape (including the rare case where a base76 length prefix is also `B.` — it tries base64url first and falls back to base76 if Brotli fails). The substitution dictionary is served at `/arx-dictionary.json` with a pre-compressed `/arx-dictionary.json.br` variant; the viewer tries the `.br` file first on default loads and falls back to JSON. The arx2 overlay dictionary follows the same `.br`-then-JSON default load pattern. +- `arx2` - tuple-envelope transport + arx2 overlay substitution + the shared arx dictionary + brotli (quality 11) + the same four binary-to-text wire shapes. The compact `b` tag identifies arx2 but does not carry a dictionary version — it implies the current pinned shared arx dictionary and arx2 overlay. Existing `arx` links remain valid; async auto-selection keeps arx2 as the conservative transport-measured tuple codec. - `arx3` - the same tuple envelope, overlay substitution, shared arx dictionary, and brotli bytes as arx2, with a different selection rule: baseBMP may win by decoded visible character length instead of conservative percent-encoded transport length. This is the compact visible URL mode for trusted surfaces that preserve Unicode fragments. If a platform rewrites, truncates, or previews links aggressively, prefer arx2/base64url or UUID mode instead. The encoder now also supports a packed wire representation (`p: 1`) that shortens key names before compression. Packed mode is transport-only; decoded envelopes normalize back to the standard shape. diff --git a/docs/testing.md b/docs/testing.md index bc5c6f5..ed1857c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -58,8 +58,10 @@ The self-hosted server has its own test suite under `tests/selfhosted/`: - `db.test.ts` — CRUD operations, TTL refresh, expiry cleanup - `validate.test.ts` — Payload validation rules - `ttl.test.ts` — TTL computation and expiry checks +- `api-catalog.test.ts` — RFC 9727 `/.well-known/api-catalog` discovery headers over a spawned export server +- `static-headers.test.ts` — precompressed `.br` static-asset header contract (Content-Type/Content-Encoding/Vary, immutable `_next` caching) -These tests use `// @vitest-environment node` to run with Node.js instead of jsdom, since they depend on `better-sqlite3` (a native module). +Most of these tests use `// @vitest-environment node` to run with Node.js instead of jsdom, since the SQLite-backed tests depend on `better-sqlite3` (a native module); `api-catalog.test.ts` instead runs in the default environment and exercises the self-hosted export server in a spawned child process. They run as part of the standard `npm run test` command. diff --git a/docs/url-fragments.md b/docs/url-fragments.md index f8d703a..1ec93dd 100644 --- a/docs/url-fragments.md +++ b/docs/url-fragments.md @@ -10,7 +10,7 @@ Everything before `#` loads the static app. Everything after `#` is the artifact ## What the parts mean -- The first character after `#` is a single codec tag. Here `c` means the `arx3` codec (and its active dictionary version). +- The first character after `#` is a single codec tag. Here `c` means the `arx3` codec. (The tag does not carry a dictionary version; it implies the current dictionary.) - `` is the encoded artifact bundle. The tag char identifies the codec: @@ -24,7 +24,7 @@ The tag char identifies the codec: #c (arx3) ``` -For `arx`, `arx2`, and `arx3`, the tag also encodes the active dictionary version. +For `arx`, `arx2`, and `arx3`, the compact tag does not carry a dictionary version — it implies the current dictionary (the build pins the newest supported version and rejects a newer one). Only the legacy header below carries an explicit dictionary version. Older links may use the legacy shape, which the viewer still decodes: diff --git a/skills/agent-render-linking/SKILL.md b/skills/agent-render-linking/SKILL.md index 9d4baaa..d99a662 100644 --- a/skills/agent-render-linking/SKILL.md +++ b/skills/agent-render-linking/SKILL.md @@ -34,9 +34,10 @@ character identifying the codec: #c (arx3) ``` -The single tag char encodes the codec (and, for `arx`/`arx2`/`arx3`, the active -dictionary version); the payload follows immediately after it. The legacy -`#agent-render=v1...` form still decodes, but the +The single tag char identifies the codec; for `arx`/`arx2`/`arx3` it implies the +current dictionary but does not carry a dictionary version. The payload follows +immediately after it. The legacy `#agent-render=v1..` form +(arx-family carry an extra `.` segment) still decodes, but the viewer no longer emits it — always build the compact form. Supported codecs: @@ -224,7 +225,7 @@ To use the dictionary for local `arx` encoding: - Base1k uses 1774 Unicode code points (U+00A1–U+07FF, skipping combining diacriticals and soft hyphen). ~10.79 bits/char - Base64url: standard `A-Za-z0-9-_` (no padding), prefix `B.` — ASCII-only, survives Discord/Slack/Teams-style handling better than Unicode-heavy fragments - Base76 uses 77 ASCII fragment-safe characters. ~6.27 bits/char -5. Prepend the tag `a` to form the fragment (the compact tag encodes the active dictionary version, so use the same dictionary version for substitution) +5. Prepend the tag `a` to form the fragment (the compact tag does not carry a dictionary version — it implies the current dictionary, so always substitute using the build's current dictionary) The dictionary includes JSON envelope boilerplate patterns, JSON-escaped Markdown syntax, and programming-language patterns that are already present in the shipped corpus. The viewer tries the pre-compressed dictionary first on default ARX/ARX2/ARX3 encode or decode paths, falls back to the JSON file, and falls back again to its built-in table if external fetches fail. @@ -245,9 +246,9 @@ Then apply substitutions in this order: 2. Fetch and apply `https://agent-render.com/arx-dictionary.json` 3. Brotli-compress at quality 11 4. Try baseBMP, base1k, base64url, and base76; choose the shortest transport representation -5. Prepend the tag `b` (the compact tag encodes the shared arx dictionary version) +5. Prepend the tag `b` (the compact tag does not carry a dictionary version — it implies the current shared arx dictionary and arx2 overlay) -For `arx3`, use the same tuple, substitution, and brotli bytes as arx2, then choose the baseBMP wire when the visible character count is the optimization target. Prepend the tag `c` (the compact tag encodes the shared arx dictionary version). Do not invent a new dictionary entry unless it is backed by corpus evidence and improves the benchmark gate. +For `arx3`, use the same tuple, substitution, and brotli bytes as arx2, then try the same four wire shapes, but measure the baseBMP wire by decoded visible character length (rather than conservative transport length) and pick the shortest candidate — so the dense baseBMP wire can win on Unicode-preserving surfaces. Prepend the tag `c` (the compact tag does not carry a dictionary version — it implies the current shared arx dictionary and arx2 overlay). Do not invent a new dictionary entry unless it is backed by corpus evidence and improves the benchmark gate. ## Practical limits @@ -282,20 +283,20 @@ Use platform-specific link text only on surfaces that support it cleanly. Prefer standard Markdown links: ```md -[Short summary](https://agent-render.com/#agent-render=...) +[Short summary](https://agent-render.com/#) ``` Examples: -- `[Weekly report](https://agent-render.com/#agent-render=...)` -- `[Config diff](https://agent-render.com/#agent-render=...)` -- `[CSV snapshot](https://agent-render.com/#agent-render=...)` +- `[Weekly report](https://agent-render.com/#)` +- `[Config diff](https://agent-render.com/#)` +- `[CSV snapshot](https://agent-render.com/#)` ### Telegram Prefer HTML links because OpenClaw Telegram outbound text uses `parse_mode: "HTML"`. ```html -Short summary +Short summary ``` ### Slack @@ -303,7 +304,7 @@ Prefer HTML links because OpenClaw Telegram outbound text uses `parse_mode: "HTM Prefer Slack `mrkdwn` link syntax: ```text - +|Short summary> ``` ### All other OpenClaw chat surfaces diff --git a/skills/selfhosted-agent-render/SKILL.md b/skills/selfhosted-agent-render/SKILL.md index 31121c4..f845dba 100644 --- a/skills/selfhosted-agent-render/SKILL.md +++ b/skills/selfhosted-agent-render/SKILL.md @@ -1,6 +1,6 @@ --- name: selfhosted-agent-render -description: Create and manage agent-render artifacts via a self-hosted UUID-based server. Use when an agent needs public/share-friendly rendered artifacts through short UUID links instead of fragment-encoded URLs. Ideal for public/social sharing, corporate proxy/link-scanning environments, payloads that exceed the ~8 KB fragment budget, platforms that mangle long URLs, or when the agent and viewer run on the same machine. Supports markdown, code, diffs, CSV, and JSON — same artifact kinds and envelope validation as the fragment-based product. The self-hosted server stores payloads in SQLite with a 24-hour sliding TTL. +description: Create and manage agent-render artifacts via a self-hosted UUID-based server. Use when an agent needs public/share-friendly rendered artifacts through short UUID links instead of fragment-encoded URLs. Ideal for public/social sharing, corporate proxy/link-scanning environments, payloads that exceed the ~8 KB fragment budget, platforms that mangle long URLs, or when the agent and viewer run on the same machine. Supports markdown, code, diffs, CSV, and JSON — same artifact kinds as the fragment-based product (the server stores the payload string after a length/non-empty check; full envelope validation happens client-side when the viewer renders). The self-hosted server stores payloads in SQLite with a 24-hour sliding TTL. --- # Self-Hosted Agent Render @@ -49,7 +49,7 @@ Response (`201`): } ``` -The `payload` field is the same payload string used in fragment links — the compact fragment body after `#` (a single codec tag char followed by the payload). Use the same envelope format and codecs (`plain`, `lz`, `deflate`, `arx`, `arx2`, `arx3`) described in the `agent-render-linking` skill. The legacy `agent-render=v1..` form is also accepted for back-compatibility. +The `payload` field is the same payload string used in fragment links — the compact fragment body after `#` (a single codec tag char followed by the payload). Use the same envelope format and codecs (`plain`, `lz`, `deflate`, `arx`, `arx2`, `arx3`) described in the `agent-render-linking` skill. The legacy `agent-render=v1..` form (arx-family carry an extra `.` segment) is also accepted for back-compatibility. ### Read an artifact diff --git a/src/lib/payload/arx-codec.ts b/src/lib/payload/arx-codec.ts index 2aa0ca4..859a846 100644 --- a/src/lib/payload/arx-codec.ts +++ b/src/lib/payload/arx-codec.ts @@ -1340,7 +1340,7 @@ export async function arxCompressUnicode(json: string): Promise { /** * Compress with the arx pipeline using baseBMP high-density encoding. - * Produces ~32% fewer characters than base1k (~55% fewer than base76) + * Produces ~32% fewer characters than base1k (~60% fewer than base76) * by using ~62k safe BMP code points (~15.92 bits/char). */ export async function arxCompressBMP(json: string): Promise { diff --git a/src/lib/payload/schema.ts b/src/lib/payload/schema.ts index f3efef4..f2fd816 100644 --- a/src/lib/payload/schema.ts +++ b/src/lib/payload/schema.ts @@ -9,8 +9,9 @@ export type ArtifactKind = (typeof artifactKinds)[number]; export type PayloadCodec = (typeof codecs)[number]; // Compact fragment header: a single URL-unreserved tag char replaces the legacy -// `agent-render=v1...` prefix. The tag encodes (wire version 1, codec, and -// dictVersion); arx-family tags imply dictVersion 1. The payload stays self-describing (base64url +// `agent-render=v1...` prefix. The tag encodes wire version 1 and codec; it +// does not carry a dictionary version — arx-family tags imply the build's current pinned dictionary +// (version 1). The payload stays self-describing (base64url // `B.` prefix, baseBMP U+FFF0 marker, base76/base1k length prefix), so the alphabet is not in the // header. Tags come from the RFC-3986 unreserved set so they never percent-escape, and none can // begin the legacy `agent-render=` literal, which keeps the two header forms unambiguous on decode.