diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 7c11a23..0c6f15f 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -2,19 +2,27 @@ ## Test Fixtures -`tests/fixtures/encode/*.json` are copied from the [toons](https://github.com/alesanfra/toons) -project by Alessandro Sanfratello, licensed under the Apache License 2.0. +`tests/fixtures/encode/*.json` (except `key-folding.json`) are copied from the +[toon-format/spec](https://github.com/toon-format/spec) repository, licensed +under the MIT License. They are the official language-agnostic encode fixture +suite for TOON spec v4.1. No modifications have been made to the fixture +contents. -These fixtures implement the TOON v1.4 encoding specification test suite. -No modifications have been made to the fixture contents. +`key-folding.json` is etoon-local. It started from the +[toons](https://github.com/alesanfra/toons) project by Alessandro Sanfratello +(Apache License 2.0) and carries etoon's own `@`/`$`/`#` sigil-prefix cases. The +spec removed key folding in v4.0, so this file guards etoon's `fold_keys` +extension rather than a spec requirement. ## TOON Specification -The TOON (Token-Oriented Object Notation) format is defined by the -[toon-format/toon](https://github.com/toon-format/toon) project (MIT License). +The TOON (Token-Oriented Object Notation) format is defined by +[toon-format/spec](https://github.com/toon-format/spec) (MIT License); the +reference TypeScript implementation lives in +[toon-format/toon](https://github.com/toon-format/toon) (MIT License). ## License -This project is licensed under the Apache License 2.0, consistent with the -upstream `toons` project to facilitate fixture reuse and potential contributions -back to the TOON ecosystem. +This project is licensed under the Apache License 2.0, which is compatible with +both the MIT-licensed spec fixtures and the Apache 2.0 `toons` fixtures it +reuses. diff --git a/Cargo.lock b/Cargo.lock index 7412ef8..cd39741 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "etoon" -version = "0.6.0" +version = "0.7.0" dependencies = [ "itoa", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 4189f0f..3b401af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "etoon" -version = "0.6.0" +version = "0.7.0" edition = "2021" -description = "Fast TOON (Token-Oriented Object Notation) encoder. 8x faster than toons, 2.7x faster than the official TS SDK." +description = "Fast TOON (Token-Oriented Object Notation) encoder tracking spec v4.1. Up to 7.6x faster than toons, 3.0-8.3x faster than the official TS SDK." license = "Apache-2.0" repository = "https://github.com/coseto6125/etoon" keywords = ["toon", "serialization", "llm", "json", "encoder"] diff --git a/README.md b/README.md index 8d6612a..1de8e0a 100644 --- a/README.md +++ b/README.md @@ -7,25 +7,30 @@ Fast [TOON](https://github.com/toon-format/toon) (Token-Oriented Object Notation) encoder for Python, Rust, and CLI. -**8× faster than `toons`**, **2.7× faster than the official TS SDK**, byte-identical output. +**Up to 7.6× faster than `toons`**, **3.0–8.3× faster than the official TS SDK**, byte-identical output — tracking **TOON spec v4.1**. [中文說明](https://github.com/coseto6125/etoon/blob/main/README.zh-TW.md) ## Performance Per-call encode time across representative payloads (`etoon` = Python/PyO3, -best-of-7 × 2000 calls). `✓` = output byte-identical to etoon; `✗` = the -encoder deviates from the TOON spec (e.g. py-rtoon emits `0.0` where the spec -requires `0`). - -| Payload (encode) | etoon | toons | py-rtoon | @toon-format/toon (TS) | -|-------------------------|---------|----------------|----------------|------------------------| -| 1000 uniform objects | 169 µs | 888 µs (5.2×✓) | 868 µs (5.1×✗) | 455 µs (2.7×✓) | -| deep nested | 123 µs | 291 µs (2.4×✓) | 737 µs (6.0×✗) | 602 µs (4.9×✓) | -| 1000 string records | 93 µs | 747 µs (8.0×✓) | 596 µs (6.4×✓) | 640 µs (6.9×✓) | -| 500 mixed objects | 136 µs | 755 µs (5.5×✓) | 599 µs (4.4×✗) | 1165 µs (8.6×✓) | - -**2.4–8.6× faster** than every other encoder, with **byte-identical, spec-canonical** output (toons and the TS SDK match byte-for-byte; py-rtoon does not). +best-of-7 × 400 calls). `✓` = output byte-identical to etoon; `✗` = the encoder +differs — either it deviates from the spec (py-rtoon emits `0.0` where the spec +requires `0`) or it still implements spec v3.x and expands the v4.1 collapsing +forms into nested blocks. + +| Payload (encode) | etoon | toons | py-rtoon | @toon-format/toon 4.1 (TS) | +|--------------------------|---------|-----------------|-----------------|----------------------------| +| 1000 uniform objects | 171 µs | 848 µs (5.0×✓) | 772 µs (4.5×✗) | 519 µs (3.0×✓) | +| deep nested | 110 µs | 272 µs (2.5×✓) | 662 µs (6.0×✗) | 611 µs (5.6×✓) | +| 1000 string records | 89 µs | 674 µs (7.6×✓) | 540 µs (6.1×✓) | 640 µs (7.2×✓) | +| 500 mixed objects | 140 µs | 680 µs (4.8×✓) | 541 µs (3.9×✗) | 1166 µs (8.3×✓) | +| 1000 nested field groups | 190 µs | 1103 µs (5.8×✗) | 953 µs (5.0×✗) | 586 µs (3.1×✓) | +| 1000 keyed-tabular rows | 105 µs | 679 µs (6.5×✗) | 562 µs (5.4×✗) | 603 µs (5.8×✓) | + +**2.5–8.3× faster** than every other encoder, with **byte-identical, +spec-canonical** output: only the official TS SDK matches etoon byte-for-byte on +all six payloads. The CLI (`… | etoon`) adds process-spawn + pipe I/O on top — fine for shell pipelines / LLM logs, but for in-process use prefer the PyO3 `dumps` (no spawn, @@ -257,14 +262,45 @@ Key optimizations: - **sonic-rs SIMD JSON parser** (~7× faster than serde_json) - **orjson bridge** — single boundary crossing (vs PyO3-based alternatives) - **uniform-order table fast path** — skips 300 key lookups per 50-row table +- **first-row column probe** — an array or empty-object value rules out tabular + form from one element alone, so mixed arrays reach list form in O(columns) - **itoa specialized integer formatting** ## Compatibility -Output is byte-identical to the `toons` Python package (Apache 2.0) and the -official `toon-format/toon` TypeScript SDK. Passes **111/111** TOON spec -fixtures covering primitives, objects, arrays (primitive/tabular/nested/bulleted), -and whitespace. +Tracks **TOON spec v4.1**. Output is byte-identical to the official +`toon-format/toon` TypeScript SDK 4.1, and passes **178/179** cases of the +official [`toon-format/spec`](https://github.com/toon-format/spec) encode +fixture suite — every case except one requiring a non-default `indentSize` +(etoon hardcodes 2 spaces). + +### v4 collapsing forms + +Spec v4.0 added two forms that cut nesting out of common shapes, both +implemented here: + +```bash +# Nested field groups (§9.3) — uniform nested objects become header columns +echo '[{"id":1,"customer":{"name":"Ada","country":"DK"},"total":99}]' | etoon +# orders[1]{id,customer{name,country},total}: +# 1,Ada,DK,99 + +# Keyed tabular (§9.5) — an object of uniform objects becomes a keyed table +echo '{"alpha":{"host":"a.example.com","port":8080},"beta":{"host":"b.example.com","port":9090}}' | etoon +# [2:]{host,port}: +# alpha: a.example.com,8080 +# beta: b.example.com,9090 +``` + +On the benchmark payloads these cut encoded size by **76.6%** (nested field +groups) and **31.9%** (keyed tabular) against the v3.x nested output. + +Spec v4.0 also **removed** key folding and path expansion — folded output is +still valid TOON (dotted keys are literal keys), but no decoder re-nests it, so +`fold_keys` is now an etoon extension rather than a spec option. The upstream +rationale is in [`.out-of-scope/key-folding.md`](https://github.com/toon-format/spec/blob/main/.out-of-scope/key-folding.md): +0.00% token savings on the reference benchmarks, wire ambiguity against literal +dotted keys, and incompatibility with streaming decode. ## Sigil-prefixed keys (`@`, `$`, `#`) @@ -307,13 +343,14 @@ Savings increase with volume — 50 entries reach **35%+** (tiktoken) as the tab ## Advanced options -> These are [TOON spec](https://github.com/toon-format/toon) optional parameters, intended for **programmatic use in your codebase** (Python / Rust library calls). The CLI `| etoon` pipe for LLM workflows uses defaults and does not need these. +> Intended for **programmatic use in your codebase** (Python / Rust library calls). The CLI `| etoon` pipe for LLM workflows uses defaults and does not need these. ```python -# Custom delimiter (when values contain commas) +# Custom delimiter (when values contain commas) — TOON spec §11 etoon.dumps(data, delimiter="|") # or "\t" # Key folding: collapse {a:{b:{c:1}}} → "a.b.c: 1" +# etoon extension — removed from the spec in v4.0, so nothing re-nests it. etoon.dumps(data, fold_keys=True) etoon.dumps(data, fold_keys=True, flatten_depth=2) # partial fold ``` @@ -322,9 +359,13 @@ etoon.dumps(data, fold_keys=True, flatten_depth=2) # partial fold - Integers > 2⁶³ are lossily coerced via f64 (works for most common big integers that happen to be representable; arbitrary-precision is not supported). -- Custom `indent` is hardcoded to 2 spaces (TOON spec default). +- `indentSize` is hardcoded to 2 spaces (TOON spec default). +- Encoder only — etoon does not decode TOON back to JSON. ## License -Apache 2.0. Test fixtures in `tests/fixtures/` are sourced from the -[toons](https://github.com/alesanfra/toons) project (Apache 2.0). +Apache 2.0. Test fixtures in `tests/fixtures/` come from the +[toon-format/spec](https://github.com/toon-format/spec) suite (MIT), except the +etoon-local `key-folding.json` which derives from +[toons](https://github.com/alesanfra/toons) (Apache 2.0). See +[ATTRIBUTION.md](ATTRIBUTION.md). diff --git a/README.zh-TW.md b/README.zh-TW.md index 54a75d4..3897756 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -7,25 +7,28 @@ 快速的 [TOON](https://github.com/toon-format/toon) (Token-Oriented Object Notation) 編碼器，支援 Python、Rust、CLI。 -**比 `toons` 快 8 倍**、**比官方 TS SDK 快 2.7 倍**，輸出 byte-identical。 +**比 `toons` 最多快 7.6 倍**、**比官方 TS SDK 快 3.0–8.3 倍**，輸出 byte-identical，對齊 **TOON spec v4.1**。 [English](https://github.com/coseto6125/etoon/blob/main/README.md) ## 效能 -各種代表性 payload 的單次編碼時間（`etoon` = Python/PyO3，best-of-7 × 2000 次）。 -`✓` = 輸出與 etoon byte-identical；`✗` = 該編碼器偏離 TOON spec（例如 py-rtoon -對整數值浮點輸出 `0.0`，spec 要求 `0`）。 +各種代表性 payload 的單次編碼時間（`etoon` = Python/PyO3，best-of-7 × 400 次）。 +`✓` = 輸出與 etoon byte-identical；`✗` = 該編碼器輸出不同 — 或偏離 spec（py-rtoon +對整數值浮點輸出 `0.0`，spec 要求 `0`），或仍停在 spec v3.x、把 v4.1 的收合形式 +展開成巢狀區塊。 -| Payload（編碼） | etoon | toons | py-rtoon | @toon-format/toon (TS) | -|-------------------------|---------|----------------|----------------|------------------------| -| 1000 筆同構物件 | 169 µs | 888 µs (5.2×✓) | 868 µs (5.1×✗) | 455 µs (2.7×✓) | -| 深層巢狀 | 123 µs | 291 µs (2.4×✓) | 737 µs (6.0×✗) | 602 µs (4.9×✓) | -| 1000 筆字串記錄 | 93 µs | 747 µs (8.0×✓) | 596 µs (6.4×✓) | 640 µs (6.9×✓) | -| 500 筆混合物件 | 136 µs | 755 µs (5.5×✓) | 599 µs (4.4×✗) | 1165 µs (8.6×✓) | +| Payload（編碼） | etoon | toons | py-rtoon | @toon-format/toon 4.1 (TS) | +|--------------------------|---------|-----------------|-----------------|----------------------------| +| 1000 筆同構物件 | 171 µs | 848 µs (5.0×✓) | 772 µs (4.5×✗) | 519 µs (3.0×✓) | +| 深層巢狀 | 110 µs | 272 µs (2.5×✓) | 662 µs (6.0×✗) | 611 µs (5.6×✓) | +| 1000 筆字串記錄 | 89 µs | 674 µs (7.6×✓) | 540 µs (6.1×✓) | 640 µs (7.2×✓) | +| 500 筆混合物件 | 140 µs | 680 µs (4.8×✓) | 541 µs (3.9×✗) | 1166 µs (8.3×✓) | +| 1000 筆巢狀欄位群組 | 190 µs | 1103 µs (5.8×✗) | 953 µs (5.0×✗) | 586 µs (3.1×✓) | +| 1000 筆 keyed 表格列 | 105 µs | 679 µs (6.5×✗) | 562 µs (5.4×✗) | 603 µs (5.8×✓) | -**比所有其他編碼器快 2.4–8.6 倍**，且輸出 **byte-identical、符合 spec canonical** -（toons 與 TS SDK 逐位元組一致；py-rtoon 不符）。 +**比所有其他編碼器快 2.5–8.3 倍**，且輸出 **byte-identical、符合 spec canonical**： +六個 payload 全部逐位元組一致的只有官方 TS SDK。 CLI（`… | etoon`）在此之上多了進程啟動 + pipe I/O — 適合 shell pipeline / LLM log， 但程式內呼叫請優先用 PyO3 的 `dumps`（無進程啟動、無 pipe）。Auto-detect 模式 @@ -256,14 +259,43 @@ Python dict → orjson.dumps → JSON bytes → sonic-rs (SIMD parse) → walk - **sonic-rs SIMD JSON parser**（比 serde_json 快 ~7×） - **orjson bridge** — 只跨一次邊界（vs PyO3-based 方案需多次） - **uniform-order table fast path** — 50 筆 row 省 300 次 key lookup +- **first-row column probe** — 只看第一個元素就能排除 tabular（含 array 或空物件 + 的欄位），混合陣列以 O(欄位數) 落到 list form - **itoa 特化整數格式化** ## 相容性 -輸出與 Python 套件 `toons`（Apache 2.0）和官方 `toon-format/toon` -TypeScript SDK **byte-identical**。通過 **111/111** TOON spec fixtures -涵蓋 primitives、objects、arrays（primitive/tabular/nested/bulleted）、 -whitespace。 +對齊 **TOON spec v4.1**。輸出與官方 `toon-format/toon` TypeScript SDK 4.1 +**byte-identical**，並通過官方 +[`toon-format/spec`](https://github.com/toon-format/spec) encode fixture 套件 +**178/179** — 唯一未過的是需要非預設 `indentSize` 的案例（etoon 固定 2 spaces）。 + +### v4 收合形式 + +spec v4.0 新增兩種把巢狀壓平的形式，兩者都已實作： + +```bash +# 巢狀欄位群組（§9.3）— 同構的巢狀物件變成表頭欄位 +echo '[{"id":1,"customer":{"name":"Ada","country":"DK"},"total":99}]' | etoon +# orders[1]{id,customer{name,country},total}: +# 1,Ada,DK,99 + +# Keyed tabular（§9.5）— 值為同構物件的物件變成帶 key 的表格 +echo '{"alpha":{"host":"a.example.com","port":8080},"beta":{"host":"b.example.com","port":9090}}' | etoon +# [2:]{host,port}: +# alpha: a.example.com,8080 +# beta: b.example.com,9090 +``` + +在 benchmark payload 上，相較 v3.x 的巢狀輸出分別縮小 **76.6%**（巢狀欄位群組） +與 **31.9%**（keyed tabular）。 + +spec v4.0 同時**移除**了 key folding 與 path expansion — 折疊後的輸出仍是合法 +TOON（點號 key 就是字面 key），但沒有 decoder 會把它還原成巢狀，所以 `fold_keys` +現在是 etoon 的擴充而非 spec 選項。上游理由見 +[`.out-of-scope/key-folding.md`](https://github.com/toon-format/spec/blob/main/.out-of-scope/key-folding.md)： +在參考 benchmark 上 token 節省 0.00%、與字面點號 key 產生線上歧義、且與 streaming +decode 不相容。 ## Sigil 前綴 key（`@`、`$`、`#`） @@ -306,13 +338,14 @@ echo '[{"@timestamp":"2026-04-06T12:00:01Z","@message":"POST /api/v1/users 504", ## 進階選項 -> 這些是 [TOON spec](https://github.com/toon-format/toon) 提供的可選參數，適用於 **codebase 內的程式呼叫**（Python / Rust library）。CLI 的 `| etoon` pipe 使用預設值，不需要設定這些。 +> 適用於 **codebase 內的程式呼叫**（Python / Rust library）。CLI 的 `| etoon` pipe 使用預設值，不需要設定這些。 ```python -# 自訂分隔符（資料含逗號時使用） +# 自訂分隔符（資料含逗號時使用）— TOON spec §11 etoon.dumps(data, delimiter="|") # 或 "\t" # Key folding：壓扁 {a:{b:{c:1}}} → "a.b.c: 1" +# etoon 擴充 — spec v4.0 已移除，沒有 decoder 會還原它。 etoon.dumps(data, fold_keys=True) etoon.dumps(data, fold_keys=True, flatten_depth=2) # 部分 fold ``` @@ -321,9 +354,13 @@ etoon.dumps(data, fold_keys=True, flatten_depth=2) # 部分 fold - 超過 2⁶³ 的整數會被降為 f64（多數能整數表示的 1e20 等仍可來回， 但任意精度不支援）。 -- `indent` 固定 2 spaces（TOON spec 預設）。 +- `indentSize` 固定 2 spaces（TOON spec 預設）。 +- 只做編碼 — etoon 不提供 TOON → JSON 的解碼。 ## 授權 Apache 2.0。`tests/fixtures/` 測試檔案來自 -[toons](https://github.com/alesanfra/toons) 專案（Apache 2.0）。 +[toon-format/spec](https://github.com/toon-format/spec) 官方套件（MIT）， +唯獨 etoon 自有的 `key-folding.json` 衍生自 +[toons](https://github.com/alesanfra/toons)（Apache 2.0）。詳見 +[ATTRIBUTION.md](ATTRIBUTION.md)。 diff --git a/benches/compare.py b/benches/compare.py index 6639006..a0ef8ab 100644 --- a/benches/compare.py +++ b/benches/compare.py @@ -42,6 +42,16 @@ def make_payloads() -> dict[str, object]: {"id": i, "meta": {"ok": True, "score": i * 0.1}, "tags": ["a", "b"]} for i in range(500) ], + # v4.1 collapsing forms: nested field groups (§9.3) and keyed tabular + # (§9.5). Encoders still on spec v3.x expand these into nested blocks. + "groups_1000": [ + {"id": i, "customer": {"name": f"n{i}", "country": "DK"}, "total": i} + for i in range(1000) + ], + "keyed_1000": { + f"e{i}": {"host": f"h{i}.example.com", "port": 8000 + i} + for i in range(1000) + }, } diff --git a/package.json b/package.json index fa89357..1398d27 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "@toon-format/toon": "^2.3.0" + "@toon-format/toon": "^4.1.0" } } diff --git a/pyproject.toml b/pyproject.toml index af1f9e4..659c4ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "etoon" dynamic = ["version"] -description = "Fast TOON (Token-Oriented Object Notation) encoder. Byte-identical to the TOON spec, 8x faster than toons and 2.7x faster than the official TS SDK." +description = "Fast TOON (Token-Oriented Object Notation) encoder tracking spec v4.1. Byte-identical to the official TS SDK, up to 7.6x faster than toons." requires-python = ">=3.10" license = "Apache-2.0" readme = "README.md" diff --git a/python/etoon/__init__.py b/python/etoon/__init__.py index ecd7412..72af1af 100644 --- a/python/etoon/__init__.py +++ b/python/etoon/__init__.py @@ -10,7 +10,7 @@ from etoon._etoon import dumps_bytes as _dumps_bytes -__version__ = "0.6.0" +__version__ = "0.7.0" __all__ = ["dumps"] Delimiter = Literal[",", "\t", "|"] @@ -40,14 +40,19 @@ def dumps( fold_keys: If True, collapse single-key object chains into dot-notation keys (``{"a": {"b": 1}}`` → ``"a.b: 1"``). Safe mode: skips folding when a segment needs quoting, and avoids collisions with sibling keys. + An etoon extension: the spec dropped key folding in v4.0, so folded + output is still valid TOON (dotted keys are literal keys) but no + decoder re-nests it. flatten_depth: Maximum chain length when ``fold_keys=True``. ``None`` means unlimited; ``0`` disables folding. - empty_array_bare: If True (default, TOON spec v3.1), emit empty arrays as - canonical ``[]`` / ``key: []`` instead of the legacy ``[0]:`` form. - A bare array *element* that is itself empty (e.g. ``[[], []]``) always + empty_array_bare: If True (default), emit empty arrays as canonical + ``[]`` / ``key: []`` instead of the legacy ``[0]:`` form. A bare + array *element* that is itself empty (e.g. ``[[], []]``) always keeps ``- [0]:`` per spec §9.2; object fields use ``key: []``. - escape_controls: If True (default, TOON spec v3.1), escape control chars - U+0000–U+001F (except ``\n`` ``\r`` ``\t``) as ``\uXXXX`` with lowercase hex. + Setting it False produces output the spec forbids since v3.1. + escape_controls: If True (default), escape control chars U+0000–U+001F + (except ``\n`` ``\r`` ``\t``) as ``\uXXXX`` with lowercase hex. + Setting it False produces output the spec forbids since v3.1. max_depth: Maximum JSON nesting depth for **raw bytes/bytearray input** (default ``1000``). Input nested deeper is rejected with ``ValueError`` before parsing, guarding against a stack overflow diff --git a/src/toon.rs b/src/toon.rs index 60d23d5..2855b96 100644 --- a/src/toon.rs +++ b/src/toon.rs @@ -1,7 +1,7 @@ //! TOON encoder core (sonic-rs backend). //! //! Input: JSON bytes (from orjson.dumps on Python side). -//! Output: TOON string, matching TOON spec v3.1. +//! Output: TOON string, matching TOON spec v4.1. //! //! Delimiter is monomorphized via const generics (`DELIM: u8`) so the //! byte-match inner loops fold away when emitting default-comma output. @@ -9,20 +9,25 @@ use sonic_rs::{Array, JsonContainerTrait, JsonType, JsonValueTrait, Object, Value}; use std::fmt::Write as _; -/// Encoder configuration matching TOON spec v3.1 options. +/// Encoder configuration. The spec's only encoder options are `delimiter` and +/// `indentSize` (§13); the rest are etoon extensions or resource guards. #[derive(Clone, Copy)] pub struct Config { /// Delimiter between array/tabular values. Must be `,`, `\t`, or `|`. pub delimiter: u8, - /// If true, fold single-key object chains into dot-notation keys (safe mode). + /// If true, fold single-key object chains into dot-notation keys (safe + /// mode). An etoon extension: the spec removed key folding in v4.0, so + /// nothing re-nests the output. pub key_folding: bool, /// Max fold depth (segments). None = unlimited. 0 disables folding. pub flatten_depth: Option, - /// If true (v3.1), emit empty arrays as canonical `[]` / `key: []` - /// instead of the legacy `[0]:` / `key[0]:` length-marker form. + /// If true, emit empty arrays as canonical `[]` / `key: []` instead of the + /// legacy `[0]:` / `key[0]:` length-marker form. False emits output the + /// spec has forbidden since v3.1. pub empty_array_bare: bool, - /// If true (v3.1), escape control chars U+0000–U+001F (except the named - /// `\n` `\r` `\t`) as `\uXXXX` with lowercase hex. + /// If true, escape control chars U+0000–U+001F (except the named `\n` `\r` + /// `\t`) as `\uXXXX` with lowercase hex. False emits output the spec has + /// forbidden since v3.1. pub escape_controls: bool, /// Max JSON nesting depth. Input deeper than this is rejected before /// parsing, so neither the sonic-rs DOM parser nor the recursive emitter @@ -90,7 +95,10 @@ fn write_root(v: &Value, cfg: &Config, out: &mut String) { match v.get_type() { JsonType::Object => { let m = v.as_object().unwrap(); - if !m.is_empty() { + if let Some(fields) = keyed_fields(m) { + // Root keyed tabular header is keyless: `[N:]{fields}:` (§9.5). + write_keyed_table::(m, &fields, 0, cfg, out); + } else if !m.is_empty() { // Folding is attempted at the top-level object; nested object // bodies re-apply it via write_value_after_key (spec §13.4). write_object_body::(m, 0, cfg, cfg.key_folding, out); @@ -98,11 +106,11 @@ fn write_root(v: &Value, cfg: &Config, out: &mut String) { } JsonType::Array => { let arr = v.as_array().unwrap(); - // Root empty array: v3.1 canonical bare `[]` (no leading colon). + // Root empty array: canonical bare `[]` (no leading colon). if arr.is_empty() && cfg.empty_array_bare { out.push_str("[]"); } else { - write_array_suffix::(arr, 0, cfg, out); + write_array_suffix::(arr, 0, cfg, true, out); } } _ => write_scalar::(v, cfg, out), @@ -194,6 +202,10 @@ fn write_value_after_key( let child = v.as_object().unwrap(); if child.is_empty() { out.push(':'); + } else if let Some(fields) = keyed_fields(child) { + // Keyed tabular form replaces the nested object body; the + // header attaches directly to the key just written (§9.5). + write_keyed_table::(child, &fields, key_indent, cfg, out); } else { out.push_str(":\n"); // Folding restarts only at a branch point (multi-key object). @@ -206,11 +218,11 @@ fn write_value_after_key( } JsonType::Array => { let arr = v.as_array().unwrap(); - // Object value: v3.1 canonical `key: []`; legacy `key[0]:` otherwise. + // Object value: canonical `key: []`; legacy `key[0]:` otherwise. if arr.is_empty() && cfg.empty_array_bare { out.push_str(": []"); } else { - write_array_suffix::(arr, key_indent, cfg, out); + write_array_suffix::(arr, key_indent, cfg, true, out); } } _ => { @@ -230,8 +242,89 @@ fn write_empty_array_legacy(out: &mut String) { out.push_str("]:"); } -fn write_array_suffix(arr: &Array, indent: usize, cfg: &Config, out: &mut String) { - let _ = cfg; +/// Emit a field list `{f1f2{sub}…}` for a tabular or keyed header, +/// recursing into nested field groups (§9.3). +fn write_field_list(fields: &[Field], cfg: &Config, out: &mut String) { + out.push('{'); + for (i, f) in fields.iter().enumerate() { + if i > 0 { + out.push(DELIM as char); + } + match f { + Field::Leaf(k) => write_key(k, cfg, out), + Field::Group(k, sub) => { + write_key(k, cfg, out); + write_field_list::(sub, cfg, out); + } + } + } + out.push('}'); +} + +/// Emit one row's cells in depth-first pre-order of the field list, so the cell +/// count equals the header's leaf-field count (§9.3). +fn write_row_cells( + m: &Object, + fields: &[Field], + cfg: &Config, + first: &mut bool, + out: &mut String, +) { + for (idx, f) in fields.iter().enumerate() { + match f { + Field::Leaf(k) => { + if !*first { + out.push(DELIM as char); + } + *first = false; + write_scalar::(column_value(m, idx, k).unwrap(), cfg, out); + } + Field::Group(k, sub) => { + let child = column_value(m, idx, k).unwrap().as_object().unwrap(); + write_row_cells::(child, sub, cfg, first, out); + } + } + } +} + +/// Emit the keyed tabular body `[N:]{fields}:` plus one entry row per +/// entry (§9.5). The caller has already written the key, if any — at the root +/// the header is keyless. +fn write_keyed_table( + m: &Object, + fields: &[Field], + indent: usize, + cfg: &Config, + out: &mut String, +) { + out.push('['); + let mut len_buf = itoa::Buffer::new(); + out.push_str(len_buf.format(m.len())); + out.push(':'); + if DELIM != b',' { + out.push(DELIM as char); + } + out.push(']'); + write_field_list::(fields, cfg, out); + out.push(':'); + + for (k, v) in m.iter() { + out.push('\n'); + write_indent(indent + 1, out); + write_key(k, cfg, out); + out.push_str(": "); + let mut first = true; + write_row_cells::(v.as_object().unwrap(), fields, cfg, &mut first, out); + } +} + +fn write_array_suffix( + arr: &Array, + indent: usize, + cfg: &Config, + allow_tabular: bool, + out: &mut String, +) { if arr.is_empty() { write_empty_array_legacy::(out); return; @@ -258,7 +351,31 @@ fn write_array_suffix(arr: &Array, indent: usize, cfg: &Config, return; } - if let Some((keys, uniform_order)) = table_keys(arr) { + // A keyless fields-bearing header is valid only at the document root (§6), + // so an array sitting in list-item position takes list form even when its + // elements would otherwise be tabular-eligible (§9.4). + let shape = if allow_tabular { + table_shape(arr) + } else { + None + }; + + if let Some(Table::Nested(fields)) = &shape { + write_field_list::(fields, cfg, out); + out.push(':'); + for item in arr.iter() { + out.push('\n'); + write_indent(indent + 1, out); + let mut first = true; + write_row_cells::(item.as_object().unwrap(), fields, cfg, &mut first, out); + } + return; + } + + if let Some(Table::Flat(keys, uniform_order)) = shape { + // Writes the field list inline rather than through write_field_list: + // flat tables are the hot path, and routing them through `Field` would + // allocate a tree for a list of names that are all leaves. out.push('{'); for (i, k) in keys.iter().enumerate() { if i > 0 { @@ -319,7 +436,8 @@ fn write_list_item(v: &Value, l: usize, cfg: &Config, out: &mut } JsonType::Array => { out.push(' '); - write_array_suffix::(v.as_array().unwrap(), l, cfg, out); + // List-item position: no keyless tabular header here (§9.4). + write_array_suffix::(v.as_array().unwrap(), l, cfg, false, out); } _ => { out.push(' '); @@ -390,19 +508,17 @@ fn scan_exceeds_depth(bytes: &[u8], max_depth: usize) -> Option { // memchr2 jumps straight to the next `"` or `\`. i += 1; loop { - match memchr::memchr2(b'"', b'\\', &bytes[i..]) { - Some(p) => { - if bytes[i + p] == b'"' { - i += p + 1; - break; - } - // backslash: skip the escaped byte - i += p + 2; - if i >= n { - return None; - } - } - None => return None, // unterminated string + // No `"` or `\` left: the string is unterminated, so there + // is no further nesting to find. + let p = memchr::memchr2(b'"', b'\\', &bytes[i..])?; + if bytes[i + p] == b'"' { + i += p + 1; + break; + } + // backslash: skip the escaped byte + i += p + 2; + if i >= n { + return None; } } } @@ -441,6 +557,143 @@ fn is_scalar(v: &Value) -> bool { !matches!(v.get_type(), JsonType::Object | JsonType::Array) } +/// One column of a tabular header (spec §9.3): a bare leaf field, or a nested +/// field group whose sub-columns are themselves leaves or groups. Nesting depth +/// is unbounded. +enum Field<'a> { + Leaf(&'a str), + Group(&'a str, Vec>), +} + +/// Tabular shape of an array of objects (§9.3). +enum Table<'a> { + /// Every column is uniform-primitive. The flag records whether all rows + /// share the first row's key order, letting cells be emitted by iterating + /// values in place instead of looking each key up. + Flat(Vec<&'a str>, bool), + /// At least one nested-uniform column, emitted as a nested field group. + Nested(Vec>), +} + +/// Value of column `k` in `m`. Rows normally share the header's key order, so +/// try position `idx` first and fall back to a lookup only when it differs. +#[inline] +fn column_value<'a>(m: &'a Object, idx: usize, k: &str) -> Option<&'a Value> { + match m.iter().nth(idx) { + Some((ik, iv)) if ik == k => Some(iv), + _ => m.get(&k), + } +} + +/// First-row probe for the §9.3 column rules: an array value or an empty object +/// disqualifies its column outright, so a mismatch is visible from one object +/// alone. Callers use it to bail in O(columns) before collecting every row; +/// `build_fields` re-checks each column itself. +#[inline] +fn columns_could_be_uniform(first: &Object) -> bool { + !first.is_empty() + && first.iter().all(|(_, v)| match v.get_type() { + JsonType::Array => false, + JsonType::Object => !v.as_object().unwrap().is_empty(), + _ => true, + }) +} + +/// Field tree shared by `objs` (§9.3 column classification), or None when any +/// column is neither uniform-primitive nor nested-uniform. Also used for the +/// entry values of a keyed tabular object (§9.5). +fn build_fields<'a>(objs: &[&'a Object]) -> Option>> { + let first = *objs.first()?; + if first.is_empty() { + return None; + } + for m in &objs[1..] { + if m.len() != first.len() { + return None; + } + } + + let mut fields = Vec::with_capacity(first.len()); + for (idx, (k, v0)) in first.iter().enumerate() { + match v0.get_type() { + JsonType::Object => { + let sub0 = v0.as_object().unwrap(); + if sub0.is_empty() { + return None; + } + let mut subs = Vec::with_capacity(objs.len()); + subs.push(sub0); + for m in &objs[1..] { + let sub = column_value(m, idx, k)?.as_object()?; + if sub.is_empty() { + return None; + } + subs.push(sub); + } + fields.push(Field::Group(k, build_fields(&subs)?)); + } + // Arrays disqualify the column outright; so does any row whose + // value at this key is not a primitive. + JsonType::Array => return None, + _ => { + for m in &objs[1..] { + if !is_scalar(column_value(m, idx, k)?) { + return None; + } + } + fields.push(Field::Leaf(k)); + } + } + } + Some(fields) +} + +fn table_shape<'a>(arr: &'a Array) -> Option> { + if let Some((keys, uniform_order)) = table_keys(arr) { + return Some(Table::Flat(keys, uniform_order)); + } + // Flat detection bails at the first non-primitive value, but a column of + // uniform objects still qualifies as a nested field group (§9.3), so retry + // with the recursive walk. Probe the first element before walking all of + // them: with no object column there is nothing the flat pass missed, and a + // disqualifying value is usually already visible here — that keeps the + // common mixed-array case (a tabular-looking array with one list column) + // from paying for a full scan on its way to list form. + let probe = arr.iter().next()?.as_object()?; + if !columns_could_be_uniform(probe) + || !probe + .iter() + .any(|(_, v)| matches!(v.get_type(), JsonType::Object)) + { + return None; + } + + let mut objs = Vec::with_capacity(arr.len()); + for v in arr.iter() { + objs.push(v.as_object()?); + } + Some(Table::Nested(build_fields(&objs)?)) +} + +/// Field tree when `m` qualifies for keyed tabular form (§9.5): at least two +/// entries, every entry value a non-empty object, one shared key set, and every +/// column uniform-primitive or nested-uniform. +fn keyed_fields<'a>(m: &'a Object) -> Option>> { + if m.len() < 2 { + return None; + } + // Cheap reject before allocating: most objects fail on their first entry. + let probe = m.iter().next()?.1.as_object()?; + if !columns_could_be_uniform(probe) { + return None; + } + let mut objs = Vec::with_capacity(m.len()); + for (_, v) in m.iter() { + objs.push(v.as_object()?); + } + build_fields(&objs) +} + fn table_keys<'a>(arr: &'a Array) -> Option<(Vec<&'a str>, bool)> { let first_v = arr.iter().next()?; let first = first_v.as_object()?; @@ -606,7 +859,7 @@ fn value_needs_quoting(s: &str, escape_controls: bool) -> bool } let bytes = s.as_bytes(); match bytes[0] { - b'-' | b'[' | b'{' | b'"' | b'#' | b' ' | b'\t' => return true, + b'-' | b'#' | b' ' | b'\t' => return true, _ => {} } match bytes[bytes.len() - 1] { @@ -618,7 +871,10 @@ fn value_needs_quoting(s: &str, escape_controls: bool) -> bool // and stays as a separate branch only for DELIM = '|'. for &b in bytes { match b { - b':' | b'\n' | b'\r' | b'\t' | b'"' | b'\\' => return true, + // Brackets and braces anywhere in the value, not just at position 0 + // (spec §7.2) — an unquoted `]` would otherwise close a header the + // decoder is scanning. + b':' | b'\n' | b'\r' | b'\t' | b'"' | b'\\' | b'[' | b']' | b'{' | b'}' => return true, // Other U+0000–U+001F controls force quoting so write_quoted can // emit `\u00XX` (TOON spec v3.1); only when the option is on. _ if escape_controls && b < 0x20 => return true, @@ -632,9 +888,11 @@ fn value_needs_quoting(s: &str, escape_controls: bool) -> bool looks_like_number(bytes) } +/// Numeric-like per spec §7.2: `^[+-]?[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]+)?$`. +/// The leading sign includes `+`, so `"+1"` is quoted and survives round-trip. fn looks_like_number(bytes: &[u8]) -> bool { let mut i = 0; - if bytes[0] == b'-' { + if matches!(bytes[0], b'-' | b'+') { i = 1; if i == bytes.len() { return false; @@ -894,6 +1152,104 @@ mod tests { ); } + // ── Keyed tabular form (spec §9.5) ── + // The happy paths live in tests/fixtures/encode/objects-keyed.json; these + // pin the detection boundaries, where the object must stay nested. + + #[test] + fn test_keyed_table_needs_two_entries() { + // A single entry stays nested — the header would cost more than it saves. + assert_eq!(enc(r#"{"m":{"a":{"x":1}}}"#), "m:\n a:\n x: 1"); + assert_eq!( + enc(r#"{"m":{"a":{"x":1},"b":{"x":2}}}"#), + "m[2:]{x}:\n a: 1\n b: 2" + ); + } + + #[test] + fn test_keyed_table_rejects_non_uniform_columns() { + // Mismatched key sets, a non-object entry, and an array column each + // disqualify the whole object (§9.5 detection). + assert_eq!( + enc(r#"{"m":{"a":{"x":1},"b":{"y":2}}}"#), + "m:\n a:\n x: 1\n b:\n y: 2" + ); + assert_eq!( + enc(r#"{"m":{"a":{"x":1},"b":7}}"#), + "m:\n a:\n x: 1\n b: 7" + ); + assert_eq!( + enc(r#"{"m":{"a":{"x":[1]},"b":{"x":[2]}}}"#), + "m:\n a:\n x[1]: 1\n b:\n x[1]: 2" + ); + } + + #[test] + fn test_keyed_table_not_used_for_array_elements() { + // The `q` column mixes an object with an array, so the array takes list + // form. Its first element is keyed-eligible on its own (two entries, + // one shared key set) but stays nested: array elements are anonymous + // and there is no `- [N:]{…}:` list item (§9.5, §10). + assert_eq!( + enc(r#"{"a":[{"p":{"x":1},"q":{"x":2}},{"p":{"x":3},"q":[9]}]}"#), + "a[2]:\n - p:\n x: 1\n q:\n x: 2\n - p:\n x: 3\n q[1]: 9" + ); + } + + #[test] + fn test_keyed_eligible_column_becomes_nested_field_group() { + // In a tabular column, a keyed-eligible object encodes as a nested + // field group rather than a keyed table (§9.5). + assert_eq!( + enc(r#"{"a":[{"p":{"x":1},"q":{"x":2}}]}"#), + "a[1]{p{x},q{x}}:\n 1,2" + ); + } + + // ── Nested field groups (spec §9.3) ── + + #[test] + fn test_nested_field_group_rejects_empty_object_column() { + // A column of empty objects has no subfields to declare, so the array + // falls back to list form. + assert_eq!(enc(r#"{"a":[{"n":{}},{"n":{}}]}"#), "a[2]:\n - n:\n - n:"); + } + + #[test] + fn test_nested_field_group_rejects_mixed_null_and_object_column() { + // `null` is a primitive, so the column is neither uniform-primitive nor + // nested-uniform (§9.3) and the array takes list form. + assert_eq!( + enc(r#"{"a":[{"n":{"x":1}},{"n":null}]}"#), + "a[2]:\n - n:\n x: 1\n - n: null" + ); + } + + #[test] + fn test_nested_field_group_tolerates_row_key_reordering() { + // Key order may vary per element; cells still follow the header order. + assert_eq!( + enc(r#"{"a":[{"id":1,"g":{"x":1,"y":2}},{"g":{"y":4,"x":3},"id":2}]}"#), + "a[2]{id,g{x,y}}:\n 1,1,2\n 2,3,4" + ); + } + + // ── String quoting (spec §7.2) ── + + #[test] + fn test_quotes_leading_plus_numeric_like_string() { + assert_eq!(enc(r#"{"a":"+1"}"#), r#"a: "+1""#); + assert_eq!(enc(r#"{"a":"+1.5e-3"}"#), r#"a: "+1.5e-3""#); + // A plus that does not form a number stays unquoted. + assert_eq!(enc(r#"{"a":"+x"}"#), "a: +x"); + } + + #[test] + fn test_quotes_brackets_and_braces_anywhere_in_value() { + assert_eq!(enc(r#"{"a":"x[1]"}"#), r#"a: "x[1]""#); + assert_eq!(enc(r#"{"a":"a}b"}"#), r#"a: "a}b""#); + } + // ── Depth guard (P0: prevents sonic-rs/emitter stack overflow) ── #[test] diff --git a/tests/fixtures/encode/arrays-nested.json b/tests/fixtures/encode/arrays-nested.json index f414dc8..69c570e 100644 --- a/tests/fixtures/encode/arrays-nested.json +++ b/tests/fixtures/encode/arrays-nested.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Nested and mixed array encoding - arrays of arrays, mixed type arrays, root arrays", + "description": "Nested and mixed array encoding – arrays of arrays, mixed type arrays, root arrays", "tests": [ { "name": "encodes nested arrays of primitives", @@ -42,19 +42,19 @@ "specSection": "9.1" }, { - "name": "encodes root-level array of uniform objects in tabular format", + "name": "encodes root-level array of uniform objects in tabular form", "input": [{ "id": 1 }, { "id": 2 }], "expected": "[2]{id}:\n 1\n 2", "specSection": "9.3" }, { - "name": "encodes root-level array of non-uniform objects in list format", + "name": "encodes root-level array of non-uniform objects in list form", "input": [{ "id": 1 }, { "id": 2, "name": "Ada" }], "expected": "[2]:\n - id: 1\n - id: 2\n name: Ada", "specSection": "9.4" }, { - "name": "encodes root-level array mixing primitive, object, and array of objects in list format", + "name": "encodes root-level array mixing primitive, object, and array of objects in list form", "input": ["summary", { "id": 1, "name": "Ada" }, [{ "id": 2 }, { "status": "draft" }]], "expected": "[3]:\n - summary\n - id: 1\n name: Ada\n - [2]:\n - id: 2\n - status: draft", "specSection": "9.4" @@ -86,7 +86,7 @@ "specSection": "8" }, { - "name": "uses list format for arrays mixing primitives and objects", + "name": "uses list form for arrays mixing primitives and objects", "input": { "items": [1, { "a": 1 }, "text"] }, @@ -94,12 +94,39 @@ "specSection": "9.4" }, { - "name": "uses list format for arrays mixing objects and arrays", + "name": "uses list form for arrays mixing objects and arrays", "input": { "items": [{ "a": 1 }, [1, 2]] }, "expected": "items[2]:\n - a: 1\n - [2]: 1,2", "specSection": "9.4" + }, + { + "name": "quotes hash-leading string as list item", + "input": { + "items": ["#x", { "a": 1 }] + }, + "expected": "items[2]:\n - \"#x\"\n - a: 1", + "specSection": "7.2" + }, + { + "name": "uses list form for a tabular-eligible array in list-item position", + "input": { + "a": [ + [ + { + "x": 1 + }, + { + "x": 2 + } + ] + ] + }, + "expected": "a[1]:\n - [2]:\n - x: 1\n - x: 2", + "specSection": "9.3", + "note": "A keyless fields-bearing header is valid only at the document root", + "minSpecVersion": "4.1" } ] } diff --git a/tests/fixtures/encode/arrays-objects.json b/tests/fixtures/encode/arrays-objects.json index 9646619..ad8342b 100644 --- a/tests/fixtures/encode/arrays-objects.json +++ b/tests/fixtures/encode/arrays-objects.json @@ -1,10 +1,10 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Arrays of objects encoding - list format for non-uniform objects and complex structures", + "description": "Arrays of objects encoding – list form for non-uniform objects and complex structures", "tests": [ { - "name": "uses list format for objects with different fields", + "name": "uses list form for objects with different fields", "input": { "items": [ { "id": 1, "name": "First" }, @@ -15,14 +15,16 @@ "specSection": "9.4" }, { - "name": "uses list format for objects with nested values", + "name": "uses list form for objects with nested values", "input": { "items": [ - { "id": 1, "nested": { "x": 1 } } + { "id": 1, "nested": { "x": 1 } }, + { "id": 2, "nested": { "y": 2 } } ] }, - "expected": "items[1]:\n - id: 1\n nested:\n x: 1", - "specSection": "9.4" + "expected": "items[2]:\n - id: 1\n nested:\n x: 1\n - id: 2\n nested:\n y: 2", + "specSection": "9.4", + "note": "The nested column mixes key sets {x} and {y}, so it is not nested-uniform (§9.3), so the array uses list form" }, { "name": "preserves field order in list items - array first", @@ -41,7 +43,7 @@ "specSection": "10" }, { - "name": "uses list format for objects containing arrays of arrays", + "name": "uses list form for objects containing arrays of arrays", "input": { "items": [ { "matrix": [[1, 2], [3, 4]], "name": "grid" } @@ -51,7 +53,7 @@ "specSection": "10" }, { - "name": "uses tabular format for nested uniform object arrays", + "name": "uses tabular form for nested uniform object arrays", "input": { "items": [ { "users": [{ "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" }], "status": "active" } @@ -59,10 +61,10 @@ }, "expected": "items[1]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob\n status: active", "specSection": "10", - "note": "YAML-style encoding for list-item objects with tabular array as first field" + "note": "Tabular header on hyphen line with rows at depth +2 and sibling fields at depth +1" }, { - "name": "uses list format for nested object arrays with mismatched keys", + "name": "uses list form for nested object arrays with mismatched keys", "input": { "items": [ { "users": [{ "id": 1, "name": "Ada" }, { "id": 2 }], "status": "active" } @@ -72,7 +74,7 @@ "specSection": "10" }, { - "name": "uses list format for objects with multiple array fields", + "name": "uses list form for objects with multiple array fields", "input": { "items": [{ "nums": [1, 2], "tags": ["a", "b"], "name": "test" }] }, @@ -80,7 +82,7 @@ "specSection": "10" }, { - "name": "uses list format for objects with only array fields", + "name": "uses list form for objects with only array fields", "input": { "items": [{ "nums": [1, 2, 3], "tags": ["a", "b"] }] }, @@ -88,7 +90,7 @@ "specSection": "10" }, { - "name": "encodes objects with empty arrays in list format", + "name": "encodes objects with empty arrays in list form", "input": { "items": [ { "name": "Ada", "data": [] } @@ -144,7 +146,7 @@ "specSection": "9.3" }, { - "name": "uses list format when one object has nested field", + "name": "uses list form when one object has nested field", "input": { "items": [ { "id": 1, "data": "string" }, @@ -155,14 +157,35 @@ "specSection": "9.4" }, { - "name": "uses expanded list for arrays containing empty objects", + "name": "uses list form for arrays containing empty objects", "input": { "items": [{}, {}] }, "expected": "items[2]:\n -\n -", "specSection": "9.4", - "minSpecVersion": "3.2", - "note": "Empty objects {} MUST NOT use tabular form per §9.3; encoded via §9.4 expanded list with bare hyphen markers per §10" + "note": "Empty objects are excluded from tabular detection (§9.3), so the array uses list form with bare hyphen markers (§10)" + }, + { + "name": "encodes a keyed-eligible object in a tabular column as a nested field group", + "input": { + "a": [ + { + "t": { + "p": { + "x": 1 + }, + "q": { + "x": 2 + } + }, + "other": 9 + } + ] + }, + "expected": "a[1]{t{p{x},q{x}},other}:\n 1,2,9", + "specSection": "9.5", + "note": "Keyed tabular form is mandated in object-field and root positions, not in a column", + "minSpecVersion": "4.1" } ] } diff --git a/tests/fixtures/encode/arrays-primitive.json b/tests/fixtures/encode/arrays-primitive.json index 1059c60..5644091 100644 --- a/tests/fixtures/encode/arrays-primitive.json +++ b/tests/fixtures/encode/arrays-primitive.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Primitive array encoding - inline arrays of strings, numbers, booleans", + "description": "Primitive array encoding – inline arrays of strings, numbers, booleans", "tests": [ { "name": "encodes string arrays inline", @@ -98,6 +98,14 @@ }, "expected": "items[3]: \"[5]\",\"- item\",\"{key}\"", "specSection": "9.1" + }, + { + "name": "quotes hash-leading string in inline array", + "input": { + "tags": ["#a", "b"] + }, + "expected": "tags[2]: \"#a\",b", + "specSection": "7.2" } ] } diff --git a/tests/fixtures/encode/arrays-tabular.json b/tests/fixtures/encode/arrays-tabular.json index b213e9a..2365042 100644 --- a/tests/fixtures/encode/arrays-tabular.json +++ b/tests/fixtures/encode/arrays-tabular.json @@ -1,10 +1,10 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Tabular array encoding - arrays of uniform objects with primitive values", + "description": "Tabular array encoding – arrays of uniform objects with primitive values", "tests": [ { - "name": "encodes arrays of uniform objects in tabular format", + "name": "encodes arrays of uniform objects in tabular form", "input": { "items": [ { "sku": "A1", "qty": 2, "price": 9.99 }, @@ -15,7 +15,7 @@ "specSection": "9.3" }, { - "name": "encodes null values in tabular format", + "name": "encodes null values in tabular form", "input": { "items": [ { "id": 1, "value": null }, @@ -68,6 +68,115 @@ }, "expected": "\"\"[2]{id,name}:\n 1,Ada\n 2,Bob", "specSection": "9.3" + }, + { + "name": "quotes hash-leading string in tabular cell", + "input": { + "items": [{ "tag": "#a" }, { "tag": "b" }] + }, + "expected": "items[2]{tag}:\n \"#a\"\n b", + "specSection": "7.2", + "note": "Unquoted, the first row would read as a comment line on decode" + }, + { + "name": "collapses a uniform nested object column into a nested field group", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "expected": "orders[2]{id,customer{name,country},total}:\n 1,Ada,DK,99\n 2,Bob,UK,149", + "specSection": "9.3" + }, + { + "name": "collapses sibling nested field groups with depth-first row layout", + "input": { + "shipments": [ + { "id": "s1", "sender": { "name": "ACME", "city": "Berlin" }, "receiver": { "name": "Globex", "city": "Oslo" } } + ] + }, + "expected": "shipments[1]{id,sender{name,city},receiver{name,city}}:\n s1,ACME,Berlin,Globex,Oslo", + "specSection": "9.3" + }, + { + "name": "collapses nested field groups recursively without a depth cap", + "input": { + "items": [ + { "id": 1, "geo": { "point": { "lat": 1.5, "lon": 2.5 } } }, + { "id": 2, "geo": { "point": { "lat": 3, "lon": 4 } } } + ] + }, + "expected": "items[2]{id,geo{point{lat,lon}}}:\n 1,1.5,2.5\n 2,3,4", + "specSection": "9.3" + }, + { + "name": "uses the active delimiter inside nested field groups", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "options": { "delimiter": "|" }, + "expected": "orders[2|]{id|customer{name|country}|total}:\n 1|Ada|DK|99\n 2|Bob|UK|149", + "specSection": "9.3" + }, + { + "name": "quotes subfield names inside nested field groups per key encoding", + "input": { + "items": [ + { "id": 1, "customer": { "full name": "Ada", "country": "UK" } } + ] + }, + "expected": "items[1]{id,customer{\"full name\",country}}:\n 1,Ada,UK", + "specSection": "9.3" + }, + { + "name": "falls back to list form when nested object keys differ per row", + "input": { + "entries": [ + { "id": 1, "meta": { "a": 1 } }, + { "id": 2, "meta": { "b": 2 } } + ] + }, + "expected": "entries[2]:\n - id: 1\n meta:\n a: 1\n - id: 2\n meta:\n b: 2", + "specSection": "9.3", + "note": "The meta column is not nested-uniform (differing key sets), so the whole array uses §9.4" + }, + { + "name": "falls back to list form when a column mixes null and objects", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada" } }, + { "id": 2, "customer": null } + ] + }, + "expected": "orders[2]:\n - id: 1\n customer:\n name: Ada\n - id: 2\n customer: null", + "specSection": "9.3", + "note": "A null cell is a primitive, so the customer column is neither uniform-primitive nor nested-uniform" + }, + { + "name": "falls back to list form when a nested object contains an array", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "tags": ["x"] } } + ] + }, + "expected": "orders[1]:\n - id: 1\n customer:\n name: Ada\n tags[1]: x", + "specSection": "9.3", + "note": "Array values disqualify a column; rows must contain only primitive cells" + }, + { + "name": "falls back to list form when a nested column contains an empty object", + "input": { + "items": [ + { "id": 1, "meta": {} } + ] + }, + "expected": "items[1]:\n - id: 1\n meta:", + "specSection": "9.3", + "note": "Nested-uniform requires non-empty objects, mirroring the empty-object exclusion for elements" } ] } diff --git a/tests/fixtures/encode/delimiters.json b/tests/fixtures/encode/delimiters.json index 5079916..03c9271 100644 --- a/tests/fixtures/encode/delimiters.json +++ b/tests/fixtures/encode/delimiters.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Delimiter options - tab and pipe delimiters, delimiter-aware quoting", + "description": "Delimiter options – tab and pipe delimiters, delimiter-aware quoting", "tests": [ { "name": "encodes primitive arrays with tab delimiter", diff --git a/tests/fixtures/encode/objects-keyed.json b/tests/fixtures/encode/objects-keyed.json new file mode 100644 index 0000000..7fd4e87 --- /dev/null +++ b/tests/fixtures/encode/objects-keyed.json @@ -0,0 +1,155 @@ +{ + "version": "4.0", + "category": "encode", + "description": "Keyed tabular encoding – objects whose values are uniform objects collapse into keyed headers with entry rows", + "tests": [ + { + "name": "encodes objects of uniform objects in keyed tabular form", + "input": { + "servers": { + "alpha": { "host": "a.example.com", "port": 8080 }, + "beta": { "host": "b.example.com", "port": 9090 } + } + }, + "expected": "servers[2:]{host,port}:\n alpha: a.example.com,8080\n beta: b.example.com,9090", + "specSection": "9.5" + }, + { + "name": "encodes an eligible root object with a keyless keyed header", + "input": { + "alice": { "age": 30, "city": "Berlin" }, + "bob": { "age": 25, "city": "Oslo" } + }, + "expected": "[2:]{age,city}:\n alice: 30,Berlin\n bob: 25,Oslo", + "specSection": "9.5", + "note": "The keyless keyed header is valid only at the root (§5)" + }, + { + "name": "collapses uniform nested object columns inside keyed headers", + "input": { + "regions": { + "eu": { "name": "Europe", "geo": { "lat": 50, "lon": 10 } }, + "us": { "name": "America", "geo": { "lat": 40, "lon": -100 } } + } + }, + "expected": "regions[2:]{name,geo{lat,lon}}:\n eu: Europe,50,10\n us: America,40,-100", + "specSection": "9.5" + }, + { + "name": "orders fields by the first entry value's encounter order", + "input": { + "m": { + "a": { "x": 1, "y": 2 }, + "b": { "y": 4, "x": 3 } + } + }, + "expected": "m[2:]{x,y}:\n a: 1,2\n b: 3,4", + "specSection": "9.5" + }, + { + "name": "uses the active delimiter in keyed headers and entry-row cells", + "input": { + "servers": { + "alpha": { "host": "a.example.com", "port": 8080 }, + "beta": { "host": "b.example.com", "port": 9090 } + } + }, + "options": { "delimiter": "|" }, + "expected": "servers[2:|]{host|port}:\n alpha: a.example.com|8080\n beta: b.example.com|9090", + "specSection": "9.5", + "note": "The keyed marker precedes the delimiter symbol: [2:|]" + }, + { + "name": "quotes entry keys per key encoding", + "input": { + "ids": { + "42": { "v": 1 }, + "my key": { "v": 2 } + } + }, + "expected": "ids[2:]{v}:\n \"42\": 1\n \"my key\": 2", + "specSection": "9.5", + "note": "Entry keys follow §7.3, the same rule as object keys" + }, + { + "name": "quotes entry-row cells containing the active delimiter", + "input": { + "notes": { + "n1": { "text": "a,b" }, + "n2": { "text": "c" } + } + }, + "expected": "notes[2:]{text}:\n n1: \"a,b\"\n n2: c", + "specSection": "9.5" + }, + { + "name": "keeps single-entry objects in nested form", + "input": { + "config": { + "only": { "a": 1, "b": 2 } + } + }, + "expected": "config:\n only:\n a: 1\n b: 2", + "specSection": "9.5", + "note": "Keyed tabular detection requires at least two entries" + }, + { + "name": "keeps objects in nested form when entry values have differing key sets", + "input": { + "envs": { + "dev": { "host": "x" }, + "prod": { "host": "y", "port": 1 } + } + }, + "expected": "envs:\n dev:\n host: x\n prod:\n host: y\n port: 1", + "specSection": "9.5" + }, + { + "name": "keeps objects in nested form when a value is primitive", + "input": { + "m": { + "a": { "x": 1 }, + "b": 2 + } + }, + "expected": "m:\n a:\n x: 1\n b: 2", + "specSection": "9.5" + }, + { + "name": "keeps objects in nested form when an entry value contains an array", + "input": { + "m": { + "a": { "tags": ["x"] }, + "b": { "tags": ["y"] } + } + }, + "expected": "m:\n a:\n tags[1]: x\n b:\n tags[1]: y", + "specSection": "9.5", + "note": "Entry-row cells must be primitive leaves; an array value disqualifies the column" + }, + { + "name": "emits a keyed header on the hyphen line when it is the first field of a list item", + "input": { + "items": [ + { "config": { "a": { "x": 1 }, "b": { "x": 2 } }, "status": "ok" }, + { "status": "down" } + ] + }, + "expected": "items[2]:\n - config[2:]{x}:\n a: 1\n b: 2\n status: ok\n - status: down", + "specSection": "10", + "note": "Entry rows at depth +2, sibling fields at depth +1, mirroring tabular arrays on hyphen lines" + }, + { + "name": "never encodes an anonymous array element in keyed tabular form", + "input": { + "items": [ + { "a": { "x": 1 }, "b": { "x": 2 } }, + 5 + ] + }, + "expected": "items[2]:\n - a:\n x: 1\n b:\n x: 2\n - 5", + "specSection": "9.5", + "note": "The element object is keyed-eligible but has no key; the keyless form is root-only, so it encodes per §10" + } + ] +} diff --git a/tests/fixtures/encode/objects.json b/tests/fixtures/encode/objects.json index 5a26204..a63f92c 100644 --- a/tests/fixtures/encode/objects.json +++ b/tests/fixtures/encode/objects.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Object encoding - simple objects, nested objects, key encoding", + "description": "Object encoding – simple objects, nested objects, key encoding", "tests": [ { "name": "preserves key order in objects", @@ -172,6 +172,23 @@ "expected": "\"\": 1", "specSection": "7.3" }, + { + "name": "quotes non-ASCII key", + "input": { + "café": 1 + }, + "expected": "\"café\": 1", + "specSection": "7.3", + "note": "The unquoted-key pattern is ASCII-only, so every non-ASCII key is quoted" + }, + { + "name": "quotes CJK key", + "input": { + "名前": "x" + }, + "expected": "\"名前\": x", + "specSection": "7.3" + }, { "name": "escapes newline in key", "input": { @@ -202,8 +219,7 @@ "a\u0004b": 1 }, "expected": "\"a\\u0004b\": 1", - "specSection": "7.1", - "minSpecVersion": "3.1" + "specSection": "7.1" }, { "name": "escapes U+001F control character in key via \\uXXXX", @@ -211,8 +227,7 @@ "x\u001fy": 2 }, "expected": "\"x\\u001fy\": 2", - "specSection": "7.1", - "minSpecVersion": "3.1" + "specSection": "7.1" }, { "name": "encodes deeply nested objects", @@ -233,6 +248,50 @@ }, "expected": "user:", "specSection": "8" + }, + { + "name": "encodes __proto__ own property as an ordinary key", + "input": { + "__proto__": "polluted", + "safe": true + }, + "expected": "__proto__: polluted\nsafe: true", + "specSection": "15", + "note": "No key has special meaning, so the encoder emits prototype-named own entries like any other (§8)" + }, + { + "name": "encodes constructor and prototype own properties as ordinary keys", + "input": { + "constructor": 1, + "prototype": 2 + }, + "expected": "constructor: 1\nprototype: 2", + "specSection": "15" + }, + { + "name": "encodes __proto__ as a tabular field name", + "input": { + "rows": [ + { + "__proto__": "a", + "x": 1 + }, + { + "__proto__": "b", + "x": 2 + } + ] + }, + "expected": "rows[2]{__proto__,x}:\n a,1\n b,2", + "specSection": "15" + }, + { + "name": "quotes hash-leading string in object field value", + "input": { + "note": "#x" + }, + "expected": "note: \"#x\"", + "specSection": "7.2" } ] } diff --git a/tests/fixtures/encode/primitives.json b/tests/fixtures/encode/primitives.json index da3b19b..9d6041a 100644 --- a/tests/fixtures/encode/primitives.json +++ b/tests/fixtures/encode/primitives.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Primitive value encoding - strings, numbers, booleans, null", + "description": "Primitive value encoding – strings, numbers, booleans, null", "tests": [ { "name": "encodes safe strings without quotes", @@ -25,8 +25,7 @@ "name": "quotes string that looks like true", "input": "true", "expected": "\"true\"", - "specSection": "7.2", - "note": "String representation of boolean must be quoted" + "specSection": "7.2" }, { "name": "quotes string that looks like false", @@ -63,7 +62,7 @@ "input": "05", "expected": "\"05\"", "specSection": "7.2", - "note": "Leading zeros make it non-numeric" + "note": "Matches the numeric-like pattern, so it MUST be quoted" }, { "name": "escapes newline in string", @@ -99,15 +98,13 @@ "name": "quotes string with array-like syntax", "input": "[3]: x,y", "expected": "\"[3]: x,y\"", - "specSection": "7.2", - "note": "Looks like array header" + "specSection": "7.2" }, { "name": "quotes string starting with hyphen-space", "input": "- item", "expected": "\"- item\"", - "specSection": "7.2", - "note": "Looks like list item marker" + "specSection": "7.2" }, { "name": "quotes single hyphen as object value", @@ -170,6 +167,13 @@ "expected": "hello 👋 world", "specSection": "7.2" }, + { + "name": "encodes emoji inside a quoted string", + "input": "a,🚀", + "expected": "\"a,🚀\"", + "specSection": "7.1", + "note": "Quoting is triggered by the delimiter; the supplementary scalar stays literal UTF-8, never a surrogate escape" + }, { "name": "encodes positive integer", "input": 42, @@ -198,29 +202,27 @@ "name": "encodes negative zero as zero", "input": -0, "expected": "0", - "specSection": "2", - "note": "Negative zero normalizes to zero" + "specSection": "2" }, { - "name": "encodes scientific notation as decimal", + "name": "encodes large integer without exponent notation", "input": 1000000, "expected": "1000000", "specSection": "2", - "note": "1e6 input, but represented as decimal" + "note": "Within the canonical decimal range" }, { - "name": "encodes small decimal from scientific notation", + "name": "encodes small decimal without exponent notation", "input": 0.000001, "expected": "0.000001", "specSection": "2", - "note": "1e-6 input" + "note": "Lower bound of the canonical decimal range" }, { "name": "encodes large number", "input": 100000000000000000000, "expected": "100000000000000000000", - "specSection": "2", - "note": "1e20" + "specSection": "2" }, { "name": "encodes MAX_SAFE_INTEGER", @@ -232,8 +234,7 @@ "name": "encodes repeating decimal with full precision", "input": 0.3333333333333333, "expected": "0.3333333333333333", - "specSection": "2", - "note": "Result of 1/3 in JavaScript" + "specSection": "2" }, { "name": "encodes true", @@ -252,6 +253,26 @@ "input": null, "expected": "null", "specSection": "2" + }, + { + "name": "quotes leading-plus numeric-like string", + "input": "+1", + "expected": "\"+1\"", + "specSection": "7.2", + "note": "Unquoted, a v3 host-parser decoder would read the number 1" + }, + { + "name": "quotes string equal to hash", + "input": "#", + "expected": "\"#\"", + "specSection": "7.2", + "note": "Comment marker at position 0 must be quoted" + }, + { + "name": "quotes string starting with hash", + "input": "#hello", + "expected": "\"#hello\"", + "specSection": "7.2" } ] } diff --git a/tests/fixtures/encode/whitespace.json b/tests/fixtures/encode/whitespace.json index e370d59..1c6eae3 100644 --- a/tests/fixtures/encode/whitespace.json +++ b/tests/fixtures/encode/whitespace.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Whitespace and formatting invariants - no trailing spaces, no trailing newlines", + "description": "Whitespace and formatting invariants – no trailing spaces, no trailing newlines", "tests": [ { "name": "produces no trailing newline at end of output", @@ -9,8 +9,7 @@ "id": 123 }, "expected": "id: 123", - "specSection": "12", - "note": "Output should not end with newline character" + "specSection": "12" }, { "name": "maintains proper indentation for nested structures", @@ -22,11 +21,10 @@ "items": ["a", "b"] }, "expected": "user:\n id: 123\n name: Ada\nitems[2]: a,b", - "specSection": "12", - "note": "2-space indentation, no trailing spaces on any line" + "specSection": "12" }, { - "name": "respects custom indent size option", + "name": "respects custom indentSize option", "input": { "user": { "name": "Ada", @@ -36,9 +34,18 @@ "expected": "user:\n name: Ada\n role: admin", "specSection": "12", "options": { - "indent": 4 + "indentSize": 4 + } + }, + { + "name": "leaves non-ASCII whitespace unquoted", + "input": { + "k": "\u00a0x\u00a0" }, - "note": "4-space indentation for nested objects when indent option is set to 4" + "expected": "k: \u00a0x\u00a0", + "specSection": "7.2", + "note": "The whitespace trigger is U+0020 and U+0009 only", + "minSpecVersion": "4.1" } ] } diff --git a/tests/test_spec_fixtures.py b/tests/test_spec_fixtures.py index 1c4a087..8e93131 100644 --- a/tests/test_spec_fixtures.py +++ b/tests/test_spec_fixtures.py @@ -1,8 +1,9 @@ """ Run the TOON spec fixtures against etoon. -Encode fixtures track the official `toon-format/spec` repo (TOON spec v3.1). -`key-folding.json` additionally carries etoon's `@`/`$`/`#` sigil-prefix cases. +Encode fixtures are the official `toon-format/spec` suite (TOON spec v4.1). +`key-folding.json` is etoon-local: the spec dropped key folding in v4.0, so it +guards etoon's own `fold_keys` extension plus the `@`/`$`/`#` sigil-prefix cases. See ATTRIBUTION.md. Each fixture file contains N test cases with (input, expected) pairs. """ @@ -14,10 +15,10 @@ FIXTURES_DIR = pathlib.Path(__file__).parent / "fixtures" / "encode" -# Fixtures we currently support (tier 1 MVP — no delimiters, no key-folding) SUPPORTED_FIXTURES = { "primitives.json", "objects.json", + "objects-keyed.json", "arrays-primitive.json", "arrays-tabular.json", "arrays-objects.json", @@ -37,7 +38,8 @@ def _collect_cases(): for test in data["tests"]: opts = test.get("options") or {} # Skip tests requiring non-default indent; we hardcode 2-space. - if opts.get("indent", 2) != 2: + # `indent` is the pre-v3.3 spelling, kept for the local fixture. + if opts.get("indentSize", opts.get("indent", 2)) != 2: continue kwargs = {} if "delimiter" in opts: