From 1966a0ca25a89a65cf64fa49c383f1aa1fb16ad7 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 13:04:29 +0200 Subject: [PATCH 01/10] Write a colon or hash that is not an indicator as a plain YAML scalar (#80) YamlWriter quoted every value containing `:`, `#`, `"` or `\`, so URLs like `https://xemantic.com/contact` or `C#` came out needlessly double-quoted. A `:` is now only a mapping indicator when followed by a space or at the end of the value, and a `#` only starts a comment after a space; other occurrences, and inner `"` / `\`, stay plain. YAML 1.1 sexagesimals (`12:30`) remain quoted, since Psych and PyYAML type them as numbers. The parser's lenient reading of a mapping indicator inside a plain value (`k: Note: see`) is documented and pinned as a DIVERGENCE. Closes #80 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commonMain/kotlin/YamlParser.kt | 5 +++ .../src/commonMain/kotlin/YamlWriter.kt | 26 ++++++++--- .../src/commonTest/kotlin/YamlParserTest.kt | 18 ++++++++ .../commonTest/kotlin/YamlRoundTripTest.kt | 44 +++++++++++++++++++ .../src/commonTest/kotlin/YamlWriterTest.kt | 2 +- 5 files changed, 89 insertions(+), 6 deletions(-) diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index de47a5a..4aaacbb 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -56,6 +56,11 @@ import kotlinx.coroutines.flow.FlowCollector * directives are not recognised — a multi-document stream is outside the * subset. * + * DIVERGENCE (lenient plain scalars): a plain value containing a mapping + * indicator — `k: Note: see`, `k: ends:` — is read as the string after the + * first `: `, where YAML (Psych, PyYAML) rejects the line. This does not + * make such a value safe to write plain: [YamlWriter] still quotes it. + * * Streaming: a `key: value` line commits on its newline. A bare `key:` (or * `-`) is held for one line to decide between a nested block and a null * value. A block scalar is buffered until it closes — never past the diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 682d35a..145a714 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -35,8 +35,9 @@ import com.xemantic.markanywhere.SemanticEvent * - a scalar with a `type` (`bool`, `int`, `float`, `null`, `timestamp`) is * written bare; a string is written plain unless a plain scalar would * re-parse as something else — reserved literals, numbers, timestamps, - * a leading indicator, `: `, ` #`, surrounding whitespace, a control - * character — in which case it is double-quoted with the YAML escapes; + * YAML 1.1 sexagesimals (`12:30`), a leading indicator, `: ` or a + * trailing `:`, ` #`, surrounding whitespace, a control character — in + * which case it is double-quoted with the YAML escapes; * - a multi-line string becomes a literal block scalar (`|`, `|-`, `|+` * according to its trailing newlines) unless its first line starts with * whitespace or it has no content, which fall back to a quoted scalar; @@ -244,14 +245,29 @@ private const val YAML_INDICATORS = "-?:,[]{}#&*!|>'\"%@`" private val Char.isYamlControl: Boolean get() = this < ' ' || this == '\u007f' || this == '\u0085' || this == '\u2028' || this == '\u2029' +// A YAML 1.1 base-60 number (`12:30`, `-1:30`, `190:20:30.15`): YAML 1.2 — +// and so `yamlScalarType` — reads it as a string, but Psych (Jekyll) and +// PyYAML type it as an int / float, so the writer keeps it quoted, in the +// same spirit as the YAML 1.1 booleans (`yes`, `on`). Anchored, see the +// note on the patterns in `YamlParser.kt`. +private val YAML_1_1_SEXAGESIMAL = Regex("""^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+(\.[0-9_]*)?$""") + // A plain scalar the parser would not read back as the same string: one it -// would type (`yamlScalarType`), or whose shape is an indicator, a comment, -// a mapping colon, surrounding whitespace or a control character. +// would type (`yamlScalarType`), or whose shape is an indicator, a comment +// (`#` after a space), a mapping colon (`:` before a space or at the end), +// surrounding whitespace or a control character. A `:` or `#` anywhere else +// is plain content (`https://x/y`, `C#`), as is an inner `"` or `\`. private fun needsQuoting(s: String): Boolean { if (s.isEmpty()) return true if (s.first().isWhitespace() || s.last().isWhitespace()) return true if (s[0] in YAML_INDICATORS) return true - for (c in s) if (c == ':' || c == '#' || c == '"' || c == '\\' || c.isYamlControl) return true + for (i in s.indices) { + val c = s[i] + if (c.isYamlControl) return true + if (c == ':' && (i + 1 == s.length || s[i + 1] == ' ')) return true + if (c == '#' && s[i - 1] == ' ') return true + } + if (YAML_1_1_SEXAGESIMAL.matches(s)) return true return yamlScalarType(s) != null } diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 7b46280..33d16db 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -704,6 +704,24 @@ class YamlParserTest { } } + @Test + fun `should DIVERGENCE accept a mapping indicator inside a plain value`() = runTest { + // given — YAML readers (Psych, PyYAML) reject both lines + val textFlow = """ + note: Note: see + end: ends: + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "note") { +"Note: see" } + "entry"("key" to "end") { +"ends:" } + } + } + @Test fun `should be ready for a new document after finish`() = runTest { // given — the first document ends with a pending `key:` that finish diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index da884ba..a1de57c 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -64,6 +64,50 @@ class YamlRoundTripTest { } } + @Test + fun `should write a colon or hash that is not an indicator plain`() = runTest { + // given — `:` not followed by a space and `#` not after a space are + // plain-scalar content for every YAML reader (issue #80) + val values = listOf( + "https://xemantic.com/contact", "a:b", "C#", "a#b", "say \"hi\"", "C:\\path", + ) + for (value in values) { + val source = "k: $value\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs source + reparsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "k") { +value } + } + } + } + + @Test + fun `should keep quoting a colon or hash that is an indicator and YAML 1_1 sexagesimals`() = runTest { + // given — a mapping colon, a comment, a leading indicator, and base-60 + // numbers a YAML 1.1 reader (Psych, PyYAML) would type as int / float + val values = listOf( + "Note: see", "ends:", "a #b", "12:30", "-1:30", "190:20:30.15", "#tag", ": x", + ) + for (value in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: \"$value\"\n" + reparsed.mergeAdjacentText() sameAs events + } + } + @Test fun `should round-trip keys that need quoting`() = runTest { // given diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt index 5cd9634..982d32c 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt @@ -241,7 +241,7 @@ class YamlWriterTest { dash: "- not an item" padded: " spaced " tabbed: "a\tb" - url: "http://x#y" + url: http://x#y page.section: News """.trimIndent() + "\n" } From 7a7cfb09142124ab0781be219a78317b1e6bddde Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 15:22:59 +0200 Subject: [PATCH 02/10] Quote plain scalars a YAML 1.1 reader would type YamlWriter already quoted YAML 1.1 sexagesimals (`12:30`), but Psych (Jekyll) and PyYAML also type other shapes YAML 1.2 reads as strings: booleans / nulls in any letter case (`yEs`, `nULL`), integers and floats with `_` / `,` separators, binary, signed hex, leading-zero octal, floats with a leading or trailing dot, looser timestamps (`2024-5-1`, offsets without a colon), and PyYAML's `=` / `<<` tags. The new `isYaml11Typed` covers the union, and both values and keys that match it are now double-quoted. Also guards the `#` comment check against index 0, and documents that a sequence item `- Note: see` is a compact mapping, not a lenient scalar. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commonMain/kotlin/YamlParser.kt | 50 +++++++++++++++++-- .../src/commonMain/kotlin/YamlWriter.kt | 22 +++----- .../commonTest/kotlin/YamlRoundTripTest.kt | 40 ++++++++++++++- 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 4aaacbb..8757bfe 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -56,10 +56,12 @@ import kotlinx.coroutines.flow.FlowCollector * directives are not recognised — a multi-document stream is outside the * subset. * - * DIVERGENCE (lenient plain scalars): a plain value containing a mapping - * indicator — `k: Note: see`, `k: ends:` — is read as the string after the - * first `: `, where YAML (Psych, PyYAML) rejects the line. This does not - * make such a value safe to write plain: [YamlWriter] still quotes it. + * DIVERGENCE (lenient plain scalars): a mapping entry's plain value + * containing a mapping indicator — `k: Note: see`, `k: ends:` — is read as + * the string after the first `: `, where YAML (Psych, PyYAML) rejects the + * line. A sequence item is not lenient: `- Note: see` is a compact mapping + * (an item holding the entry `Note`), as in YAML. This does not make such a + * value safe to write plain: [YamlWriter] still quotes it. * * Streaming: a `key: value` line commits on its newline. A bare `key:` (or * `-`) is held for one line to decide between a nested block and a null @@ -672,3 +674,43 @@ internal fun yamlScalarType(text: String): String? = when { YAML_TIMESTAMP.matches(text) -> "timestamp" else -> null } + +// YAML 1.1 plain-scalar shapes that YAML 1.2 — and so [yamlScalarType] — +// reads as a string, but that a YAML 1.1 reader still types: Psych (Jekyll) +// and PyYAML. The union of both is covered, erring on the side of too much: +// - booleans / nulls in any letter case (`yEs`, `nULL` — Psych ignores case); +// - integers with `_` or `,` separators, binary, signed hex, leading-zero +// octal (`1_000`, `1,000`, `0b101`, `+0x1F`, `0755`); +// - floats with separators, a trailing or leading dot (`1_000.5`, `1.`, `.5`); +// - base-60 numbers (`12:30`, `+1:30`, `190:20:30.15`); +// - timestamps Psych accepts beyond the 1.2 shape: one-digit month / day +// (`2024-5-1`), an offset without a colon (`+0100`); +// - PyYAML's value / merge tags (`=`, `<<`), which its SafeLoader cannot +// construct. +// Anchored, see the note above. +private val YAML_1_1_WORDS = setOf("yes", "no", "true", "false", "on", "off", "null") + +private val YAML_1_1_NUMBER = Regex( + """^(?:[-+]?0b[01_,]+|[-+]?0x[0-9a-fA-F_,]+|[-+]?[0-9][0-9_,]*""" + + """|[-+]?(?:[0-9][0-9_,]*)?\.[0-9_]*(?:[eE][-+]?[0-9]+)?""" + + """|[-+]?[0-9][0-9_,]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?)$""" +) + +private val YAML_1_1_TIMESTAMP = Regex( + """^-?[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}""" + + """(?:(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9]{1,2}:?(?:[0-9]{2})?))?)?$""" +) + +// Whether a YAML 1.1 reader (Psych, PyYAML) would type this plain scalar, +// or refuse it, where [yamlScalarType] reads a string. The writer quotes +// such a value, in the same spirit as the 1.1 booleans in [YAML_BOOLS]. +// Every shape starts with a sign, a digit or a dot, or is one of a few +// words, so the regexes only run on a string that can match them. +internal fun isYaml11Typed(text: String): Boolean { + if (text.isEmpty()) return false + if (text == "=" || text == "<<") return true + if (text.length <= 5 && text.lowercase() in YAML_1_1_WORDS) return true + val first = text[0] + if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return false + return YAML_1_1_NUMBER.matches(text) || YAML_1_1_TIMESTAMP.matches(text) +} diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 145a714..6c09b79 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -35,8 +35,9 @@ import com.xemantic.markanywhere.SemanticEvent * - a scalar with a `type` (`bool`, `int`, `float`, `null`, `timestamp`) is * written bare; a string is written plain unless a plain scalar would * re-parse as something else — reserved literals, numbers, timestamps, - * YAML 1.1 sexagesimals (`12:30`), a leading indicator, `: ` or a - * trailing `:`, ` #`, surrounding whitespace, a control character — in + * shapes a YAML 1.1 reader types (`12:30`, `1_000`, `yEs`), a leading + * indicator, `: ` or a trailing `:`, ` #`, surrounding whitespace, a + * control character — in * which case it is double-quoted with the YAML escapes; * - a multi-line string becomes a literal block scalar (`|`, `|-`, `|+` * according to its trailing newlines) unless its first line starts with @@ -232,7 +233,7 @@ public class YamlWriter( // re-detectable. The character rules are shared with that check. private fun renderKey(key: String?): String { val k = key ?: "" - return if (isIdentifierKey(k) && yamlScalarType(k) == null) k else quoted(k) + return if (isIdentifierKey(k) && yamlScalarType(k) == null && !isYaml11Typed(k)) k else quoted(k) } } @@ -245,15 +246,9 @@ private const val YAML_INDICATORS = "-?:,[]{}#&*!|>'\"%@`" private val Char.isYamlControl: Boolean get() = this < ' ' || this == '\u007f' || this == '\u0085' || this == '\u2028' || this == '\u2029' -// A YAML 1.1 base-60 number (`12:30`, `-1:30`, `190:20:30.15`): YAML 1.2 — -// and so `yamlScalarType` — reads it as a string, but Psych (Jekyll) and -// PyYAML type it as an int / float, so the writer keeps it quoted, in the -// same spirit as the YAML 1.1 booleans (`yes`, `on`). Anchored, see the -// note on the patterns in `YamlParser.kt`. -private val YAML_1_1_SEXAGESIMAL = Regex("""^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+(\.[0-9_]*)?$""") - // A plain scalar the parser would not read back as the same string: one it -// would type (`yamlScalarType`), or whose shape is an indicator, a comment +// would type (`yamlScalarType`) or a YAML 1.1 reader would (`isYaml11Typed`), +// or whose shape is an indicator, a comment // (`#` after a space), a mapping colon (`:` before a space or at the end), // surrounding whitespace or a control character. A `:` or `#` anywhere else // is plain content (`https://x/y`, `C#`), as is an inner `"` or `\`. @@ -265,10 +260,9 @@ private fun needsQuoting(s: String): Boolean { val c = s[i] if (c.isYamlControl) return true if (c == ':' && (i + 1 == s.length || s[i + 1] == ' ')) return true - if (c == '#' && s[i - 1] == ' ') return true + if (c == '#' && i > 0 && s[i - 1] == ' ') return true } - if (YAML_1_1_SEXAGESIMAL.matches(s)) return true - return yamlScalarType(s) != null + return yamlScalarType(s) != null || isYaml11Typed(s) } private fun quoted(s: String): String = buildString { diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index a1de57c..59c1e58 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -91,7 +91,7 @@ class YamlRoundTripTest { // given — a mapping colon, a comment, a leading indicator, and base-60 // numbers a YAML 1.1 reader (Psych, PyYAML) would type as int / float val values = listOf( - "Note: see", "ends:", "a #b", "12:30", "-1:30", "190:20:30.15", "#tag", ": x", + "Note: see", "ends:", "a #b", "12:30", "+1:30", "190:20:30.15", "#tag", ": x", ) for (value in values) { val events = semanticEvents { @@ -108,6 +108,44 @@ class YamlRoundTripTest { } } + @Test + fun `should quote a string a YAML 1_1 reader would type`() = runTest { + // given — plain scalars YAML 1.2 reads as strings, but Psych (Jekyll) + // or PyYAML type as a number, time, boolean or null, or refuse + val values = listOf( + "2024-05-01T10:00:00+0100", "2024-05-01 10:00:00 +0100", "2024-5-1", + "1_000", "1,000", "0b101", "+0x1F", "0755", "1_000.5", "1.0e+5", ".e+4", + "yEs", "oFF", "nULL", "=", "<<", + ) + for (value in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: \"$value\"\n" + reparsed.mergeAdjacentText() sameAs events + } + } + + @Test + fun `should quote a key a YAML 1_1 reader would type`() = runTest { + // given — Psych reads booleans in any letter case + val events = semanticEvents { + "entry"("key" to "yEs") { +"v" } + } + + // when + val rendered = events.renderYaml() + + // then + rendered sameAs "\"yEs\": v\n" + } + @Test fun `should round-trip keys that need quoting`() = runTest { // given From 65e5814e3e55d1058729b4af6b54bb8b3b198e95 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 15:56:53 +0200 Subject: [PATCH 03/10] Escape every character YAML does not allow and quote more YAML 1.1 words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YamlWriter now escapes every character outside YAML's printable set (YAML 1.2 §5.1): C1 controls, U+FFFE / U+FFFF and lone surrogates, which Psych and PyYAML refuse anywhere in a document, even in a block scalar. Surrogate pairs (emoji) stay plain. isYaml11Typed also covers the one-letter booleans `y` / `n` (YAML 1.1, go-yaml v2) and the special floats in any letter case (`.Nan`, `+.InF`), deriving its word set from the parser's reserved literals. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commonMain/kotlin/YamlParser.kt | 19 ++++-- .../src/commonMain/kotlin/YamlWriter.kt | 48 +++++++++----- .../commonTest/kotlin/YamlRoundTripTest.kt | 66 +++++++++++++++++-- .../src/commonTest/kotlin/YamlWriterTest.kt | 4 +- 4 files changed, 108 insertions(+), 29 deletions(-) diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 8757bfe..e168355 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -678,17 +678,24 @@ internal fun yamlScalarType(text: String): String? = when { // YAML 1.1 plain-scalar shapes that YAML 1.2 — and so [yamlScalarType] — // reads as a string, but that a YAML 1.1 reader still types: Psych (Jekyll) // and PyYAML. The union of both is covered, erring on the side of too much: -// - booleans / nulls in any letter case (`yEs`, `nULL` — Psych ignores case); -// - integers with `_` or `,` separators, binary, signed hex, leading-zero -// octal (`1_000`, `1,000`, `0b101`, `+0x1F`, `0755`); -// - floats with separators, a trailing or leading dot (`1_000.5`, `1.`, `.5`); +// - booleans / nulls in any letter case (`yEs`, `nULL` — Psych ignores case), +// and the one-letter booleans `y` / `n` (the YAML 1.1 spec, go-yaml v2); +// - the special floats in any letter case (`.Nan`, `+.InF` — Psych again); +// - integers with `_` or `,` separators, binary, signed hex +// (`1_000`, `1,000`, `0b101`, `+0x1F`); +// - floats with separators or a digit-less mantissa (`1_000.5`, `1,000.5`, +// `.e+4`); // - base-60 numbers (`12:30`, `+1:30`, `190:20:30.15`); // - timestamps Psych accepts beyond the 1.2 shape: one-digit month / day // (`2024-5-1`), an offset without a colon (`+0100`); // - PyYAML's value / merge tags (`=`, `<<`), which its SafeLoader cannot // construct. // Anchored, see the note above. -private val YAML_1_1_WORDS = setOf("yes", "no", "true", "false", "on", "off", "null") +private val YAML_1_1_WORDS = + (YAML_BOOLS + YAML_NULLS).mapTo(mutableSetOf()) { it.lowercase() } + + setOf("y", "n", ".inf", "+.inf", "-.inf", ".nan") + +private val YAML_1_1_WORD_MAX_LENGTH = YAML_1_1_WORDS.maxOf { it.length } private val YAML_1_1_NUMBER = Regex( """^(?:[-+]?0b[01_,]+|[-+]?0x[0-9a-fA-F_,]+|[-+]?[0-9][0-9_,]*""" + @@ -709,7 +716,7 @@ private val YAML_1_1_TIMESTAMP = Regex( internal fun isYaml11Typed(text: String): Boolean { if (text.isEmpty()) return false if (text == "=" || text == "<<") return true - if (text.length <= 5 && text.lowercase() in YAML_1_1_WORDS) return true + if (text.length <= YAML_1_1_WORD_MAX_LENGTH && text.lowercase() in YAML_1_1_WORDS) return true val first = text[0] if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return false return YAML_1_1_NUMBER.matches(text) || YAML_1_1_TIMESTAMP.matches(text) diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 6c09b79..f371bc4 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -37,8 +37,8 @@ import com.xemantic.markanywhere.SemanticEvent * re-parse as something else — reserved literals, numbers, timestamps, * shapes a YAML 1.1 reader types (`12:30`, `1_000`, `yEs`), a leading * indicator, `: ` or a trailing `:`, ` #`, surrounding whitespace, a - * control character — in - * which case it is double-quoted with the YAML escapes; + * character outside YAML's printable set — in which case it is + * double-quoted with the YAML escapes; * - a multi-line string becomes a literal block scalar (`|`, `|-`, `|+` * according to its trailing newlines) unless its first line starts with * whitespace or it has no content, which fall back to a quoted scalar; @@ -202,7 +202,10 @@ public class YamlWriter( private fun canBlock(text: String): Boolean { val first = text[0] if (first == ' ' || first == '\t' || first == '\n') return false - for (c in text) if (c != '\n' && c != '\t' && c < ' ') return false + for (i in text.indices) { + val c = text[i] + if (c != '\n' && c != '\t' && text.isYamlUnprintableAt(i)) return false + } return text.trimEnd('\n').isNotEmpty() } @@ -241,16 +244,28 @@ public class YamlWriter( // double-quoted output (see YAML 1.2 §6.4 / §6.6). private const val YAML_INDICATORS = "-?:,[]{}#&*!|>'\"%@`" -// A character a plain scalar cannot carry: C0 / DEL / NEL and the Unicode -// line and paragraph separators (YAML 1.2 §5.1 printable characters). -private val Char.isYamlControl: Boolean - get() = this < ' ' || this == '\u007f' || this == '\u0085' || this == '\u2028' || this == '\u2029' +// Whether the character at [i] must be escaped: it is outside YAML's +// printable set (YAML 1.2 §5.1 — C0 controls, DEL, the C1 controls, +// U+FFFE / U+FFFF, a surrogate not part of a pair), which Psych and PyYAML +// refuse anywhere in a document, or a line break for a YAML 1.1 reader +// (NEL, the Unicode line and paragraph separators). A lone surrogate has no +// valid YAML representation at all; its `\u` escape is still the only way to +// keep it. +private fun String.isYamlUnprintableAt(i: Int): Boolean { + val c = this[i] + return when { + c.isHighSurrogate() -> i + 1 == length || !this[i + 1].isLowSurrogate() + c.isLowSurrogate() -> i == 0 || !this[i - 1].isHighSurrogate() + else -> c < ' ' || c in '\u007f'..'\u009f' || + c == '\u2028' || c == '\u2029' || c == '\ufffe' || c == '\uffff' + } +} // A plain scalar the parser would not read back as the same string: one it // would type (`yamlScalarType`) or a YAML 1.1 reader would (`isYaml11Typed`), // or whose shape is an indicator, a comment // (`#` after a space), a mapping colon (`:` before a space or at the end), -// surrounding whitespace or a control character. A `:` or `#` anywhere else +// surrounding whitespace or an unprintable character. A `:` or `#` anywhere else // is plain content (`https://x/y`, `C#`), as is an inner `"` or `\`. private fun needsQuoting(s: String): Boolean { if (s.isEmpty()) return true @@ -258,7 +273,7 @@ private fun needsQuoting(s: String): Boolean { if (s[0] in YAML_INDICATORS) return true for (i in s.indices) { val c = s[i] - if (c.isYamlControl) return true + if (s.isYamlUnprintableAt(i)) return true if (c == ':' && (i + 1 == s.length || s[i + 1] == ' ')) return true if (c == '#' && i > 0 && s[i - 1] == ' ') return true } @@ -267,14 +282,13 @@ private fun needsQuoting(s: String): Boolean { private fun quoted(s: String): String = buildString { +'"' - for (c in s) when { - c == '\\' -> +"\\\\" - c == '"' -> +"\\\"" - c == '\n' -> +"\\n" - c == '\r' -> +"\\r" - c == '\t' -> +"\\t" - c.isYamlControl -> +("\\u" + c.code.toString(16).padStart(4, '0')) - else -> +c + for (i in s.indices) when (val c = s[i]) { + '\\' -> +"\\\\" + '"' -> +"\\\"" + '\n' -> +"\\n" + '\r' -> +"\\r" + '\t' -> +"\\t" + else -> if (s.isYamlUnprintableAt(i)) +("\\u" + c.code.toString(16).padStart(4, '0')) else +c } +'"' } diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 59c1e58..9648b96 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -86,6 +86,29 @@ class YamlRoundTripTest { } } + @Test + fun `should write a colon or hash that is not an indicator plain in a sequence item`() = runTest { + // given — an item's content is first checked for a compact mapping + // (`- key: value`), so a `:` or `#` there takes a different path + // through the parser than an entry value + val source = "k:\n - https://xemantic.com/contact\n - a:b\n - C#\n - a#b\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs source + reparsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "k") { + "item" { +"https://xemantic.com/contact" } + "item" { +"a:b" } + "item" { +"C#" } + "item" { +"a#b" } + } + } + } + @Test fun `should keep quoting a colon or hash that is an indicator and YAML 1_1 sexagesimals`() = runTest { // given — a mapping colon, a comment, a leading indicator, and base-60 @@ -114,8 +137,8 @@ class YamlRoundTripTest { // or PyYAML type as a number, time, boolean or null, or refuse val values = listOf( "2024-05-01T10:00:00+0100", "2024-05-01 10:00:00 +0100", "2024-5-1", - "1_000", "1,000", "0b101", "+0x1F", "0755", "1_000.5", "1.0e+5", ".e+4", - "yEs", "oFF", "nULL", "=", "<<", + "1_000", "1,000", "0b101", "+0x1F", "1_000.5", "1,000.5", "1.5_0", ".e+4", + "yEs", "oFF", "nULL", "y", "Y", "n", "N", ".Nan", ".iNf", "+.InF", "-.INf", "=", "<<", ) for (value in values) { val events = semanticEvents { @@ -134,16 +157,51 @@ class YamlRoundTripTest { @Test fun `should quote a key a YAML 1_1 reader would type`() = runTest { - // given — Psych reads booleans in any letter case + // given — Psych reads booleans in any letter case, go-yaml v2 reads + // `y` / `n` as booleans val events = semanticEvents { "entry"("key" to "yEs") { +"v" } + "entry"("key" to "n") { +"v" } } // when val rendered = events.renderYaml() // then - rendered sameAs "\"yEs\": v\n" + rendered sameAs "\"yEs\": v\n\"n\": v\n" + } + + @Test + fun `should escape every character YAML does not allow in a document`() = runTest { + // given — C1 controls (mojibake like a Windows-1252 apostrophe read + // as Latin-1), the U+FFFE / U+FFFF non-characters and lone surrogates + // are outside YAML's printable set (YAML 1.2 §5.1): Psych and PyYAML + // refuse the whole document when they appear raw, even in a block + // scalar + val values = listOf( + "It\u0092s" to "\"It\\u0092s\"", + "a\u0080b" to "\"a\\u0080b\"", + "\u009f" to "\"\\u009f\"", + "x\uFFFEy" to "\"x\\ufffey\"", + "x\uFFFF" to "\"x\\uffff\"", + "lone \uD800 high" to "\"lone \\ud800 high\"", + "lone \uDC00 low" to "\"lone \\udc00 low\"", + "first\u0092\nsecond" to "\"first\\u0092\\nsecond\"", + "smile 😀" to "smile 😀", + ) + for ((value, expected) in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: $expected\n" + reparsed.mergeAdjacentText() sameAs events + } } @Test diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt index 982d32c..0e482d4 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlWriterTest.kt @@ -317,7 +317,7 @@ class YamlWriterTest { // after its dash; the line must still be terminated val flow = semanticEvents { "item" { "x" { } } - "item" { +"y" } + "item" { +"next" } } // when @@ -326,7 +326,7 @@ class YamlWriterTest { // then yaml sameAs """ - - - y + - next """.trimIndent() + "\n" } From b0bb7c84ce2a7cb397a68aa623df3544b08c326e Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 16:17:35 +0200 Subject: [PATCH 04/10] Type the plain scalars a YAML 1.1 reader types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser now types the shapes Psych (Jekyll), PyYAML and go-yaml v2 type even though YAML 1.2 reads them as strings: one-letter and mixed-case booleans / nulls, `_` / `,` separated numbers, binary, signed hex, sexagesimals and looser timestamps — including Jekyll's documented `2016-01-01 12:00:00 -0500`, which the writer used to quote and so rewrote on round-trip. Quoting only in the writer is not enough; the typing rules now live in one place, `YamlScalarType.kt`, shared by both sides through `isTypedPlainScalar`. The writer double-quotes a value holding a line break, carriage return or tab, and escapes a byte order mark (YAML 1.2 §5.2). Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 3 +- .../src/commonMain/kotlin/YamlParser.kt | 83 ---------------- .../src/commonMain/kotlin/YamlScalarType.kt | 95 +++++++++++++++++++ .../src/commonMain/kotlin/YamlWriter.kt | 58 ++++++----- .../src/commonTest/kotlin/YamlParserTest.kt | 64 +++++++++++++ .../commonTest/kotlin/YamlRoundTripTest.kt | 46 ++++++++- 6 files changed, 238 insertions(+), 111 deletions(-) create mode 100644 markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt diff --git a/CLAUDE.md b/CLAUDE.md index e7747f9..3b0db83 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -543,7 +543,8 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' The key is an attribute, not the mark name, on purpose: keys collide with tag names (`title` would hit `simplifyHtml`'s head-mode `match("title")`) and `og:title` would look like `` custom markup. `entry`/`item` are Markdown-native **only inside** a `frontmatter` (the Markdown renderer creates a `YamlWriter` on the mark, routes every following event into `YamlWriter.collect`, and treats the first `unmark` arriving at `depth == 0` as the wrapper's own close; outside one they fall through to the raw-tag path), so they are deliberately NOT in `MARKDOWN_NATIVE_MARK_NAMES`. Any other mark is **transparent** to the writer (children lay out as if it were absent), which is what lets `renderYaml()` accept a `frontmatter`-wrapped stream unchanged. - The writer's quoting rules reuse the parser's `yamlScalarType` (one copy of the reserved literals / int / float / timestamp rules) and are pinned against it by `YamlRoundTripTest` in `markanywhere-yaml`; `FrontMatterRoundTripTest` in `markanywhere-parse` (whose tests depend on render) covers only the fences and the wrapper — extend the yaml one when you touch either side of the codec. + The writer's quoting rules reuse the parser's `yamlScalarType` (`YamlScalarType.kt`, one copy of the reserved literals / int / float / timestamp rules) and are pinned against it by `YamlRoundTripTest` in `markanywhere-yaml`; `FrontMatterRoundTripTest` in `markanywhere-parse` (whose tests depend on render) covers only the fences and the wrapper — extend the yaml one when you touch either side of the codec. + A shape some front matter reader (Psych/Jekyll, PyYAML, go-yaml v2) types must be typed by the **parser**, not merely quoted by the writer — writer-only quoting rewrites the source on round-trip (Jekyll's `date: 2016-01-01 12:00:00 -0500` came back as a quoted string). `renderYaml()` **keeps the trailing newline** (unlike `renderMarkdown()` / `renderHtml()`, which suppress one): a `|+` block scalar at the end of the document needs it, and dropping it broke the round-trip for `z\n\n`. - Front matter **detection** requires line 2 to be a mapping key (identifier or quoted, then `:` + whitespace/EOL — `isYamlKeyLine`, a public function of `markanywhere-yaml`, NOT a private copy in `FrontMatterFilter`); a `# comment` cannot be line 2 (`---` + `# Heading` is a thematic break + heading), and neither can `http://…` or a `- item`. Consequence for the YAML writer: `YamlWriter.renderKey` quotes every key that is not identifier-shaped (`"a b"`, `"42"`, `"og:title"`), even where YAML would not require it, so whatever `simplifyHtml` emits first (a `` with a space, say) still re-detects. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index e168355..745dc0b 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -638,86 +638,3 @@ private fun StringBuilder.appendCodePointCompat(code: Int) { append(((v shr 10) + 0xD800).toChar()) append(((v and 0x3FF) + 0xDC00).toChar()) } - -private val YAML_NULLS = setOf("null", "Null", "NULL", "~") - -// YAML 1.2 core schema booleans plus the 1.1 forms still common in front -// matter (Jekyll's and Hugo's YAML readers accept them). -private val YAML_BOOLS = setOf( - "true", "True", "TRUE", "false", "False", "FALSE", - "yes", "Yes", "YES", "no", "No", "NO", - "on", "On", "ON", "off", "Off", "OFF" -) - -// Each pattern is explicitly anchored: Kotlin/JS resolves `matches` through -// the leftmost match, so an unanchored alternation lets the first branch win -// on a prefix (`0x1F` matched `[0-9]+` as `0`, a full timestamp matched its -// date-only branch) and the whole-input check then fails — JVM backtracks -// across the branches and never showed it. -private val YAML_INT = Regex("""^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$""") - -private val YAML_FLOAT = Regex( - """^(?:[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?|[-+]?\.(inf|Inf|INF)|\.(nan|NaN|NAN))$""" -) - -private val YAML_TIMESTAMP = Regex( - """^(?:[0-9]{4}-[0-9]{2}-[0-9]{2}""" + - """|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}([Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(\.[0-9]*)?([ \t]*(Z|[-+][0-9]{1,2}(:[0-9]{2})?))?)$""" -) - -// The `type` a plain scalar resolves to, or null for a string. -internal fun yamlScalarType(text: String): String? = when { - text in YAML_NULLS -> "null" - text in YAML_BOOLS -> "bool" - YAML_INT.matches(text) -> "int" - YAML_FLOAT.matches(text) -> "float" - YAML_TIMESTAMP.matches(text) -> "timestamp" - else -> null -} - -// YAML 1.1 plain-scalar shapes that YAML 1.2 — and so [yamlScalarType] — -// reads as a string, but that a YAML 1.1 reader still types: Psych (Jekyll) -// and PyYAML. The union of both is covered, erring on the side of too much: -// - booleans / nulls in any letter case (`yEs`, `nULL` — Psych ignores case), -// and the one-letter booleans `y` / `n` (the YAML 1.1 spec, go-yaml v2); -// - the special floats in any letter case (`.Nan`, `+.InF` — Psych again); -// - integers with `_` or `,` separators, binary, signed hex -// (`1_000`, `1,000`, `0b101`, `+0x1F`); -// - floats with separators or a digit-less mantissa (`1_000.5`, `1,000.5`, -// `.e+4`); -// - base-60 numbers (`12:30`, `+1:30`, `190:20:30.15`); -// - timestamps Psych accepts beyond the 1.2 shape: one-digit month / day -// (`2024-5-1`), an offset without a colon (`+0100`); -// - PyYAML's value / merge tags (`=`, `<<`), which its SafeLoader cannot -// construct. -// Anchored, see the note above. -private val YAML_1_1_WORDS = - (YAML_BOOLS + YAML_NULLS).mapTo(mutableSetOf()) { it.lowercase() } + - setOf("y", "n", ".inf", "+.inf", "-.inf", ".nan") - -private val YAML_1_1_WORD_MAX_LENGTH = YAML_1_1_WORDS.maxOf { it.length } - -private val YAML_1_1_NUMBER = Regex( - """^(?:[-+]?0b[01_,]+|[-+]?0x[0-9a-fA-F_,]+|[-+]?[0-9][0-9_,]*""" + - """|[-+]?(?:[0-9][0-9_,]*)?\.[0-9_]*(?:[eE][-+]?[0-9]+)?""" + - """|[-+]?[0-9][0-9_,]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?)$""" -) - -private val YAML_1_1_TIMESTAMP = Regex( - """^-?[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}""" + - """(?:(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9]{1,2}:?(?:[0-9]{2})?))?)?$""" -) - -// Whether a YAML 1.1 reader (Psych, PyYAML) would type this plain scalar, -// or refuse it, where [yamlScalarType] reads a string. The writer quotes -// such a value, in the same spirit as the 1.1 booleans in [YAML_BOOLS]. -// Every shape starts with a sign, a digit or a dot, or is one of a few -// words, so the regexes only run on a string that can match them. -internal fun isYaml11Typed(text: String): Boolean { - if (text.isEmpty()) return false - if (text == "=" || text == "<<") return true - if (text.length <= YAML_1_1_WORD_MAX_LENGTH && text.lowercase() in YAML_1_1_WORDS) return true - val first = text[0] - if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return false - return YAML_1_1_NUMBER.matches(text) || YAML_1_1_TIMESTAMP.matches(text) -} diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt new file mode 100644 index 0000000..701b084 --- /dev/null +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xemantic.markanywhere.yaml + +// The one copy of the plain-scalar typing rules: [YamlParser] resolves a +// plain scalar's `type` with them, [YamlWriter] quotes a string they would +// type. A plain scalar is typed when any reader front matter is written for +// would not read it as a string — the YAML 1.2 core schema, plus the YAML 1.1 +// shapes Psych (Jekyll), PyYAML and go-yaml v2 (Hugo) still type, erring on +// the side of too much: +// - booleans and nulls in any letter case (`yEs`, `nULL` — Psych ignores +// case), and the one-letter booleans `y` / `n` (the YAML 1.1 spec, go-yaml +// v2); +// - the special floats in any letter case (`.Nan`, `+.InF`); +// - integers with `_` or `,` separators, binary, signed hex +// (`1_000`, `1,000`, `0b101`, `+0x1F`); +// - floats with separators or a digit-less mantissa (`1_000.5`, `.e+4`) — +// but never a shape with no digit at all (`.`, `+.`), which every reader +// reads as a string; +// - base-60 numbers (`12:30` an int, `190:20:30.15` a float); +// - timestamps with a one-digit month / day (`2024-5-1`) or an offset +// without a colon (`2016-01-01 12:00:00 -0500`, Jekyll's documented +// format). +// Typing such a value keeps a source document as written: the writer writes +// a typed scalar bare, so only a *string* of one of these shapes is quoted. + +private val YAML_NULLS = setOf("null", "~") + +private val YAML_BOOLS = setOf("true", "false", "yes", "no", "on", "off", "y", "n") + +private val YAML_SPECIAL_FLOATS = setOf(".inf", "+.inf", "-.inf", ".nan") + +// the longest word above, so a long string is never lowercased +private const val YAML_WORD_MAX_LENGTH = 5 + +// Each pattern is explicitly anchored: Kotlin/JS resolves `matches` through +// the leftmost match, so an unanchored alternation lets the first branch win +// on a prefix (`0x1F` matched `[0-9]+` as `0`, a full timestamp matched its +// date-only branch) and the whole-input check then fails — JVM backtracks +// across the branches and never showed it. +private val YAML_INT = Regex( + """^[-+]?(?:[0-9][0-9_,]*|0b[01_,]+|0o[0-7]+|0x[0-9a-fA-F_,]+|[0-9][0-9_,]*(?::[0-5]?[0-9])+)$""" +) + +// the lookahead requires a digit somewhere, in the mantissa or the exponent +private val YAML_FLOAT = Regex( + """^(?=[^0-9]*[0-9])[-+]?(?:(?:[0-9][0-9_,]*)?\.[0-9_]*(?:[eE][-+]?[0-9]+)?""" + + """|[0-9]+[eE][-+]?[0-9]+|[0-9][0-9_,]*(?::[0-5]?[0-9])+\.[0-9_]*)$""" +) + +private val YAML_TIMESTAMP = Regex( + """^-?[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}""" + + """(?:(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9]{1,2}:?(?:[0-9]{2})?))?)?$""" +) + +// The `type` a plain scalar resolves to, or null for a string. +internal fun yamlScalarType(text: String): String? { + if (text.isEmpty()) return null + if (text.length <= YAML_WORD_MAX_LENGTH) { + val word = text.lowercase() + if (word in YAML_NULLS) return "null" + if (word in YAML_BOOLS) return "bool" + if (word in YAML_SPECIAL_FLOATS) return "float" + } + // every numeric shape starts with a sign, a digit or a dot, so the + // patterns only run on a string that can match them + val first = text[0] + if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return null + return when { + YAML_INT.matches(text) -> "int" + YAML_FLOAT.matches(text) -> "float" + YAML_TIMESTAMP.matches(text) -> "timestamp" + else -> null + } +} + +// Whether a plain scalar would not read back as this string: the parser +// types it, or it is one of PyYAML's value / merge tags (`=`, `<<`), which +// its SafeLoader refuses to construct. +internal fun isTypedPlainScalar(text: String): Boolean = + yamlScalarType(text) != null || text == "=" || text == "<<" diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index f371bc4..96bab8d 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -35,10 +35,10 @@ import com.xemantic.markanywhere.SemanticEvent * - a scalar with a `type` (`bool`, `int`, `float`, `null`, `timestamp`) is * written bare; a string is written plain unless a plain scalar would * re-parse as something else — reserved literals, numbers, timestamps, - * shapes a YAML 1.1 reader types (`12:30`, `1_000`, `yEs`), a leading - * indicator, `: ` or a trailing `:`, ` #`, surrounding whitespace, a - * character outside YAML's printable set — in which case it is - * double-quoted with the YAML escapes; + * including the YAML 1.1 shapes the parser types (`12:30`, `1_000`, + * `yEs`), a leading indicator, `: ` or a trailing `:`, ` #`, surrounding + * whitespace, a line break or tab, a character YAML requires escaped — + * in which case it is double-quoted with the YAML escapes; * - a multi-line string becomes a literal block scalar (`|`, `|-`, `|+` * according to its trailing newlines) unless its first line starts with * whitespace or it has no content, which fall back to a quoted scalar; @@ -46,9 +46,10 @@ import com.xemantic.markanywhere.SemanticEvent * text is a bare `key:`; `type=seq` / `type=map` with no children are * `[]` / `{}`; * - a key is written plain only when identifier-shaped (letters, digits, - * `_`, `-`, `.`, starting with a letter or `_`) and not a reserved - * literal — a YAML 1.1 reader takes a bare `yes:` as a boolean key — and - * double-quoted otherwise, even where YAML would not require it; + * `_`, `-`, `.`, starting with a letter or `_`) and not a shape a plain + * scalar would be typed as — a YAML 1.1 reader takes a bare `yes:`, `y:` + * or `nULL:` as a boolean / null key — and double-quoted otherwise, even + * where YAML would not require it; * - a `text` child of a container (the parser's verbatim fallback for a * line outside its YAML subset) is written as-is, on its own line(s). * @@ -204,7 +205,8 @@ public class YamlWriter( if (first == ' ' || first == '\t' || first == '\n') return false for (i in text.indices) { val c = text[i] - if (c != '\n' && c != '\t' && text.isYamlUnprintableAt(i)) return false + // a carriage return would be read as a line break + if (c == '\r' || text.isYamlUnprintableAt(i)) return false } return text.trimEnd('\n').isNotEmpty() } @@ -229,14 +231,14 @@ public class YamlWriter( return sb.toString() } - // A key is written plain only when it is identifier-shaped and not a - // reserved literal: the Markdown parser's front matter detection requires + // A key is written plain only when it is identifier-shaped and would not + // be typed (`yes`, `y`, `nULL`): the Markdown parser's front matter detection requires // the first line to pass `isYamlKeyLine` (such a key, or a quoted one), // so quoting everything else keeps whatever entry comes first // re-detectable. The character rules are shared with that check. private fun renderKey(key: String?): String { val k = key ?: "" - return if (isIdentifierKey(k) && yamlScalarType(k) == null && !isYaml11Typed(k)) k else quoted(k) + return if (isIdentifierKey(k) && !isTypedPlainScalar(k)) k else quoted(k) } } @@ -244,28 +246,32 @@ public class YamlWriter( // double-quoted output (see YAML 1.2 §6.4 / §6.6). private const val YAML_INDICATORS = "-?:,[]{}#&*!|>'\"%@`" -// Whether the character at [i] must be escaped: it is outside YAML's -// printable set (YAML 1.2 §5.1 — C0 controls, DEL, the C1 controls, -// U+FFFE / U+FFFF, a surrogate not part of a pair), which Psych and PyYAML -// refuse anywhere in a document, or a line break for a YAML 1.1 reader -// (NEL, the Unicode line and paragraph separators). A lone surrogate has no -// valid YAML representation at all; its `\u` escape is still the only way to -// keep it. +// Whether the character at [i] may only appear `\u`-escaped: it is outside +// YAML's printable set (YAML 1.2 §5.1 — C0 controls other than tab, line +// feed and carriage return, DEL, the C1 controls, U+FFFE / U+FFFF, a +// surrogate not part of a pair), which Psych and PyYAML refuse anywhere in a +// document; a byte order mark, which must not appear inside a document +// (§5.2); or a line break for a YAML 1.1 reader (NEL, the Unicode line and +// paragraph separators). A lone surrogate has no valid YAML representation +// at all; its `\u` escape is still the only way to keep it. Tab, line feed +// and carriage return are printable — each writer path decides where it can +// keep them. private fun String.isYamlUnprintableAt(i: Int): Boolean { val c = this[i] return when { c.isHighSurrogate() -> i + 1 == length || !this[i + 1].isLowSurrogate() c.isLowSurrogate() -> i == 0 || !this[i - 1].isHighSurrogate() - else -> c < ' ' || c in '\u007f'..'\u009f' || - c == '\u2028' || c == '\u2029' || c == '\ufffe' || c == '\uffff' + else -> (c < ' ' && c != '\t' && c != '\n' && c != '\r') || + c in '\u007f'..'\u009f' || c == '\u2028' || c == '\u2029' || + c == '\ufeff' || c == '\ufffe' || c == '\uffff' } } // A plain scalar the parser would not read back as the same string: one it -// would type (`yamlScalarType`) or a YAML 1.1 reader would (`isYaml11Typed`), -// or whose shape is an indicator, a comment -// (`#` after a space), a mapping colon (`:` before a space or at the end), -// surrounding whitespace or an unprintable character. A `:` or `#` anywhere else +// would type (`isTypedPlainScalar`), or whose shape is an indicator, a +// comment (`#` after a space), a mapping colon (`:` before a space or at the +// end), surrounding whitespace, a line break or tab, or an unprintable +// character. A `:` or `#` anywhere else // is plain content (`https://x/y`, `C#`), as is an inner `"` or `\`. private fun needsQuoting(s: String): Boolean { if (s.isEmpty()) return true @@ -273,11 +279,11 @@ private fun needsQuoting(s: String): Boolean { if (s[0] in YAML_INDICATORS) return true for (i in s.indices) { val c = s[i] - if (s.isYamlUnprintableAt(i)) return true + if (c == '\n' || c == '\r' || c == '\t' || s.isYamlUnprintableAt(i)) return true if (c == ':' && (i + 1 == s.length || s[i + 1] == ' ')) return true if (c == '#' && i > 0 && s[i - 1] == ' ') return true } - return yamlScalarType(s) != null || isYaml11Typed(s) + return isTypedPlainScalar(s) } private fun quoted(s: String): String = buildString { diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 33d16db..cd108ae 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -301,6 +301,70 @@ class YamlParserTest { } } + @Test + fun `should type the plain scalars a YAML 1_1 reader types`() = runTest { + // given — YAML 1.2 reads these as strings, but Psych (Jekyll), PyYAML + // or go-yaml v2 type them, and front matter is written for those + // readers: Jekyll's documented date format carries a colonless offset + val textFlow = """ + date: 2016-01-01 12:00:00 -0500 + short: 2024-5-1 + answer: y + shout: N + mixed: yEs + nil: nULL + big: 1_000 + grouped: 1,000 + bin: 0b101 + signed: +0x1F + money: 1_000.5 + nan: .NaN + time: 12:30 + angle: 190:20:30.15 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "date", "type" to "timestamp") { +"2016-01-01 12:00:00 -0500" } + "entry"("key" to "short", "type" to "timestamp") { +"2024-5-1" } + "entry"("key" to "answer", "type" to "bool") { +"y" } + "entry"("key" to "shout", "type" to "bool") { +"N" } + "entry"("key" to "mixed", "type" to "bool") { +"yEs" } + "entry"("key" to "nil", "type" to "null") { +"nULL" } + "entry"("key" to "big", "type" to "int") { +"1_000" } + "entry"("key" to "grouped", "type" to "int") { +"1,000" } + "entry"("key" to "bin", "type" to "int") { +"0b101" } + "entry"("key" to "signed", "type" to "int") { +"+0x1F" } + "entry"("key" to "money", "type" to "float") { +"1_000.5" } + "entry"("key" to "nan", "type" to "float") { +".NaN" } + "entry"("key" to "time", "type" to "int") { +"12:30" } + "entry"("key" to "angle", "type" to "float") { +"190:20:30.15" } + } + } + + @Test + fun `should not type a float shape without a digit`() = runTest { + // given — no reader types a lone dot or a sign and a dot + val textFlow = """ + dot: . + signed: +. + under: ._ + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "dot") { +"." } + "entry"("key" to "signed") { +"+." } + "entry"("key" to "under") { +"._" } + } + } + @Test fun `should keep a quoted scalar a string`() = runTest { // given — quoting suppresses the type resolution diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 9648b96..13b485c 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -166,9 +166,51 @@ class YamlRoundTripTest { // when val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() // then rendered sameAs "\"yEs\": v\n\"n\": v\n" + reparsed.mergeAdjacentText() sameAs events + } + + @Test + fun `should keep YAML 1_1 typed front matter values as written`() = runTest { + // given — Jekyll's documented date format, a short date, go-yaml v2 + // booleans and Psych numbers: the parser types them, so the writer + // writes them back bare instead of quoting them into strings + val source = """ + date: 2016-01-01 12:00:00 -0500 + short: 2024-5-1 + answer: y + mixed: yEs + big: 1_000 + time: 12:30 + """.trimIndent() + "\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + + // then + rendered sameAs source + } + + @Test + fun `should not quote a float shape without a digit`() = runTest { + // given + val values = listOf(".", "+.", "._") + for (value in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: $value\n" + reparsed.mergeAdjacentText() sameAs events + } } @Test @@ -177,7 +219,7 @@ class YamlRoundTripTest { // as Latin-1), the U+FFFE / U+FFFF non-characters and lone surrogates // are outside YAML's printable set (YAML 1.2 §5.1): Psych and PyYAML // refuse the whole document when they appear raw, even in a block - // scalar + // scalar; a byte order mark must not appear inside a document (§5.2) val values = listOf( "It\u0092s" to "\"It\\u0092s\"", "a\u0080b" to "\"a\\u0080b\"", @@ -187,6 +229,8 @@ class YamlRoundTripTest { "lone \uD800 high" to "\"lone \\ud800 high\"", "lone \uDC00 low" to "\"lone \\udc00 low\"", "first\u0092\nsecond" to "\"first\\u0092\\nsecond\"", + "\uFEFFTitle" to "\"\\ufeffTitle\"", + "first\n\uFEFFsecond" to "\"first\\n\\ufeffsecond\"", "smile 😀" to "smile 😀", ) for ((value, expected) in values) { From 58b822fa864918a6545fa8dd178ea22267fb2890 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 20:54:10 +0200 Subject: [PATCH 05/10] Type a plain scalar exactly when a front matter reader does YamlScalarType now transcribes the number rules of PyYAML, Psych and go-yaml v2 separately instead of one merged pattern, which typed shapes none of them reads as a number (`1,`, `1_,2`). go-yaml v2 deletes every `_` first, so `1_e5` / `+_1` / `0_x1` are numbers, and it takes an upper-case base prefix (`0X1F`). A timestamp is typed only when it is in range: a date must be in the calendar, a date and time within what Psych's Time accepts. Shapes a reader refuses to load (`=`, `<<`, `0x_`, `.e+4`, `2024-13-45`) stay strings in the parser but are still quoted by the writer, pinned as DIVERGENCE. The colon / comment indicator checks are shared between parser and writer via isMappingColonAt / isCommentStartAt, so the writer now also treats a tab like a space there. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 4 + markanywhere-yaml/README.md | 8 +- .../src/commonMain/kotlin/YamlParser.kt | 17 +- .../src/commonMain/kotlin/YamlScalarType.kt | 156 ++++++++++++++---- .../src/commonMain/kotlin/YamlWriter.kt | 17 +- .../src/commonTest/kotlin/YamlParserTest.kt | 106 ++++++++++++ .../commonTest/kotlin/YamlRoundTripTest.kt | 67 ++++++++ 7 files changed, 324 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3b0db83..06414db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -545,6 +545,10 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Any other mark is **transparent** to the writer (children lay out as if it were absent), which is what lets `renderYaml()` accept a `frontmatter`-wrapped stream unchanged. The writer's quoting rules reuse the parser's `yamlScalarType` (`YamlScalarType.kt`, one copy of the reserved literals / int / float / timestamp rules) and are pinned against it by `YamlRoundTripTest` in `markanywhere-yaml`; `FrontMatterRoundTripTest` in `markanywhere-parse` (whose tests depend on render) covers only the fences and the wrapper — extend the yaml one when you touch either side of the codec. A shape some front matter reader (Psych/Jekyll, PyYAML, go-yaml v2) types must be typed by the **parser**, not merely quoted by the writer — writer-only quoting rewrites the source on round-trip (Jekyll's `date: 2016-01-01 12:00:00 -0500` came back as a quoted string). + The converse holds too: a shape **no** reader types must not be typed, or `type=int` stops meaning a number (`1,` once came back `type=int`). + So `YamlScalarType.kt` transcribes each reader's number rules separately instead of merging them into one pattern — the separator rules differ per reader (go-yaml v2 deletes every `_` before parsing, so `1_e5` / `+_1` are numbers to Hugo), and a merged pattern types shapes none of them does. + Check a change against the real readers (PyYAML `safe_load`, Ruby `Psych.unsafe_load`, `gopkg.in/yaml.v2` into `map[string]interface{}`) over generated strings, not by reading their docs — go-yaml v2 decodes a timestamp into a *string*, Psych normalises an out-of-range date *and time* but rejects an out-of-range date alone. + The one sanctioned writer-only quoting is a shape a reader **refuses to load** (`=`, `<<`, `0x_`, `.e+4`, `2024-13-45`): there is no type to report, so the parser keeps it a string and the rewrite is pinned as `DIVERGENCE` in `YamlRoundTripTest`. `renderYaml()` **keeps the trailing newline** (unlike `renderMarkdown()` / `renderHtml()`, which suppress one): a `|+` block scalar at the end of the document needs it, and dropping it broke the round-trip for `z\n\n`. - Front matter **detection** requires line 2 to be a mapping key (identifier or quoted, then `:` + whitespace/EOL — `isYamlKeyLine`, a public function of `markanywhere-yaml`, NOT a private copy in `FrontMatterFilter`); a `# comment` cannot be line 2 (`---` + `# Heading` is a thematic break + heading), and neither can `http://…` or a `- item`. Consequence for the YAML writer: `YamlWriter.renderKey` quotes every key that is not identifier-shaped (`"a b"`, `"42"`, `"og:title"`), even where YAML would not require it, so whatever `simplifyHtml` emits first (a `` with a space, say) still re-detects. diff --git a/markanywhere-yaml/README.md b/markanywhere-yaml/README.md index 18f70ab..21b6e63 100644 --- a/markanywhere-yaml/README.md +++ b/markanywhere-yaml/README.md @@ -39,7 +39,10 @@ The vocabulary is deliberately small and the key lives in an attribute, so a key - `entry` (attribute `key`) is a mapping entry; `item` is a sequence element. A root mapping is a run of top-level `entry` marks, a root sequence a run of top-level `item` marks — there is no wrapper mark. - The kind of value follows from the children: text only is a scalar, nested `entry` marks are a mapping, nested `item` marks are a sequence. -- `type` is present only on a non-string scalar: `bool`, `int`, `float`, `null`, `timestamp` (YAML 1.2 core schema, plus the 1.1 `yes`/`no`/`on`/`off` booleans so a Jekyll / Hugo document round-trips as written). +- `type` is present only on a non-string scalar: `bool`, `int`, `float`, `null`, `timestamp`. + A plain scalar is typed exactly when some reader front matter is written for reads it as something other than a string — the YAML 1.2 core schema, plus the YAML 1.1 shapes Psych (Jekyll), PyYAML and go-yaml v2 (Hugo) still type — so a Jekyll / Hugo document round-trips as written. + Beyond the core schema that means booleans and nulls in any letter case and `y` / `n`, numbers with `_` / `,` separators or a binary / octal / upper-case base prefix (`1_000`, `1,000`, `0b101`, `0X1F`), base-60 numbers (`12:30` is an `int`), and timestamps with a one-digit month or day or a colonless offset (`2016-01-01 12:00:00 -0500`); a date only when it is in the calendar. + So `type="int"` / `type="float"` says a reader reads a number, not that the text is a Kotlin literal: reading the value yourself takes that reader's rules (drop the separators, resolve the prefix or the base 60). No text and no type is an empty string; a bare `key:` is `type="null"`; an empty `[]` / `{}` is `type="seq"` / `type="map"` with no children. - All marks are untagged. @@ -56,7 +59,8 @@ Anchors, aliases and tags are not resolved: a plain scalar starting with `&`, `* ## Writing `renderYaml()` emits block style, one line per scalar, two spaces per nesting level. -A string that would re-parse as another type or shape (`"true"`, `"42"`, `"- dash"`, `"a: b"`, surrounding whitespace, control characters) is double-quoted; a multi-line string becomes a literal block scalar with the chomping indicator its trailing newlines call for; a key is written plain only when identifier-shaped and not a reserved literal. +A string that would re-parse as another type or shape (`"true"`, `"y"`, `"42"`, `"12:30"`, `"- dash"`, `"a: b"`, surrounding whitespace) is double-quoted, with every character YAML does not allow in a document `\u`-escaped; a multi-line string becomes a literal block scalar with the chomping indicator its trailing newlines call for; a key is written plain only when identifier-shaped and not a typed word (`"yes"`, `"y"`, `"null"`). +DIVERGENCE: a string some reader refuses to load when plain — `=`, `<<`, a base prefix with no digit (`0x_`), a date not in the calendar (`2024-13-45`) — is quoted too, although the parser reads it back as a plain string, so the first render quotes such a value in the source. Any mark other than `entry` / `item` is transparent, so a `frontmatter` wrapper renders the same with or without it. Every line is terminated — a non-empty document ends with `\n`, which a `|+` block scalar at the end needs. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 745dc0b..7b4ead7 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -35,8 +35,12 @@ import kotlinx.coroutines.flow.FlowCollector * sequence is nested `item` marks. The root is a mapping (top-level * `entry` marks) or a sequence (top-level `item` marks) — no wrapper mark. * - A non-string scalar carries `type` = `bool` / `int` / `float` / `null` / - * `timestamp` (YAML 1.2 core schema, plus the 1.1 `yes`/`no`/`on`/`off` - * booleans so a Jekyll / Hugo document round-trips as written). An empty + * `timestamp`: typed exactly when a front matter reader — the YAML 1.2 + * core schema, or the YAML 1.1 shapes of Psych (Jekyll), PyYAML and + * go-yaml v2 (Hugo) — reads it as other than a string (`y`, `yEs`, + * `1_000`, `0X1F`, `12:30`, `2024-5-1`), so a Jekyll / Hugo document + * round-trips as written. A number keeps that reader's syntax (separators, + * base prefixes, base 60) in its text. An empty * flow collection carries `type` = `seq` / `map`. An empty string is an * entry with no text and no type; a missing value (`key:`) is `type=null`. * @@ -350,9 +354,8 @@ public class YamlParser( if (content == "?" || content.startsWith("? ")) return null var i = 0 while (i < content.length) { - val c = content[i] - if (c == ':' && (i + 1 == content.length || content[i + 1] == ' ' || content[i + 1] == '\t')) break - if (c == '#' && i > 0 && content[i - 1] == ' ') return null + if (content.isMappingColonAt(i)) break + if (content.isCommentStartAt(i)) return null i++ } if (i == content.length) return null @@ -553,9 +556,7 @@ private fun isBlankOrComment(s: String, from: Int): Boolean { // A ` #` (hash preceded by whitespace) starts a trailing comment. private fun stripTrailingComment(value: String): String { for (i in 1 until value.length) { - if (value[i] == '#' && (value[i - 1] == ' ' || value[i - 1] == '\t')) { - return value.substring(0, i) - } + if (value.isCommentStartAt(i)) return value.substring(0, i) } return value } diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index 701b084..64ec9e1 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -18,23 +18,31 @@ package com.xemantic.markanywhere.yaml // The one copy of the plain-scalar typing rules: [YamlParser] resolves a // plain scalar's `type` with them, [YamlWriter] quotes a string they would -// type. A plain scalar is typed when any reader front matter is written for -// would not read it as a string — the YAML 1.2 core schema, plus the YAML 1.1 -// shapes Psych (Jekyll), PyYAML and go-yaml v2 (Hugo) still type, erring on -// the side of too much: +// type. A plain scalar is typed exactly when a reader front matter is +// written for reads it as something other than a string — the YAML 1.2 +// core schema, and the YAML 1.1 shapes of Psych (Jekyll), PyYAML and go-yaml +// v2 (Hugo). The number rules are therefore transcribed per reader, each +// less the shapes that reader then refuses to construct, rather than merged +// into one pattern: every reader has its own separator rules (Psych takes +// `,` only before a digit, PyYAML takes `_` anywhere after the first digit, +// go-yaml v2 deletes every `_` first), and a merged pattern would type +// shapes none of them does (`1,`, `1_,2`). So `type=int` / `type=float` means +// some reader reads a number — reading it yourself needs that reader's +// rules (drop `_` and `,`; base 60, binary and octal prefixes). +// Covered beyond the core schema: // - booleans and nulls in any letter case (`yEs`, `nULL` — Psych ignores -// case), and the one-letter booleans `y` / `n` (the YAML 1.1 spec, go-yaml -// v2); +// case), and the one-letter booleans `y` / `n` (go-yaml v2); // - the special floats in any letter case (`.Nan`, `+.InF`); -// - integers with `_` or `,` separators, binary, signed hex -// (`1_000`, `1,000`, `0b101`, `+0x1F`); -// - floats with separators or a digit-less mantissa (`1_000.5`, `.e+4`) — -// but never a shape with no digit at all (`.`, `+.`), which every reader -// reads as a string; +// - integers with separators, binary, upper-case and signed prefixes +// (`1_000`, `1,000`, `0b101`, `0X1F`, `+0x1F`, `+_1`); +// - floats with separators (`1_000.5`, `1_e5`); // - base-60 numbers (`12:30` an int, `190:20:30.15` a float); // - timestamps with a one-digit month / day (`2024-5-1`) or an offset // without a colon (`2016-01-01 12:00:00 -0500`, Jekyll's documented -// format). +// format) — a date only when it is in the calendar, a date and time only +// within the ranges Psych's `Time` accepts (it normalises `2023-02-31`). +// Not modelled: a float that overflows (`7e700`), which go-yaml v2 alone +// reads as a string. // Typing such a value keeps a source document as written: the writer writes // a typed scalar bare, so only a *string* of one of these shapes is quoted. @@ -45,26 +53,60 @@ private val YAML_BOOLS = setOf("true", "false", "yes", "no", "on", "off", "y", " private val YAML_SPECIAL_FLOATS = setOf(".inf", "+.inf", "-.inf", ".nan") // the longest word above, so a long string is never lowercased -private const val YAML_WORD_MAX_LENGTH = 5 +private val YAML_WORD_MAX_LENGTH = + (YAML_NULLS + YAML_BOOLS + YAML_SPECIAL_FLOATS).maxOf { it.length } // Each pattern is explicitly anchored: Kotlin/JS resolves `matches` through // the leftmost match, so an unanchored alternation lets the first branch win // on a prefix (`0x1F` matched `[0-9]+` as `0`, a full timestamp matched its // date-only branch) and the whole-input check then fails — JVM backtracks // across the branches and never showed it. -private val YAML_INT = Regex( - """^[-+]?(?:[0-9][0-9_,]*|0b[01_,]+|0o[0-7]+|0x[0-9a-fA-F_,]+|[0-9][0-9_,]*(?::[0-5]?[0-9])+)$""" +private fun anchored(pattern: String) = Regex("^(?:$pattern)$") + +// PyYAML's resolver, less a base prefix with no digit (`0x_`), which it +// resolves and then fails to construct +private val PYYAML_INT = anchored( + "[-+]?0b[01_]*[01][01_]*|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)" + + "|[-+]?0x[0-9a-fA-F_]*[0-9a-fA-F][0-9a-fA-F_]*|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+" +) + +private val PYYAML_FLOAT = anchored( + """[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?""" + + """|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*""" +) + +// Psych's scalar scanner, with its default (legacy) integers that allow `,` +private val PSYCH_INT = anchored( + "[-+]?0b[01_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?(?:0|[1-9](?:[0-9]|[,_][0-9])*)" + + "|[-+]?0x[0-9a-fA-F_,]*[0-9a-fA-F][0-9a-fA-F_,]*|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}" ) -// the lookahead requires a digit somewhere, in the mantissa or the exponent -private val YAML_FLOAT = Regex( - """^(?=[^0-9]*[0-9])[-+]?(?:(?:[0-9][0-9_,]*)?\.[0-9_]*(?:[eE][-+]?[0-9]+)?""" + - """|[0-9]+[eE][-+]?[0-9]+|[0-9][0-9_,]*(?::[0-5]?[0-9])+\.[0-9_]*)$""" +private val PSYCH_FLOAT = anchored( + """[-+]?(?:[0-9][0-9_,]*\.[0-9]*|\.[0-9]+)(?:[eE][-+][0-9]+)?""" + + """|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}\.[0-9_]*""" ) -private val YAML_TIMESTAMP = Regex( - """^-?[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}""" + - """(?:(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9]{1,2}:?(?:[0-9]{2})?))?)?$""" +// go-yaml v2 deletes every `_` from a text starting with a sign or a digit, +// then tries Go's `ParseInt` with base 0 (which reads a `0x` / `0o` / `0b` +// prefix in either case), a `0b` prefix of its own (`-0b` included, which +// lets a sign follow the prefix) and a YAML 1.2 float +private val GO_YAML_INT = anchored( + "[-+]?(?:[0-9]+|0[xX][0-9a-fA-F]+|0[oO][0-7]+|0[bB][01]+)|-?0b[-+]?[01]+" +) + +private val GO_YAML_FLOAT = anchored("""[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?""") + +// … but hands a text starting with `.` to Go's `ParseFloat` as is, which +// takes an `_` only between two digits +private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9](?:_?[0-9])*(?:[eE][-+]?[0-9](?:_?[0-9])*)?""") + +// The union of the Psych and PyYAML timestamp shapes (go-yaml v2 decodes a +// timestamp into a string); the groups are the date, the time and the +// offset hours, so [isTimestampInRange] can check their values. +private val YAML_TIMESTAMP = anchored( + """(-?[0-9]{4})-([0-9]{1,2})-([0-9]{1,2})""" + + """(?:(?:[Tt]|[ \t]+)([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]*)?""" + + """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}):?(?:[0-9]{2})?))?)?""" ) // The `type` a plain scalar resolves to, or null for a string. @@ -80,16 +122,66 @@ internal fun yamlScalarType(text: String): String? { // patterns only run on a string that can match them val first = text[0] if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return null - return when { - YAML_INT.matches(text) -> "int" - YAML_FLOAT.matches(text) -> "float" - YAML_TIMESTAMP.matches(text) -> "timestamp" - else -> null + if (PYYAML_INT.matches(text) || PSYCH_INT.matches(text)) return "int" + if (PYYAML_FLOAT.matches(text) || PSYCH_FLOAT.matches(text)) return "float" + if (first == '.') { + if (GO_YAML_DOT_FLOAT.matches(text)) return "float" + } else { + val plain = text.replace("_", "") + if (GO_YAML_INT.matches(plain)) return "int" + if (GO_YAML_FLOAT.matches(plain)) return "float" } + val timestamp = YAML_TIMESTAMP.matchEntire(text) + if (timestamp != null && isTimestampInRange(timestamp.groupValues)) return "timestamp" + return null } -// Whether a plain scalar would not read back as this string: the parser -// types it, or it is one of PyYAML's value / merge tags (`=`, `<<`), which -// its SafeLoader refuses to construct. -internal fun isTypedPlainScalar(text: String): Boolean = - yamlScalarType(text) != null || text == "=" || text == "<<" +// Psych checks a date against the calendar, but normalises a date and time +// (`2023-02-31T10:00:00` is March 3rd), refusing only the values its `Time` +// cannot take; a date-only shape with a sign is not a timestamp at all. +private fun isTimestampInRange(groups: List): Boolean { + val year = groups[1].toInt() + val month = groups[2].toInt() + val day = groups[3].toInt() + if (month !in 1..12) return false + if (groups[4].isNotEmpty()) { + val offsetHours = groups[7] + return day in 1..31 && groups[4].toInt() <= 24 && groups[5].toInt() <= 59 && + groups[6].toInt() <= 60 && (offsetHours.isEmpty() || offsetHours.toInt() <= 23) + } + if (year < 0) return false + return day in 1..daysInMonth(year, month) +} + +private fun daysInMonth(year: Int, month: Int): Int = when (month) { + 2 -> if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) 29 else 28 + 4, 6, 9, 11 -> 30 + else -> 31 +} + +// Shapes some reader resolves and then refuses to load, so the whole +// document fails: PyYAML's value / merge tags (`=`, `<<`), which its +// SafeLoader cannot construct; a base prefix with no digit (`0x_`, PyYAML +// and Psych); an exponent with no mantissa digit (`.e+4`, Psych); a +// timestamp shape [isTimestampInRange] refuses (PyYAML, for a date not in +// the calendar). There is no type to report for them, so the parser keeps +// them strings — DIVERGENCE: the writer still quotes them, which rewrites +// such a plain source value on its first render. +private val YAML_REJECTED = anchored("""=|<<|[-+]?0[bx][_,]+|[-+]?\.[eE][-+][0-9]+""") + +// Whether a plain scalar would not read back as this string in every +// reader: the parser types it, or some reader refuses it. +internal fun isUnsafePlainScalar(text: String): Boolean = + yamlScalarType(text) != null || YAML_REJECTED.matches(text) || YAML_TIMESTAMP.matches(text) + +// The two indicators that can end a plain scalar in block context, shared by +// [YamlParser] (where it splits a key and strips a comment) and [YamlWriter] +// (which quotes a string holding either), so the two cannot drift apart: +// a `:` followed by whitespace or the end is a mapping colon, a `#` after +// whitespace starts a comment. Anywhere else both are plain content. + +internal fun String.isMappingColonAt(i: Int): Boolean = + this[i] == ':' && (i + 1 == length || this[i + 1] == ' ' || this[i + 1] == '\t') + +internal fun String.isCommentStartAt(i: Int): Boolean = + this[i] == '#' && i > 0 && (this[i - 1] == ' ' || this[i - 1] == '\t') diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 96bab8d..52c2e1e 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -238,7 +238,7 @@ public class YamlWriter( // re-detectable. The character rules are shared with that check. private fun renderKey(key: String?): String { val k = key ?: "" - return if (isIdentifierKey(k) && !isTypedPlainScalar(k)) k else quoted(k) + return if (isIdentifierKey(k) && !isUnsafePlainScalar(k)) k else quoted(k) } } @@ -267,11 +267,11 @@ private fun String.isYamlUnprintableAt(i: Int): Boolean { } } -// A plain scalar the parser would not read back as the same string: one it -// would type (`isTypedPlainScalar`), or whose shape is an indicator, a -// comment (`#` after a space), a mapping colon (`:` before a space or at the -// end), surrounding whitespace, a line break or tab, or an unprintable -// character. A `:` or `#` anywhere else +// A plain scalar a reader would not read back as the same string: one the +// parser would type or a reader refuses (`isUnsafePlainScalar`), or whose +// shape is an indicator, a comment (`#` after whitespace), a mapping colon +// (`:` before whitespace or at the end), surrounding whitespace, a line +// break or tab, or an unprintable character. A `:` or `#` anywhere else // is plain content (`https://x/y`, `C#`), as is an inner `"` or `\`. private fun needsQuoting(s: String): Boolean { if (s.isEmpty()) return true @@ -280,10 +280,9 @@ private fun needsQuoting(s: String): Boolean { for (i in s.indices) { val c = s[i] if (c == '\n' || c == '\r' || c == '\t' || s.isYamlUnprintableAt(i)) return true - if (c == ':' && (i + 1 == s.length || s[i + 1] == ' ')) return true - if (c == '#' && i > 0 && s[i - 1] == ' ') return true + if (s.isMappingColonAt(i) || s.isCommentStartAt(i)) return true } - return isTypedPlainScalar(s) + return isUnsafePlainScalar(s) } private fun quoted(s: String): String = buildString { diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index cd108ae..5573c38 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -320,6 +320,7 @@ class YamlParserTest { money: 1_000.5 nan: .NaN time: 12:30 + clock: 12:30:45:10 angle: 190:20:30.15 """.trimIndent().chunkedRandomly().asFlow() @@ -341,10 +342,99 @@ class YamlParserTest { "entry"("key" to "money", "type" to "float") { +"1_000.5" } "entry"("key" to "nan", "type" to "float") { +".NaN" } "entry"("key" to "time", "type" to "int") { +"12:30" } + "entry"("key" to "clock", "type" to "int") { +"12:30:45:10" } "entry"("key" to "angle", "type" to "float") { +"190:20:30.15" } } } + @Test + fun `should type the number shapes go-yaml v2 reads once underscores are removed`() = runTest { + // given — go-yaml v2 (Hugo) deletes every `_` before it parses a + // number, so an underscore next to a sign, an exponent or a base + // prefix still makes one; it also takes an upper-case base prefix + val textFlow = """ + a: 1_e5 + b: 1e5_ + c: 1_E5 + d: +_1 + e: +_1. + f: 0_x1 + g: 0X1F + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a", "type" to "float") { +"1_e5" } + "entry"("key" to "b", "type" to "float") { +"1e5_" } + "entry"("key" to "c", "type" to "float") { +"1_E5" } + "entry"("key" to "d", "type" to "int") { +"+_1" } + "entry"("key" to "e", "type" to "float") { +"+_1." } + "entry"("key" to "f", "type" to "int") { +"0_x1" } + "entry"("key" to "g", "type" to "int") { +"0X1F" } + } + } + + @Test + fun `should not type a shape no reader types`() = runTest { + // given — a trailing or doubled separator, a dot before an + // underscore, an exponent with no mantissa digit, a date that is not + // in the calendar, a date-only shape with a sign and a minute Psych's + // Time refuses: PyYAML, Psych and go-yaml v2 all read strings + val textFlow = """ + a: 1, + b: 1,,2 + c: ._0 + d: .e5 + e: 2024-2-30 + f: -2024-01-01 + g: 2024-01-01T12:60:00 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a") { +"1," } + "entry"("key" to "b") { +"1,,2" } + "entry"("key" to "c") { +"._0" } + "entry"("key" to "d") { +".e5" } + "entry"("key" to "e") { +"2024-2-30" } + "entry"("key" to "f") { +"-2024-01-01" } + "entry"("key" to "g") { +"2024-01-01T12:60:00" } + } + } + + @Test + fun `should DIVERGENCE not type a plain scalar a reader refuses to load`() = runTest { + // given — PyYAML resolves `=` and `<<` to its value / merge tags and + // then refuses to construct them; it refuses a date not in the + // calendar, and PyYAML / Psych refuse a base prefix with no digit or + // an exponent with no mantissa: none of them has a type to report + val textFlow = """ + a: = + b: << + c: 2024-13-45 + d: 0x_ + e: .e+4 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a") { +"=" } + "entry"("key" to "b") { +"<<" } + "entry"("key" to "c") { +"2024-13-45" } + "entry"("key" to "d") { +"0x_" } + "entry"("key" to "e") { +".e+4" } + } + } + @Test fun `should not type a float shape without a digit`() = runTest { // given — no reader types a lone dot or a sign and a dot @@ -702,6 +792,22 @@ class YamlParserTest { // --- verbatim fallback (DIVERGENCE) ----------------------------------- + @Test + fun `should read a hash after a tab as a comment in a key line`() = runTest { + // given — `#` after any whitespace starts a comment, so the colon + // behind it is not a mapping indicator and the line is not an entry + val textFlow = "title: x\na\t#b: c\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "title") { +"x" } + +"a\t#b: c\n" + } + } + @Test fun `should DIVERGENCE keep an unrecognised line verbatim`() = runTest { // given — a complex key, a multi-line flow sequence, an unterminated diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 13b485c..347f19b 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -194,6 +194,73 @@ class YamlRoundTripTest { rendered sameAs source } + @Test + fun `should quote a string go-yaml v2 reads as a number once underscores are removed`() = runTest { + // given + val values = listOf("1_e5", "1e5_", "1_E5", "+_1", "+_1.", "0_x1", "0X1F") + for (value in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: \"$value\"\n" + reparsed.mergeAdjacentText() sameAs events + } + } + + @Test + fun `should not quote a shape no reader types`() = runTest { + // given + val values = listOf("1,", "1,,2", "._0", ".e5") + for (value in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: $value\n" + reparsed.mergeAdjacentText() sameAs events + } + } + + @Test + fun `should DIVERGENCE quote a plain scalar a reader refuses to load`() = runTest { + // given — the parser leaves these strings (no reader has a type for + // them), but PyYAML or Psych refuse the whole document when they are + // plain, so the writer quotes them and the first render rewrites the + // source; a date-shaped string outside the calendar is quoted too + val source = """ + a: = + b: << + c: 2024-13-45 + d: 2024-2-30 + e: 0x_ + f: .e+4 + """.trimIndent() + "\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + + // then + rendered sameAs """ + a: "=" + b: "<<" + c: "2024-13-45" + d: "2024-2-30" + e: "0x_" + f: ".e+4" + """.trimIndent() + "\n" + } + @Test fun `should not quote a float shape without a digit`() = runTest { // given From 060db830ff3cb725d5af3d1f2f9bd3c197374a8a Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Thu, 24 Sep 2026 21:26:48 +0200 Subject: [PATCH 06/10] Type and quote YAML scalars and keys exactly as the readers do Checked against PyYAML, Psych and go-yaml v2: - quote a date-shaped string only when PyYAML refuses it (2024-2-30 stays plain) - quote only the keys Psych types (yEs, nULL), not y / n, which go-yaml v2 decodes into a string key - stop typing -0b-1 / -0b+1 as int, type 08 / 019 / 0_9 as float - refuse hour 24 beyond 24:00:00 and offset minutes over 59 - share the mapping-colon and comment rules with flow collections - gate the refused-shape patterns on the first character Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 1 + .../src/commonMain/kotlin/YamlParser.kt | 5 +- .../src/commonMain/kotlin/YamlScalarType.kt | 73 +++++++++++++----- .../src/commonMain/kotlin/YamlWriter.kt | 11 ++- .../src/commonTest/kotlin/YamlParserTest.kt | 74 ++++++++++++++++++- .../commonTest/kotlin/YamlRoundTripTest.kt | 31 ++++++-- 6 files changed, 158 insertions(+), 37 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 06414db..dee3e38 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -553,6 +553,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - Front matter **detection** requires line 2 to be a mapping key (identifier or quoted, then `:` + whitespace/EOL — `isYamlKeyLine`, a public function of `markanywhere-yaml`, NOT a private copy in `FrontMatterFilter`); a `# comment` cannot be line 2 (`---` + `# Heading` is a thematic break + heading), and neither can `http://…` or a `- item`. Consequence for the YAML writer: `YamlWriter.renderKey` quotes every key that is not identifier-shaped (`"a b"`, `"42"`, `"og:title"`), even where YAML would not require it, so whatever `simplifyHtml` emits first (a `` with a space, say) still re-detects. It also quotes a key that is a reserved literal (`"yes":`) — a YAML 1.1 reader would take a bare `yes:` as a boolean key. + Key typing is its own rule (`isTypedPlainKey`), not the value rule: Psych types `yEs:` / `nULL:` keys, but go-yaml v2 decodes a front matter key into a *string*, so its one-letter booleans `y` / `n` — typed as values — must stay plain as keys (`x: 1\ny: 2` was once rewritten to `"y": 2`). The writer's plain-key rule (`isIdentifierKey`) and the detector share their character rules in `YamlKeyLine.kt`, and `YamlWriterTest` "should write every key as a line the front matter detector accepts" pins the two — widen one without the other and every document whose *first* entry has such a key silently loses its front matter on re-parse. **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become ``/``, only a top-level `entry key=title` counts as an existing title. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 7b4ead7..3a17d3e 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -481,7 +481,7 @@ public class YamlParser( var i = start while (i < s.length) { val c = s[i] - if (c == ':' && (i + 1 >= s.length || s[i + 1] == ' ' || s[i + 1] == ',' || s[i + 1] == '}')) break + if (s.isMappingColonAt(i) || c == ':' && (s[i + 1] == ',' || s[i + 1] == '}')) break if (c == ',' || c == '}' || c == ']' || c == '[' || c == '{') return null i++ } @@ -497,8 +497,7 @@ public class YamlParser( while (i < s.length) { val c = s[i] if (c == ',' || c == ']' || c == '}') break - if (c == ':' && (i + 1 >= s.length || s[i + 1] == ' ')) return null - if (c == '#' && i > start && s[i - 1] == ' ') return null + if (s.isMappingColonAt(i) || s.isCommentStartAt(i)) return null i++ } val text = s.substring(start, i).trim() diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index 64ec9e1..6bb956c 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -35,7 +35,8 @@ package com.xemantic.markanywhere.yaml // - the special floats in any letter case (`.Nan`, `+.InF`); // - integers with separators, binary, upper-case and signed prefixes // (`1_000`, `1,000`, `0b101`, `0X1F`, `+0x1F`, `+_1`); -// - floats with separators (`1_000.5`, `1_e5`); +// - floats with separators (`1_000.5`, `1_e5`), and a leading-zero decimal +// that is not octal (`08`, go-yaml v2); // - base-60 numbers (`12:30` an int, `190:20:30.15` a float); // - timestamps with a one-digit month / day (`2024-5-1`) or an offset // without a colon (`2016-01-01 12:00:00 -0500`, Jekyll's documented @@ -88,10 +89,12 @@ private val PSYCH_FLOAT = anchored( // go-yaml v2 deletes every `_` from a text starting with a sign or a digit, // then tries Go's `ParseInt` with base 0 (which reads a `0x` / `0o` / `0b` -// prefix in either case), a `0b` prefix of its own (`-0b` included, which -// lets a sign follow the prefix) and a YAML 1.2 float +// prefix in either case, and a bare leading `0` as octal — so `08` fails +// here and falls through to the float), a `0b` prefix of its own (which +// lets a sign follow an unsigned prefix: `0b-1`, not `-0b-1`) and a YAML 1.2 +// float private val GO_YAML_INT = anchored( - "[-+]?(?:[0-9]+|0[xX][0-9a-fA-F]+|0[oO][0-7]+|0[bB][01]+)|-?0b[-+]?[01]+" + "[-+]?(?:0[0-7]*|[1-9][0-9]*|0[xX][0-9a-fA-F]+|0[oO][0-7]+|0[bB][01]+)|0b[-+][01]+" ) private val GO_YAML_FLOAT = anchored("""[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?""") @@ -102,11 +105,20 @@ private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9](?:_?[0-9])*(?:[eE][-+]?[0-9] // The union of the Psych and PyYAML timestamp shapes (go-yaml v2 decodes a // timestamp into a string); the groups are the date, the time and the -// offset hours, so [isTimestampInRange] can check their values. +// offset, so [isTimestampInRange] can check their values. private val YAML_TIMESTAMP = anchored( """(-?[0-9]{4})-([0-9]{1,2})-([0-9]{1,2})""" + """(?:(?:[Tt]|[ \t]+)([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]*)?""" + - """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}):?(?:[0-9]{2})?))?)?""" + """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}):?([0-9]{2})?))?)?""" +) + +// PyYAML's own timestamp shape: two-digit month and day for a date alone, +// a colon in an offset with minutes. It constructs whatever matches, and +// refuses the document when a value is out of range. +private val PYYAML_TIMESTAMP = anchored( + """[0-9]{4}-[0-9]{2}-[0-9]{2}""" + + """|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?""" + + """(?:[ \t]*(?:Z|[-+][0-9]{1,2}(?::[0-9]{2})?))?""" ) // The `type` a plain scalar resolves to, or null for a string. @@ -127,7 +139,7 @@ internal fun yamlScalarType(text: String): String? { if (first == '.') { if (GO_YAML_DOT_FLOAT.matches(text)) return "float" } else { - val plain = text.replace("_", "") + val plain = if ('_' in text) text.replace("_", "") else text if (GO_YAML_INT.matches(plain)) return "int" if (GO_YAML_FLOAT.matches(plain)) return "float" } @@ -138,16 +150,23 @@ internal fun yamlScalarType(text: String): String? { // Psych checks a date against the calendar, but normalises a date and time // (`2023-02-31T10:00:00` is March 3rd), refusing only the values its `Time` -// cannot take; a date-only shape with a sign is not a timestamp at all. +// cannot take — hour 24 only as `24:00:00`, an offset under a day; a +// date-only shape with a sign is not a timestamp at all. private fun isTimestampInRange(groups: List<String>): Boolean { val year = groups[1].toInt() val month = groups[2].toInt() val day = groups[3].toInt() if (month !in 1..12) return false if (groups[4].isNotEmpty()) { + val hour = groups[4].toInt() + val minute = groups[5].toInt() + val second = groups[6].toInt() val offsetHours = groups[7] - return day in 1..31 && groups[4].toInt() <= 24 && groups[5].toInt() <= 59 && - groups[6].toInt() <= 60 && (offsetHours.isEmpty() || offsetHours.toInt() <= 23) + val offsetMinutes = groups[8] + return day in 1..31 && minute <= 59 && second <= 60 && + (hour <= 23 || hour == 24 && minute == 0 && second == 0) && + (offsetHours.isEmpty() || offsetHours.toInt() <= 23) && + (offsetMinutes.isEmpty() || offsetMinutes.toInt() <= 59) } if (year < 0) return false return day in 1..daysInMonth(year, month) @@ -162,17 +181,37 @@ private fun daysInMonth(year: Int, month: Int): Int = when (month) { // Shapes some reader resolves and then refuses to load, so the whole // document fails: PyYAML's value / merge tags (`=`, `<<`), which its // SafeLoader cannot construct; a base prefix with no digit (`0x_`, PyYAML -// and Psych); an exponent with no mantissa digit (`.e+4`, Psych); a -// timestamp shape [isTimestampInRange] refuses (PyYAML, for a date not in -// the calendar). There is no type to report for them, so the parser keeps -// them strings — DIVERGENCE: the writer still quotes them, which rewrites -// such a plain source value on its first render. +// and Psych); an exponent with no mantissa digit (`.e+4`, Psych); and, in +// [isUnsafePlainScalar], a [PYYAML_TIMESTAMP] shape out of range, which +// PyYAML refuses (`2024-13-45`, `2024-01-01 24:30:00`) — the parser does +// not type it either, or it would not get there. There is no type to report +// for them, so the parser keeps them strings — DIVERGENCE: the writer still +// quotes them, which rewrites such a plain source value on its first render. private val YAML_REJECTED = anchored("""=|<<|[-+]?0[bx][_,]+|[-+]?\.[eE][-+][0-9]+""") // Whether a plain scalar would not read back as this string in every // reader: the parser types it, or some reader refuses it. -internal fun isUnsafePlainScalar(text: String): Boolean = - yamlScalarType(text) != null || YAML_REJECTED.matches(text) || YAML_TIMESTAMP.matches(text) +internal fun isUnsafePlainScalar(text: String): Boolean { + if (yamlScalarType(text) != null) return true + // every refused shape starts with one of these, so the patterns only + // run on a string that can match them + val first = text.firstOrNull() ?: return false + if (first != '=' && first != '<' && first != '-' && first != '+' && first != '.' && first !in '0'..'9') { + return false + } + return YAML_REJECTED.matches(text) || PYYAML_TIMESTAMP.matches(text) +} + +// Whether a plain identifier-shaped mapping key would be read as something +// other than a string: Psych types the boolean and null words in any letter +// case (PyYAML only some cases of them), while go-yaml v2 decodes a front +// matter key into a string, so its one-letter booleans `y` / `n` stay keys. +// The parser never types a key, so this is quoting the writer alone does. +internal fun isTypedPlainKey(key: String): Boolean { + if (key.length > YAML_WORD_MAX_LENGTH) return false + val word = key.lowercase() + return word in YAML_NULLS || word in YAML_BOOLS && word != "y" && word != "n" +} // The two indicators that can end a plain scalar in block context, shared by // [YamlParser] (where it splits a key and strips a comment) and [YamlWriter] diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 52c2e1e..9edb9f5 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -46,10 +46,9 @@ import com.xemantic.markanywhere.SemanticEvent * text is a bare `key:`; `type=seq` / `type=map` with no children are * `[]` / `{}`; * - a key is written plain only when identifier-shaped (letters, digits, - * `_`, `-`, `.`, starting with a letter or `_`) and not a shape a plain - * scalar would be typed as — a YAML 1.1 reader takes a bare `yes:`, `y:` - * or `nULL:` as a boolean / null key — and double-quoted otherwise, even - * where YAML would not require it; + * `_`, `-`, `.`, starting with a letter or `_`) and not a key a reader + * would type — Psych takes a bare `yes:` or `nULL:` as a boolean / null + * key — and double-quoted otherwise, even where YAML would not require it; * - a `text` child of a container (the parser's verbatim fallback for a * line outside its YAML subset) is written as-is, on its own line(s). * @@ -232,13 +231,13 @@ public class YamlWriter( } // A key is written plain only when it is identifier-shaped and would not - // be typed (`yes`, `y`, `nULL`): the Markdown parser's front matter detection requires + // be typed (`yes`, `nULL`): the Markdown parser's front matter detection requires // the first line to pass `isYamlKeyLine` (such a key, or a quoted one), // so quoting everything else keeps whatever entry comes first // re-detectable. The character rules are shared with that check. private fun renderKey(key: String?): String { val k = key ?: "" - return if (isIdentifierKey(k) && !isUnsafePlainScalar(k)) k else quoted(k) + return if (isIdentifierKey(k) && !isTypedPlainKey(k)) k else quoted(k) } } diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 5573c38..1e62558 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -381,8 +381,9 @@ class YamlParserTest { fun `should not type a shape no reader types`() = runTest { // given — a trailing or doubled separator, a dot before an // underscore, an exponent with no mantissa digit, a date that is not - // in the calendar, a date-only shape with a sign and a minute Psych's - // Time refuses: PyYAML, Psych and go-yaml v2 all read strings + // in the calendar, a date-only shape with a sign, a minute Psych's + // Time refuses and a sign after a signed binary prefix: PyYAML, Psych + // and go-yaml v2 all read strings val textFlow = """ a: 1, b: 1,,2 @@ -391,6 +392,8 @@ class YamlParserTest { e: 2024-2-30 f: -2024-01-01 g: 2024-01-01T12:60:00 + h: -0b-1 + i: -0b+1 """.trimIndent().chunkedRandomly().asFlow() // when @@ -405,6 +408,8 @@ class YamlParserTest { "entry"("key" to "e") { +"2024-2-30" } "entry"("key" to "f") { +"-2024-01-01" } "entry"("key" to "g") { +"2024-01-01T12:60:00" } + "entry"("key" to "h") { +"-0b-1" } + "entry"("key" to "i") { +"-0b+1" } } } @@ -412,7 +417,8 @@ class YamlParserTest { fun `should DIVERGENCE not type a plain scalar a reader refuses to load`() = runTest { // given — PyYAML resolves `=` and `<<` to its value / merge tags and // then refuses to construct them; it refuses a date not in the - // calendar, and PyYAML / Psych refuse a base prefix with no digit or + // calendar or a time / offset out of range (which Psych reads as a + // string), and PyYAML / Psych refuse a base prefix with no digit or // an exponent with no mantissa: none of them has a type to report val textFlow = """ a: = @@ -420,6 +426,8 @@ class YamlParserTest { c: 2024-13-45 d: 0x_ e: .e+4 + f: 2024-01-01 24:30:00 + g: 2024-01-01 10:00:00 +23:99 """.trimIndent().chunkedRandomly().asFlow() // when @@ -432,6 +440,34 @@ class YamlParserTest { "entry"("key" to "c") { +"2024-13-45" } "entry"("key" to "d") { +"0x_" } "entry"("key" to "e") { +".e+4" } + "entry"("key" to "f") { +"2024-01-01 24:30:00" } + "entry"("key" to "g") { +"2024-01-01 10:00:00 +23:99" } + } + } + + @Test + fun `should type a leading-zero decimal go-yaml v2 reads as a float`() = runTest { + // given — PyYAML and Psych read these as strings; go-yaml v2 takes + // the leading zero as an octal prefix, fails on the 8 or 9 and falls + // back to a float + val textFlow = """ + a: 08 + b: 019 + c: 0_9 + d: -08 + e: 07 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a", "type" to "float") { +"08" } + "entry"("key" to "b", "type" to "float") { +"019" } + "entry"("key" to "c", "type" to "float") { +"0_9" } + "entry"("key" to "d", "type" to "float") { +"-08" } + "entry"("key" to "e", "type" to "int") { +"07" } } } @@ -790,8 +826,40 @@ class YamlParserTest { } } + @Test + fun `should read a colon before a tab as a mapping indicator in a flow mapping`() = runTest { + // given + val textFlow = "geo: {lat:\t1.5}\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "geo") { + "entry"("key" to "lat", "type" to "float") { +"1.5" } + } + } + } + // --- verbatim fallback (DIVERGENCE) ----------------------------------- + @Test + fun `should read a hash after a tab as a comment in a flow sequence`() = runTest { + // given — the comment leaves the sequence unterminated, which every + // reader refuses, so the line is outside the subset + val textFlow = "title: x\ntags: [a\t#b]\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "title") { +"x" } + +"tags: [a\t#b]\n" + } + } + @Test fun `should read a hash after a tab as a comment in a key line`() = runTest { // given — `#` after any whitespace starts a comment, so the colon diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 347f19b..04eaf99 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -157,11 +157,10 @@ class YamlRoundTripTest { @Test fun `should quote a key a YAML 1_1 reader would type`() = runTest { - // given — Psych reads booleans in any letter case, go-yaml v2 reads - // `y` / `n` as booleans + // given — Psych reads booleans and nulls in any letter case val events = semanticEvents { "entry"("key" to "yEs") { +"v" } - "entry"("key" to "n") { +"v" } + "entry"("key" to "nULL") { +"v" } } // when @@ -169,10 +168,24 @@ class YamlRoundTripTest { val reparsed = flowOf(rendered).parseYaml() // then - rendered sameAs "\"yEs\": v\n\"n\": v\n" + rendered sameAs "\"yEs\": v\n\"nULL\": v\n" reparsed.mergeAdjacentText() sameAs events } + @Test + fun `should not quote a one-letter boolean key`() = runTest { + // given — go-yaml v2 reads a plain `y` / `n` value as a boolean, but + // decodes a front matter key into a string, and neither Psych nor + // PyYAML types them at all + val source = "x: 1\ny: 2\nn: 3\nY: 4\nN: 5\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + + // then + rendered sameAs source + } + @Test fun `should keep YAML 1_1 typed front matter values as written`() = runTest { // given — Jekyll's documented date format, a short date, go-yaml v2 @@ -216,7 +229,7 @@ class YamlRoundTripTest { @Test fun `should not quote a shape no reader types`() = runTest { // given - val values = listOf("1,", "1,,2", "._0", ".e5") + val values = listOf("1,", "1,,2", "._0", ".e5", "2024-2-30", "2024-5-32") for (value in values) { val events = semanticEvents { "entry"("key" to "k") { +value } @@ -237,14 +250,15 @@ class YamlRoundTripTest { // given — the parser leaves these strings (no reader has a type for // them), but PyYAML or Psych refuse the whole document when they are // plain, so the writer quotes them and the first render rewrites the - // source; a date-shaped string outside the calendar is quoted too + // source; PyYAML refuses a timestamp shape out of range val source = """ a: = b: << c: 2024-13-45 - d: 2024-2-30 + d: 2024-01-01 24:30:00 e: 0x_ f: .e+4 + g: 2024-01-01 10:00:00 +23:99 """.trimIndent() + "\n" // when @@ -255,9 +269,10 @@ class YamlRoundTripTest { a: "=" b: "<<" c: "2024-13-45" - d: "2024-2-30" + d: "2024-01-01 24:30:00" e: "0x_" f: ".e+4" + g: "2024-01-01 10:00:00 +23:99" """.trimIndent() + "\n" } From 84166297666f0040cc4319050cdca629d0180e2c Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Thu, 24 Sep 2026 21:52:50 +0200 Subject: [PATCH 07/10] Bound timestamp offsets and comments the way the readers do - split a timestamp offset as Psych does (at the colon, else two hour digits) and bound the whole offset under a day, so `+05:99` is typed and `+530` / `+2400` stay strings - write base-prefixed digit runs so a long near-miss resolves in linear time instead of backtracking quadratically - end a flow mapping key at a ` #` comment, and start a comment at any `#` after a closed quoted scalar or flow collection - share the mapping-colon check with the key line detector Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --- CLAUDE.md | 1 - .../src/commonMain/kotlin/YamlKeyLine.kt | 4 +- .../src/commonMain/kotlin/YamlParser.kt | 12 +-- .../src/commonMain/kotlin/YamlScalarType.kt | 34 +++++--- .../src/commonTest/kotlin/YamlParserTest.kt | 77 ++++++++++++++++++- .../commonTest/kotlin/YamlRoundTripTest.kt | 5 +- 6 files changed, 110 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index dee3e38..6c399dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -546,7 +546,6 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' The writer's quoting rules reuse the parser's `yamlScalarType` (`YamlScalarType.kt`, one copy of the reserved literals / int / float / timestamp rules) and are pinned against it by `YamlRoundTripTest` in `markanywhere-yaml`; `FrontMatterRoundTripTest` in `markanywhere-parse` (whose tests depend on render) covers only the fences and the wrapper — extend the yaml one when you touch either side of the codec. A shape some front matter reader (Psych/Jekyll, PyYAML, go-yaml v2) types must be typed by the **parser**, not merely quoted by the writer — writer-only quoting rewrites the source on round-trip (Jekyll's `date: 2016-01-01 12:00:00 -0500` came back as a quoted string). The converse holds too: a shape **no** reader types must not be typed, or `type=int` stops meaning a number (`1,` once came back `type=int`). - So `YamlScalarType.kt` transcribes each reader's number rules separately instead of merging them into one pattern — the separator rules differ per reader (go-yaml v2 deletes every `_` before parsing, so `1_e5` / `+_1` are numbers to Hugo), and a merged pattern types shapes none of them does. Check a change against the real readers (PyYAML `safe_load`, Ruby `Psych.unsafe_load`, `gopkg.in/yaml.v2` into `map[string]interface{}`) over generated strings, not by reading their docs — go-yaml v2 decodes a timestamp into a *string*, Psych normalises an out-of-range date *and time* but rejects an out-of-range date alone. The one sanctioned writer-only quoting is a shape a reader **refuses to load** (`=`, `<<`, `0x_`, `.e+4`, `2024-13-45`): there is no type to report, so the parser keeps it a string and the rewrite is pinned as `DIVERGENCE` in `YamlRoundTripTest`. `renderYaml()` **keeps the trailing newline** (unlike `renderMarkdown()` / `renderHtml()`, which suppress one): a `|+` block scalar at the end of the document needs it, and dropping it broke the round-trip for `z\n\n`. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt index f953e37..89bdedc 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt @@ -44,9 +44,7 @@ public fun isYamlKeyLine(line: String): Boolean { j } while (i < line.length && line[i] == ' ') i++ - if (i >= line.length || line[i] != ':') return false - i++ - return i == line.length || line[i] == ' ' || line[i] == '\t' + return i < line.length && line.isMappingColonAt(i) } // A key [YamlWriter] may write plain: identifier-shaped, so it passes diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 3a17d3e..939baa7 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -346,8 +346,7 @@ public class YamlParser( val (key, end) = scanQuoted(content) ?: return null var i = end while (i < content.length && content[i] == ' ') i++ - if (i >= content.length || content[i] != ':') return null - if (i + 1 < content.length && content[i + 1] != ' ' && content[i + 1] != '\t') return null + if (i >= content.length || !content.isMappingColonAt(i)) return null return key to content.substring(i + 1) } if (first in KEY_FORBIDDEN_START) return null @@ -482,7 +481,7 @@ public class YamlParser( while (i < s.length) { val c = s[i] if (s.isMappingColonAt(i) || c == ':' && (s[i + 1] == ',' || s[i + 1] == '}')) break - if (c == ',' || c == '}' || c == ']' || c == '[' || c == '{') return null + if (c == ',' || c == '}' || c == ']' || c == '[' || c == '{' || s.isCommentStartAt(i)) return null i++ } val key = s.substring(start, i).trim() @@ -546,10 +545,13 @@ private fun skipSpaces(s: String, from: Int): Int { private fun isSequenceEntry(content: String): Boolean = content == "-" || content.startsWith("- ") || content.startsWith("-\t") -// True when only whitespace or a `#` comment follows index [from]. +// True when only whitespace or a `#` comment follows index [from], the end +// of a quoted scalar or a flow collection. Not [isCommentStartAt]: that is +// the rule inside a plain scalar, while after a closed token every reader +// (libyaml, PyYAML) starts a comment at any `#`, even with no space (`"x"#b`). private fun isBlankOrComment(s: String, from: Int): Boolean { val i = skipSpaces(s, from) - return i >= s.length || (s[i] == '#' && (i == from || s[i - 1] == ' ' || s[i - 1] == '\t')) + return i >= s.length || s[i] == '#' } // A ` #` (hash preceded by whitespace) starts a trailing comment. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index 6bb956c..f89819f 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -65,10 +65,13 @@ private val YAML_WORD_MAX_LENGTH = private fun anchored(pattern: String) = Regex("^(?:$pattern)$") // PyYAML's resolver, less a base prefix with no digit (`0x_`), which it -// resolves and then fails to construct +// resolves and then fails to construct. The digits after a base prefix are +// written `_*[01][01_]*`, not `[01_]*[01][01_]*`: the same strings, but the +// latter can split a digit run many ways and backtracks quadratically over +// a long near-miss. private val PYYAML_INT = anchored( - "[-+]?0b[01_]*[01][01_]*|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)" + - "|[-+]?0x[0-9a-fA-F_]*[0-9a-fA-F][0-9a-fA-F_]*|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+" + "[-+]?0b_*[01][01_]*|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)" + + "|[-+]?0x_*[0-9a-fA-F][0-9a-fA-F_]*|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+" ) private val PYYAML_FLOAT = anchored( @@ -78,8 +81,8 @@ private val PYYAML_FLOAT = anchored( // Psych's scalar scanner, with its default (legacy) integers that allow `,` private val PSYCH_INT = anchored( - "[-+]?0b[01_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?(?:0|[1-9](?:[0-9]|[,_][0-9])*)" + - "|[-+]?0x[0-9a-fA-F_,]*[0-9a-fA-F][0-9a-fA-F_,]*|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}" + "[-+]?0b[_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?(?:0|[1-9](?:[0-9]|[,_][0-9])*)" + + "|[-+]?0x[_,]*[0-9a-fA-F][0-9a-fA-F_,]*|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}" ) private val PSYCH_FLOAT = anchored( @@ -105,11 +108,11 @@ private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9](?:_?[0-9])*(?:[eE][-+]?[0-9] // The union of the Psych and PyYAML timestamp shapes (go-yaml v2 decodes a // timestamp into a string); the groups are the date, the time and the -// offset, so [isTimestampInRange] can check their values. +// offset digits, so [isTimestampInRange] can check their values. private val YAML_TIMESTAMP = anchored( """(-?[0-9]{4})-([0-9]{1,2})-([0-9]{1,2})""" + """(?:(?:[Tt]|[ \t]+)([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]*)?""" + - """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}):?([0-9]{2})?))?)?""" + """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}:?(?:[0-9]{2})?)))?)?""" ) // PyYAML's own timestamp shape: two-digit month and day for a date alone, @@ -161,17 +164,26 @@ private fun isTimestampInRange(groups: List<String>): Boolean { val hour = groups[4].toInt() val minute = groups[5].toInt() val second = groups[6].toInt() - val offsetHours = groups[7] - val offsetMinutes = groups[8] + val offset = groups[7] return day in 1..31 && minute <= 59 && second <= 60 && (hour <= 23 || hour == 24 && minute == 0 && second == 0) && - (offsetHours.isEmpty() || offsetHours.toInt() <= 23) && - (offsetMinutes.isEmpty() || offsetMinutes.toInt() <= 59) + (offset.isEmpty() || offsetMinutes(offset) < 24 * 60) } if (year < 0) return false return day in 1..daysInMonth(year, month) } +// The offset as Psych's `parse_time` splits it: at the colon when there is +// one, else up to two digits of hours and the rest as minutes (`+530` is +// 53 hours, `+070` is 7). Minutes past 59 carry into the hours (`+05:99`). +private fun offsetMinutes(offset: String): Int { + val colon = offset.indexOf(':') + val split = if (colon >= 0) colon else minOf(2, offset.length) + val hours = offset.substring(0, split).toInt() + val minutes = offset.substring(if (colon >= 0) split + 1 else split) + return hours * 60 + (if (minutes.isEmpty()) 0 else minutes.toInt()) +} + private fun daysInMonth(year: Int, month: Int): Int = when (month) { 2 -> if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) 29 else 28 4, 6, 9, 11 -> 30 diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 1e62558..3bcff5c 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -23,7 +23,11 @@ import com.xemantic.markanywhere.test.sameAs import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.test.runTest +import com.xemantic.kotlin.test.assert import kotlin.test.Test +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource +import kotlin.time.measureTime /** * Specifies the streaming YAML parser and its event representation. @@ -347,6 +351,33 @@ class YamlParserTest { } } + @Test + fun `should type a timestamp offset the way Psych splits and bounds it`() = runTest { + // given — Psych reads up to two digits as the offset hours and the + // rest as its minutes, and only bounds the whole offset under a day: + // minutes past 59 carry into the hours, and a three-digit offset is + // two hour digits and one minute digit + val textFlow = """ + a: 2024-01-01 10:00:00 +05:99 + b: 2024-01-01 10:00:00 +0599 + c: 2024-01-01 10:00:00 +19:70 + d: 2024-01-01 10:00:00 +070 + e: 2024-01-01 10:00:00 -2359 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a", "type" to "timestamp") { +"2024-01-01 10:00:00 +05:99" } + "entry"("key" to "b", "type" to "timestamp") { +"2024-01-01 10:00:00 +0599" } + "entry"("key" to "c", "type" to "timestamp") { +"2024-01-01 10:00:00 +19:70" } + "entry"("key" to "d", "type" to "timestamp") { +"2024-01-01 10:00:00 +070" } + "entry"("key" to "e", "type" to "timestamp") { +"2024-01-01 10:00:00 -2359" } + } + } + @Test fun `should type the number shapes go-yaml v2 reads once underscores are removed`() = runTest { // given — go-yaml v2 (Hugo) deletes every `_` before it parses a @@ -382,8 +413,9 @@ class YamlParserTest { // given — a trailing or doubled separator, a dot before an // underscore, an exponent with no mantissa digit, a date that is not // in the calendar, a date-only shape with a sign, a minute Psych's - // Time refuses and a sign after a signed binary prefix: PyYAML, Psych - // and go-yaml v2 all read strings + // Time refuses, a sign after a signed binary prefix and an offset + // Psych splits into 53 hours or bounds at a day: PyYAML, Psych and + // go-yaml v2 all read strings val textFlow = """ a: 1, b: 1,,2 @@ -394,6 +426,8 @@ class YamlParserTest { g: 2024-01-01T12:60:00 h: -0b-1 i: -0b+1 + j: 2024-01-01 10:00:00 +530 + k: 2024-01-01 10:00:00 +2400 """.trimIndent().chunkedRandomly().asFlow() // when @@ -410,6 +444,8 @@ class YamlParserTest { "entry"("key" to "g") { +"2024-01-01T12:60:00" } "entry"("key" to "h") { +"-0b-1" } "entry"("key" to "i") { +"-0b+1" } + "entry"("key" to "j") { +"2024-01-01 10:00:00 +530" } + "entry"("key" to "k") { +"2024-01-01 10:00:00 +2400" } } } @@ -445,6 +481,27 @@ class YamlParserTest { } } + @Test + fun `should resolve a long near-miss of a base prefixed number in linear time`() { + // given — a base prefix, digits and one character outside the base: + // a pattern that lets the digits match two ways backtracks + // quadratically over them, and every plain scalar is typed + val values = listOf("0x", "0b", "+0x", "-0b").map { prefix -> + prefix + "1".repeat(50_000) + "g" + } + + // when + val elapsed = TimeSource.Monotonic.measureTime { + for (value in values) { + assert(yamlScalarType(value) == null) + assert(!isUnsafePlainScalar(value)) + } + } + + // then + assert(elapsed < 2.seconds) + } + @Test fun `should type a leading-zero decimal go-yaml v2 reads as a float`() = runTest { // given — PyYAML and Psych read these as strings; go-yaml v2 takes @@ -876,6 +933,22 @@ class YamlParserTest { } } + @Test + fun `should read a hash after a space as a comment in a flow mapping key`() = runTest { + // given — the comment leaves the mapping unterminated, which every + // reader refuses, so the line is outside the subset + val textFlow = "title: x\ngeo: {a #b: c}\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "title") { +"x" } + +"geo: {a #b: c}\n" + } + } + @Test fun `should DIVERGENCE keep an unrecognised line verbatim`() = runTest { // given — a complex key, a multi-line flow sequence, an unterminated diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 04eaf99..fb8034e 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -198,6 +198,7 @@ class YamlRoundTripTest { mixed: yEs big: 1_000 time: 12:30 + offset: 2024-01-01 10:00:00 +05:99 """.trimIndent() + "\n" // when @@ -229,7 +230,9 @@ class YamlRoundTripTest { @Test fun `should not quote a shape no reader types`() = runTest { // given - val values = listOf("1,", "1,,2", "._0", ".e5", "2024-2-30", "2024-5-32") + val values = listOf( + "1,", "1,,2", "._0", ".e5", "2024-2-30", "2024-5-32", "2024-01-01 10:00:00 +530", + ) for (value in values) { val events = semanticEvents { "entry"("key" to "k") { +value } From 5ca49997e0d8530b8f1c91030b5bf05930da8250 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Thu, 24 Sep 2026 22:23:51 +0200 Subject: [PATCH 08/10] Match numbers without repeated regex groups and replace lone surrogates - rewrite every YAML number pattern that repeated a group (`(?:_?[0-9])*`, `(?::[0-9])+`) as a character-class run plus a hand check, since the JVM regex engine recurses once per repetition and a long digit run threw StackOverflowError - reject a string holding a character no numeric or timestamp shape can contain before running any pattern - require the offset minutes after a colon, so `+5:` is not a timestamp - refuse a date-only shape with a signed zero year (`-0000-01-01`) - write a lone surrogate as U+FFFD (DIVERGENCE: lossy), as Psych and go-yaml refuse its `\u` escape Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --- CLAUDE.md | 2 + .../src/commonMain/kotlin/YamlScalarType.kt | 60 ++++++++++++++++--- .../src/commonMain/kotlin/YamlWriter.kt | 9 ++- .../src/commonTest/kotlin/YamlParserTest.kt | 52 ++++++++++++++-- .../commonTest/kotlin/YamlRoundTripTest.kt | 53 ++++++++++++++-- 5 files changed, 156 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6c399dd..70881cb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -411,6 +411,8 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' `renderDumpMarkdown` (Gradle task, `RenderDumpMarkdown.kt` in jvmTest) runs the full `transformHtmlToMarkdown` pipeline over every dump to `build/renderedMarkdown/<name>.md` — the canonical way to regenerate the per-dump golden strings (`dumps/OpenjurTest`, `dumps/HackerNewsTest`) after a pipeline change. - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. +- A `commonMain` `Regex` applied to untrusted input must not **repeat a group** (`(?:_?[0-9])*`, `(?::[0-9])+`): `java.util.regex` matches each repetition of a group one stack frame deeper, so a few thousand repetitions throw `StackOverflowError` on the JVM (an `Error`, which nothing catches — it takes the whole pipeline down). + Repeat a character class instead (`[0-9_]*`) and check what the group expressed by hand — see `hasSeparatorsBeforeDigits` / `isBase60Tail` in `YamlScalarType.kt`, pinned by `YamlParserTest` "long run of digits". - Facts of the HTML standard needed by more than one of `parse` / `render` / `html` — HTML whitespace (`isHtmlWhitespace` / `isHtmlBlank`), void and raw text elements, `Mark.classList` — live in the leaf module `markanywhere-html-spec` (depends only on `api`), never in `markanywhere-api` (which describes only the event model) and never as private copies. Only spec facts go there, not project policy (e.g. which tags render as blocks in Markdown); the parser's `isFlankWhitespace` is deliberately a different set (CommonMark "Unicode whitespace", which includes NBSP). - `MutableMap.putIfAbsent` is **JVM-only** — not in the common stdlib, so it can't be used in `commonMain`. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index f89819f..e2f18d1 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -62,6 +62,11 @@ private val YAML_WORD_MAX_LENGTH = // on a prefix (`0x1F` matched `[0-9]+` as `0`, a full timestamp matched its // date-only branch) and the whole-input check then fails — JVM backtracks // across the branches and never showed it. +// No pattern repeats a group (`(?:_?[0-9])*`): the JVM engine matches each +// repetition of a group one stack frame deeper, so a long digit run +// overflows the stack. A digit run is a character class instead, and the +// rule the group expressed is checked by hand ([hasSeparatorsBeforeDigits], +// [hasUnderscoresBetweenDigits], [isBase60Tail]). private fun anchored(pattern: String) = Regex("^(?:$pattern)$") // PyYAML's resolver, less a base prefix with no digit (`0x_`), which it @@ -71,20 +76,29 @@ private fun anchored(pattern: String) = Regex("^(?:$pattern)$") // a long near-miss. private val PYYAML_INT = anchored( "[-+]?0b_*[01][01_]*|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)" + - "|[-+]?0x_*[0-9a-fA-F][0-9a-fA-F_]*|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+" + "|[-+]?0x_*[0-9a-fA-F][0-9a-fA-F_]*" ) private val PYYAML_FLOAT = anchored( - """[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?""" + - """|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*""" + """[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?""" ) +// PyYAML's base-60 int and float, `(?::[0-5]?[0-9])+` written as a run the +// group is the tail of, which [isBase60Tail] then checks +private val PYYAML_BASE60_INT = anchored("[-+]?[1-9][0-9_]*(:[0-9:]*)") + +private val PYYAML_BASE60_FLOAT = anchored("""[-+]?[0-9][0-9_]*(:[0-9:]*)\.[0-9_]*""") + // Psych's scalar scanner, with its default (legacy) integers that allow `,` private val PSYCH_INT = anchored( - "[-+]?0b[_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?(?:0|[1-9](?:[0-9]|[,_][0-9])*)" + + "[-+]?0b[_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?0" + "|[-+]?0x[_,]*[0-9a-fA-F][0-9a-fA-F_,]*|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}" ) +// … and its decimal, `[1-9](?:[0-9]|[,_][0-9])*`: a separator only before a +// digit, which [hasSeparatorsBeforeDigits] checks +private val PSYCH_DECIMAL_INT = anchored("[-+]?[1-9][0-9,_]*") + private val PSYCH_FLOAT = anchored( """[-+]?(?:[0-9][0-9_,]*\.[0-9]*|\.[0-9]+)(?:[eE][-+][0-9]+)?""" + """|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}\.[0-9_]*""" @@ -104,7 +118,8 @@ private val GO_YAML_FLOAT = anchored("""[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?: // … but hands a text starting with `.` to Go's `ParseFloat` as is, which // takes an `_` only between two digits -private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9](?:_?[0-9])*(?:[eE][-+]?[0-9](?:_?[0-9])*)?""") +// ([hasUnderscoresBetweenDigits] checks that) +private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9][0-9_]*(?:[eE][-+]?[0-9][0-9_]*)?""") // The union of the Psych and PyYAML timestamp shapes (go-yaml v2 decodes a // timestamp into a string); the groups are the date, the time and the @@ -112,7 +127,7 @@ private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9](?:_?[0-9])*(?:[eE][-+]?[0-9] private val YAML_TIMESTAMP = anchored( """(-?[0-9]{4})-([0-9]{1,2})-([0-9]{1,2})""" + """(?:(?:[Tt]|[ \t]+)([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]*)?""" + - """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}:?(?:[0-9]{2})?)))?)?""" + """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}(?::?[0-9]{2})?)))?)?""" ) // PyYAML's own timestamp shape: two-digit month and day for a date alone, @@ -124,6 +139,14 @@ private val PYYAML_TIMESTAMP = anchored( """(?:[ \t]*(?:Z|[-+][0-9]{1,2}(?::[0-9]{2})?))?""" ) +// Every character a numeric or timestamp shape can hold, so a string with +// any other one is a string without running a pattern: digits, signs, +// separators, base prefixes and hex digits, exponents, and a timestamp's +// `T` / `Z` and spaces. +private fun Char.canBeNumeric(): Boolean = + this in '0'..'9' || this in "+-._,:" || this in 'a'..'f' || this in 'A'..'F' || + this in "xXoO tTZ\t" + // The `type` a plain scalar resolves to, or null for a string. internal fun yamlScalarType(text: String): String? { if (text.isEmpty()) return null @@ -137,10 +160,14 @@ internal fun yamlScalarType(text: String): String? { // patterns only run on a string that can match them val first = text[0] if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return null + if (!text.all { it.canBeNumeric() }) return null if (PYYAML_INT.matches(text) || PSYCH_INT.matches(text)) return "int" + if (PSYCH_DECIMAL_INT.matches(text) && text.hasSeparatorsBeforeDigits()) return "int" + if (PYYAML_BASE60_INT.matchEntire(text)?.let { isBase60Tail(it.groupValues[1]) } == true) return "int" if (PYYAML_FLOAT.matches(text) || PSYCH_FLOAT.matches(text)) return "float" + if (PYYAML_BASE60_FLOAT.matchEntire(text)?.let { isBase60Tail(it.groupValues[1]) } == true) return "float" if (first == '.') { - if (GO_YAML_DOT_FLOAT.matches(text)) return "float" + if (GO_YAML_DOT_FLOAT.matches(text) && text.hasUnderscoresBetweenDigits()) return "float" } else { val plain = if ('_' in text) text.replace("_", "") else text if (GO_YAML_INT.matches(plain)) return "int" @@ -151,6 +178,22 @@ internal fun yamlScalarType(text: String): String? { return null } +// Whether every `,` / `_` is followed by a digit. +private fun String.hasSeparatorsBeforeDigits(): Boolean = indices.all { i -> + this[i] != ',' && this[i] != '_' || i + 1 < length && this[i + 1] in '0'..'9' +} + +// Whether every `_` sits between two digits. +private fun String.hasUnderscoresBetweenDigits(): Boolean = indices.all { i -> + this[i] != '_' || i > 0 && this[i - 1] in '0'..'9' && i + 1 < length && this[i + 1] in '0'..'9' +} + +// Whether [tail] is one or more `:` segments of `[0-5]?[0-9]`. +private fun isBase60Tail(tail: String): Boolean = + tail.split(':').drop(1).all { segment -> + segment.length == 1 || segment.length == 2 && segment[0] in '0'..'5' + } + // Psych checks a date against the calendar, but normalises a date and time // (`2023-02-31T10:00:00` is March 3rd), refusing only the values its `Time` // cannot take — hour 24 only as `24:00:00`, an offset under a day; a @@ -169,7 +212,8 @@ private fun isTimestampInRange(groups: List<String>): Boolean { (hour <= 23 || hour == 24 && minute == 0 && second == 0) && (offset.isEmpty() || offsetMinutes(offset) < 24 * 60) } - if (year < 0) return false + // `-0000` is a signed year too, though its value is not negative + if (groups[1].startsWith('-')) return false return day in 1..daysInMonth(year, month) } diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 9edb9f5..328c372 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -252,7 +252,8 @@ private const val YAML_INDICATORS = "-?:,[]{}#&*!|>'\"%@`" // document; a byte order mark, which must not appear inside a document // (§5.2); or a line break for a YAML 1.1 reader (NEL, the Unicode line and // paragraph separators). A lone surrogate has no valid YAML representation -// at all; its `\u` escape is still the only way to keep it. Tab, line feed +// at all — raw it is unprintable, and Psych and go-yaml refuse its `\u` +// escape — so [quoted] writes it as U+FFFD (DIVERGENCE: lossy). Tab, line feed // and carriage return are printable — each writer path decides where it can // keep them. private fun String.isYamlUnprintableAt(i: Int): Boolean { @@ -292,7 +293,11 @@ private fun quoted(s: String): String = buildString { '\n' -> +"\\n" '\r' -> +"\\r" '\t' -> +"\\t" - else -> if (s.isYamlUnprintableAt(i)) +("\\u" + c.code.toString(16).padStart(4, '0')) else +c + else -> when { + c.isSurrogate() && s.isYamlUnprintableAt(i) -> +'\uFFFD' + s.isYamlUnprintableAt(i) -> +("\\u" + c.code.toString(16).padStart(4, '0')) + else -> +c + } } +'"' } diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 3bcff5c..74feb3a 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -378,6 +378,19 @@ class YamlParserTest { } } + @Test + fun `should not type a timestamp whose offset ends in a colon`() { + // given — a `:` ending a plain scalar is a mapping indicator, so + // Psych refuses the line: the shape can never be a timestamp + val values = listOf("2024-01-01 12:00:00 +5:", "2024-01-01 12:00:00 -05:") + + // when + val types = values.map { yamlScalarType(it) } + + // then + assert(types == listOf(null, null)) + } + @Test fun `should type the number shapes go-yaml v2 reads once underscores are removed`() = runTest { // given — go-yaml v2 (Hugo) deletes every `_` before it parses a @@ -413,9 +426,10 @@ class YamlParserTest { // given — a trailing or doubled separator, a dot before an // underscore, an exponent with no mantissa digit, a date that is not // in the calendar, a date-only shape with a sign, a minute Psych's - // Time refuses, a sign after a signed binary prefix and an offset - // Psych splits into 53 hours or bounds at a day: PyYAML, Psych and - // go-yaml v2 all read strings + // Time refuses, a sign after a signed binary prefix, an offset + // Psych splits into 53 hours or bounds at a day and a date-only + // shape with a signed zero year: PyYAML, Psych and go-yaml v2 all + // read strings val textFlow = """ a: 1, b: 1,,2 @@ -428,6 +442,7 @@ class YamlParserTest { i: -0b+1 j: 2024-01-01 10:00:00 +530 k: 2024-01-01 10:00:00 +2400 + l: -0000-01-01 """.trimIndent().chunkedRandomly().asFlow() // when @@ -446,6 +461,7 @@ class YamlParserTest { "entry"("key" to "i") { +"-0b+1" } "entry"("key" to "j") { +"2024-01-01 10:00:00 +530" } "entry"("key" to "k") { +"2024-01-01 10:00:00 +2400" } + "entry"("key" to "l") { +"-0000-01-01" } } } @@ -498,8 +514,36 @@ class YamlParserTest { } } + // then — linear takes milliseconds, quadratic minutes; the bound + // is far from both so a slow runner cannot trip it + assert(elapsed < 20.seconds) + } + + @Test + fun `should type a long run of digits without exhausting the stack`() { + // given — a repeated group is matched recursively by the JVM regex + // engine, one frame per repetition, so a pattern that repeats one + // over a digit run overflows the stack on a long enough value + val digits = "1".repeat(100_000) + val values = mapOf( + digits to "int", + "1,".repeat(50_000) + "1" to "int", + ".$digits" to "float", + ".1" + "_1".repeat(50_000) to "float", + "1" + ":1".repeat(50_000) to "int", + "1" + ":1".repeat(50_000) + ".5" to "float", + "${digits}g" to null, + "1,".repeat(50_000) + "g" to null, + ".${digits}g" to null, + ".1" + "_1".repeat(50_000) + "g" to null, + "1" + ":1".repeat(50_000) + "g" to null, + ) + + // when + val types = values.keys.map { yamlScalarType(it) } + // then - assert(elapsed < 2.seconds) + assert(types == values.values.toList()) } @Test diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index fb8034e..9c963fb 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -301,18 +301,16 @@ class YamlRoundTripTest { @Test fun `should escape every character YAML does not allow in a document`() = runTest { // given — C1 controls (mojibake like a Windows-1252 apostrophe read - // as Latin-1), the U+FFFE / U+FFFF non-characters and lone surrogates - // are outside YAML's printable set (YAML 1.2 §5.1): Psych and PyYAML - // refuse the whole document when they appear raw, even in a block - // scalar; a byte order mark must not appear inside a document (§5.2) + // as Latin-1) and the U+FFFE / U+FFFF non-characters are outside + // YAML's printable set (YAML 1.2 §5.1): Psych and PyYAML refuse + // the whole document when they appear raw, even in a block scalar; + // a byte order mark must not appear inside a document (§5.2) val values = listOf( "It\u0092s" to "\"It\\u0092s\"", "a\u0080b" to "\"a\\u0080b\"", "\u009f" to "\"\\u009f\"", "x\uFFFEy" to "\"x\\ufffey\"", "x\uFFFF" to "\"x\\uffff\"", - "lone \uD800 high" to "\"lone \\ud800 high\"", - "lone \uDC00 low" to "\"lone \\udc00 low\"", "first\u0092\nsecond" to "\"first\\u0092\\nsecond\"", "\uFEFFTitle" to "\"\\ufeffTitle\"", "first\n\uFEFFsecond" to "\"first\\n\\ufeffsecond\"", @@ -333,6 +331,49 @@ class YamlRoundTripTest { } } + @Test + fun `should DIVERGENCE write a lone surrogate as the replacement character`() = runTest { + // given — a lone surrogate (a JS string cut mid-pair) has no YAML + // representation: raw it is outside the printable set, and Psych and + // go-yaml refuse its `\u` escape as an invalid code point, so either + // loses the whole document; U+FFFD keeps it loadable, but the + // surrogate does not survive the round-trip + val values = listOf( + "lone \uD800 high" to "lone \uFFFD high", + "lone \uDC00 low" to "lone \uFFFD low", + "\uDC00\uD800" to "\uFFFD\uFFFD", + ) + for ((value, replaced) in values) { + val events = semanticEvents { + "entry"("key" to "k") { +value } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs "k: \"$replaced\"\n" + reparsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "k") { +replaced } + } + } + } + + @Test + fun `should DIVERGENCE write a lone surrogate in a key as the replacement character`() = runTest { + // given + val events = semanticEvents { + "entry"("key" to "a\uD800") { +"v" } + } + + // when + val rendered = events.renderYaml() + + // then + rendered sameAs "\"a\uFFFD\": v\n" + } + @Test fun `should round-trip keys that need quoting`() = runTest { // given From c0402651db96e2e6957bbdaa050a0c5d90de7090 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Sat, 26 Sep 2026 14:16:31 +0200 Subject: [PATCH 09/10] Quote nested one-letter boolean keys and route timestamps and base 60 first - quote a `y` / `n` key below the top level, since go-yaml v2 decodes a nested mapping with interface{} keys and reads a plain `y:` as `true` - type a timestamp-shaped or `:`-holding scalar through its own patterns only, so the number patterns never run on it - merge the Psych and PyYAML base-60 int and float into one pattern each, with Psych's leading `0` limited to two segments - derive PyYAML's timestamp shape from the shared match instead of a second pattern - pin the libyaml comment-after-closed-token rule as a DIVERGENCE test - make the linear-time test compare two input sizes instead of an absolute bound, so slower Native / JS regex engines don't trip it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 1 + .../src/commonMain/kotlin/YamlParser.kt | 7 +- .../src/commonMain/kotlin/YamlScalarType.kt | 101 ++++++++++++------ .../src/commonMain/kotlin/YamlWriter.kt | 29 ++--- .../src/commonTest/kotlin/YamlParserTest.kt | 61 ++++++++--- .../commonTest/kotlin/YamlRoundTripTest.kt | 34 ++++++ 6 files changed, 171 insertions(+), 62 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 70881cb..a6570f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -555,6 +555,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Consequence for the YAML writer: `YamlWriter.renderKey` quotes every key that is not identifier-shaped (`"a b"`, `"42"`, `"og:title"`), even where YAML would not require it, so whatever `simplifyHtml` emits first (a `<meta name>` with a space, say) still re-detects. It also quotes a key that is a reserved literal (`"yes":`) — a YAML 1.1 reader would take a bare `yes:` as a boolean key. Key typing is its own rule (`isTypedPlainKey`), not the value rule: Psych types `yEs:` / `nULL:` keys, but go-yaml v2 decodes a front matter key into a *string*, so its one-letter booleans `y` / `n` — typed as values — must stay plain as keys (`x: 1\ny: 2` was once rewritten to `"y": 2`). + That holds only at the **top level**: a nested mapping decodes with `interface{}` keys, so go-yaml v2 reads a nested `y:` as the key `true` — the writer quotes `y` / `n` below the top level (`YamlWriter.Node.topLevel`). The writer's plain-key rule (`isIdentifierKey`) and the detector share their character rules in `YamlKeyLine.kt`, and `YamlWriterTest` "should write every key as a line the front matter detector accepts" pins the two — widen one without the other and every document whose *first* entry has such a key silently loses its front matter on re-parse. **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index 939baa7..ff6d46c 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -547,8 +547,11 @@ private fun isSequenceEntry(content: String): Boolean = // True when only whitespace or a `#` comment follows index [from], the end // of a quoted scalar or a flow collection. Not [isCommentStartAt]: that is -// the rule inside a plain scalar, while after a closed token every reader -// (libyaml, PyYAML) starts a comment at any `#`, even with no space (`"x"#b`). +// the rule inside a plain scalar, while after a closed token the front +// matter readers (PyYAML, Psych, go-yaml v2 — all libyaml's scanner) start a +// comment at any `#`, even with no space (`"x"#b`). DIVERGENCE: YAML 1.2 +// §6.6 wants whitespace first, and a strict reader (npm `yaml`, +// snakeyaml-engine) refuses such a line; the writer never produces one. private fun isBlankOrComment(s: String, from: Int): Boolean { val i = skipSpaces(s, from) return i >= s.length || s[i] == '#' diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index e2f18d1..b428ce7 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -62,11 +62,12 @@ private val YAML_WORD_MAX_LENGTH = // on a prefix (`0x1F` matched `[0-9]+` as `0`, a full timestamp matched its // date-only branch) and the whole-input check then fails — JVM backtracks // across the branches and never showed it. -// No pattern repeats a group (`(?:_?[0-9])*`): the JVM engine matches each -// repetition of a group one stack frame deeper, so a long digit run -// overflows the stack. A digit run is a character class instead, and the -// rule the group expressed is checked by hand ([hasSeparatorsBeforeDigits], -// [hasUnderscoresBetweenDigits], [isBase60Tail]). +// No pattern repeats a group (`(?:_?[0-9])*`), not even a bounded number of +// times: the JVM engine matches each repetition of a group one stack frame +// deeper, so a long digit run overflows the stack. A digit run is a +// character class instead, and the rule the group expressed is checked by +// hand ([hasSeparatorsBeforeDigits], [hasUnderscoresBetweenDigits], +// [isBase60Tail]). private fun anchored(pattern: String) = Regex("^(?:$pattern)$") // PyYAML's resolver, less a base prefix with no digit (`0x_`), which it @@ -83,26 +84,27 @@ private val PYYAML_FLOAT = anchored( """[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?""" ) -// PyYAML's base-60 int and float, `(?::[0-5]?[0-9])+` written as a run the -// group is the tail of, which [isBase60Tail] then checks -private val PYYAML_BASE60_INT = anchored("[-+]?[1-9][0-9_]*(:[0-9:]*)") +// The base-60 int and float: PyYAML's `[1-9][0-9_]*(?::[0-5]?[0-9])+` and +// Psych's `[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}` (so a leading `0` only with at +// most two segments), and for the float `[0-9][0-9_]*(?::[0-5]?[0-9])+` +// with a fraction — Psych's float takes the same with at most two segments, +// a subset. The group is written as a run it is the tail of, which +// [isBase60Tail] then checks; the first group is the leading digit. +private val BASE60_INT = anchored("[-+]?([0-9])[0-9_]*(:[0-9:]*)") -private val PYYAML_BASE60_FLOAT = anchored("""[-+]?[0-9][0-9_]*(:[0-9:]*)\.[0-9_]*""") +private val BASE60_FLOAT = anchored("""[-+]?[0-9][0-9_]*(:[0-9:]*)\.[0-9_]*""") // Psych's scalar scanner, with its default (legacy) integers that allow `,` private val PSYCH_INT = anchored( "[-+]?0b[_,]*[01][01_,]*|[-+]?0[0-7_,]+|[-+]?0" + - "|[-+]?0x[_,]*[0-9a-fA-F][0-9a-fA-F_,]*|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}" + "|[-+]?0x[_,]*[0-9a-fA-F][0-9a-fA-F_,]*" ) // … and its decimal, `[1-9](?:[0-9]|[,_][0-9])*`: a separator only before a // digit, which [hasSeparatorsBeforeDigits] checks private val PSYCH_DECIMAL_INT = anchored("[-+]?[1-9][0-9,_]*") -private val PSYCH_FLOAT = anchored( - """[-+]?(?:[0-9][0-9_,]*\.[0-9]*|\.[0-9]+)(?:[eE][-+][0-9]+)?""" + - """|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9]){1,2}\.[0-9_]*""" -) +private val PSYCH_FLOAT = anchored("""[-+]?(?:[0-9][0-9_,]*\.[0-9]*|\.[0-9]+)(?:[eE][-+][0-9]+)?""") // go-yaml v2 deletes every `_` from a text starting with a sign or a digit, // then tries Go's `ParseInt` with base 0 (which reads a `0x` / `0o` / `0b` @@ -123,22 +125,14 @@ private val GO_YAML_DOT_FLOAT = anchored("""\.[0-9][0-9_]*(?:[eE][-+]?[0-9][0-9_ // The union of the Psych and PyYAML timestamp shapes (go-yaml v2 decodes a // timestamp into a string); the groups are the date, the time and the -// offset digits, so [isTimestampInRange] can check their values. +// offset digits, so [isTimestampInRange] can check their values and +// [isPyYamlTimestamp] PyYAML's narrower shape. private val YAML_TIMESTAMP = anchored( """(-?[0-9]{4})-([0-9]{1,2})-([0-9]{1,2})""" + """(?:(?:[Tt]|[ \t]+)([0-9]{1,2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]*)?""" + """(?:[ \t]*(?:Z|[-+]([0-9]{1,2}(?::?[0-9]{2})?)))?)?""" ) -// PyYAML's own timestamp shape: two-digit month and day for a date alone, -// a colon in an offset with minutes. It constructs whatever matches, and -// refuses the document when a value is out of range. -private val PYYAML_TIMESTAMP = anchored( - """[0-9]{4}-[0-9]{2}-[0-9]{2}""" + - """|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?""" + - """(?:[ \t]*(?:Z|[-+][0-9]{1,2}(?::[0-9]{2})?))?""" -) - // Every character a numeric or timestamp shape can hold, so a string with // any other one is a string without running a pattern: digits, signs, // separators, base prefixes and hex digits, exponents, and a timestamp's @@ -161,11 +155,22 @@ internal fun yamlScalarType(text: String): String? { val first = text[0] if (first != '-' && first != '+' && first != '.' && first !in '0'..'9') return null if (!text.all { it.canBeNumeric() }) return null + // a date is the only shape with a `-` after four digits, and a time + // follows one, so a timestamp never runs the number patterns + if (text.isTimestampShaped()) { + val timestamp = YAML_TIMESTAMP.matchEntire(text) + return if (timestamp != null && isTimestampInRange(timestamp.groupValues)) "timestamp" else null + } + // … and base 60 is the only number with a `:` + if (':' in text) { + val int = BASE60_INT.matchEntire(text) + if (int != null && isBase60Int(int.groupValues)) return "int" + val float = BASE60_FLOAT.matchEntire(text) + return if (float != null && isBase60Tail(float.groupValues[1])) "float" else null + } if (PYYAML_INT.matches(text) || PSYCH_INT.matches(text)) return "int" if (PSYCH_DECIMAL_INT.matches(text) && text.hasSeparatorsBeforeDigits()) return "int" - if (PYYAML_BASE60_INT.matchEntire(text)?.let { isBase60Tail(it.groupValues[1]) } == true) return "int" if (PYYAML_FLOAT.matches(text) || PSYCH_FLOAT.matches(text)) return "float" - if (PYYAML_BASE60_FLOAT.matchEntire(text)?.let { isBase60Tail(it.groupValues[1]) } == true) return "float" if (first == '.') { if (GO_YAML_DOT_FLOAT.matches(text) && text.hasUnderscoresBetweenDigits()) return "float" } else { @@ -173,11 +178,17 @@ internal fun yamlScalarType(text: String): String? { if (GO_YAML_INT.matches(plain)) return "int" if (GO_YAML_FLOAT.matches(plain)) return "float" } - val timestamp = YAML_TIMESTAMP.matchEntire(text) - if (timestamp != null && isTimestampInRange(timestamp.groupValues)) return "timestamp" return null } +// Whether the text starts like a [YAML_TIMESTAMP]: four digits, after an +// optional `-`, then a `-`. +private fun String.isTimestampShaped(): Boolean { + val start = if (startsWith('-')) 1 else 0 + return length > start + 4 && this[start + 4] == '-' && + (start until start + 4).all { this[it] in '0'..'9' } +} + // Whether every `,` / `_` is followed by a digit. private fun String.hasSeparatorsBeforeDigits(): Boolean = indices.all { i -> this[i] != ',' && this[i] != '_' || i + 1 < length && this[i + 1] in '0'..'9' @@ -194,6 +205,23 @@ private fun isBase60Tail(tail: String): Boolean = segment.length == 1 || segment.length == 2 && segment[0] in '0'..'5' } +// Whether a [BASE60_INT] match is PyYAML's (no leading `0`, any number of +// segments) or Psych's (at most two). +private fun isBase60Int(groups: List<String>): Boolean { + val tail = groups[2] + return isBase60Tail(tail) && (groups[1] != "0" || tail.count { it == ':' } <= 2) +} + +// Whether a [YAML_TIMESTAMP] match is also PyYAML's own shape: no sign on the +// year, a two-digit month and day for a date alone, and a colon in an +// offset with minutes. +private fun isPyYamlTimestamp(groups: List<String>): Boolean { + if (groups[1].startsWith('-')) return false + if (groups[4].isEmpty()) return groups[2].length == 2 && groups[3].length == 2 + val offset = groups[7] + return offset.length <= 2 || ':' in offset +} + // Psych checks a date against the calendar, but normalises a date and time // (`2023-02-31T10:00:00` is March 3rd), refusing only the values its `Time` // cannot take — hour 24 only as `24:00:00`, an offset under a day; a @@ -238,7 +266,8 @@ private fun daysInMonth(year: Int, month: Int): Int = when (month) { // document fails: PyYAML's value / merge tags (`=`, `<<`), which its // SafeLoader cannot construct; a base prefix with no digit (`0x_`, PyYAML // and Psych); an exponent with no mantissa digit (`.e+4`, Psych); and, in -// [isUnsafePlainScalar], a [PYYAML_TIMESTAMP] shape out of range, which +// [isUnsafePlainScalar], a PyYAML timestamp shape ([isPyYamlTimestamp]) out +// of range, which // PyYAML refuses (`2024-13-45`, `2024-01-01 24:30:00`) — the parser does // not type it either, or it would not get there. There is no type to report // for them, so the parser keeps them strings — DIVERGENCE: the writer still @@ -255,18 +284,22 @@ internal fun isUnsafePlainScalar(text: String): Boolean { if (first != '=' && first != '<' && first != '-' && first != '+' && first != '.' && first !in '0'..'9') { return false } - return YAML_REJECTED.matches(text) || PYYAML_TIMESTAMP.matches(text) + if (YAML_REJECTED.matches(text)) return true + val timestamp = YAML_TIMESTAMP.matchEntire(text) + return timestamp != null && isPyYamlTimestamp(timestamp.groupValues) } // Whether a plain identifier-shaped mapping key would be read as something // other than a string: Psych types the boolean and null words in any letter -// case (PyYAML only some cases of them), while go-yaml v2 decodes a front -// matter key into a string, so its one-letter booleans `y` / `n` stay keys. +// case (PyYAML only some cases of them), and go-yaml v2 its one-letter +// booleans `y` / `n` too — except in a [topLevel] key, which it decodes into +// a front matter's `map[string]interface{}`, while a nested mapping gets +// `interface{}` keys (`params:\n y: 1` has the key `true`). // The parser never types a key, so this is quoting the writer alone does. -internal fun isTypedPlainKey(key: String): Boolean { +internal fun isTypedPlainKey(key: String, topLevel: Boolean): Boolean { if (key.length > YAML_WORD_MAX_LENGTH) return false val word = key.lowercase() - return word in YAML_NULLS || word in YAML_BOOLS && word != "y" && word != "n" + return word in YAML_NULLS || word in YAML_BOOLS && (!topLevel || word != "y" && word != "n") } // The two indicators that can end a plain scalar in block context, shared by diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt index 328c372..e9fada5 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlWriter.kt @@ -48,7 +48,7 @@ import com.xemantic.markanywhere.SemanticEvent * - a key is written plain only when identifier-shaped (letters, digits, * `_`, `-`, `.`, starting with a letter or `_`) and not a key a reader * would type — Psych takes a bare `yes:` or `nULL:` as a boolean / null - * key — and double-quoted otherwise, even where YAML would not require it; + * key, go-yaml v2 a `y:` below the top level — and double-quoted otherwise, even where YAML would not require it; * - a `text` child of a container (the parser's verbatim fallback for a * line outside its YAML subset) is written as-is, on its own line(s). * @@ -72,7 +72,9 @@ public class YamlWriter( // the column this node's own line starts at val indent: Int, // the column this node's children start at - val childIndent: Int + val childIndent: Int, + // no entry or item encloses it — a key of the root mapping + val topLevel: Boolean ) { var state: State = UNDECIDED val scalar = StringBuilder() @@ -80,7 +82,7 @@ public class YamlWriter( } private val stack = ArrayDeque<Node>().apply { - addLast(Node(OTHER, key = null, type = null, indent = 0, childIndent = 0)) + addLast(Node(OTHER, key = null, type = null, indent = 0, childIndent = 0, topLevel = true)) } // True right after `-` was written for an item whose first child @@ -111,7 +113,8 @@ public class YamlWriter( } val indent = parent.childIndent val childIndent = if (kind == OTHER) indent else indent + 2 - stack.addLast(Node(kind, event.attributes["key"], event.attributes["type"], indent, childIndent)) + val topLevel = parent.topLevel && !parent.isValue + stack.addLast(Node(kind, event.attributes["key"], event.attributes["type"], indent, childIndent, topLevel)) } private fun text(text: String) { @@ -153,14 +156,14 @@ public class YamlWriter( atLineStart = false afterDash = true } else { - out(linePrefix(node) + renderKey(node.key) + ":\n") + out(linePrefix(node) + renderKey(node) + ":\n") atLineStart = true } if (pendingVerbatim != null) writeVerbatim(pendingVerbatim) } private fun writeLine(node: Node, value: String) { - val head = if (node.kind == ITEM) "-" else renderKey(node.key) + ":" + val head = if (node.kind == ITEM) "-" else renderKey(node) + ":" out(linePrefix(node) + head + (if (value.isEmpty()) "" else " $value") + "\n") atLineStart = true } @@ -231,13 +234,13 @@ public class YamlWriter( } // A key is written plain only when it is identifier-shaped and would not - // be typed (`yes`, `nULL`): the Markdown parser's front matter detection requires + // be typed (`yes`, `nULL`, a nested `y`): the Markdown parser's front matter detection requires // the first line to pass `isYamlKeyLine` (such a key, or a quoted one), // so quoting everything else keeps whatever entry comes first // re-detectable. The character rules are shared with that check. - private fun renderKey(key: String?): String { - val k = key ?: "" - return if (isIdentifierKey(k) && !isTypedPlainKey(k)) k else quoted(k) + private fun renderKey(node: Node): String { + val k = node.key ?: "" + return if (isIdentifierKey(k) && !isTypedPlainKey(k, node.topLevel)) k else quoted(k) } } @@ -294,9 +297,9 @@ private fun quoted(s: String): String = buildString { '\r' -> +"\\r" '\t' -> +"\\t" else -> when { - c.isSurrogate() && s.isYamlUnprintableAt(i) -> +'\uFFFD' - s.isYamlUnprintableAt(i) -> +("\\u" + c.code.toString(16).padStart(4, '0')) - else -> +c + !s.isYamlUnprintableAt(i) -> +c + c.isSurrogate() -> +'\uFFFD' + else -> +("\\u" + c.code.toString(16).padStart(4, '0')) } } +'"' diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 74feb3a..4928eae 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -25,7 +25,8 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.test.runTest import com.xemantic.kotlin.test.assert import kotlin.test.Test -import kotlin.time.Duration.Companion.seconds +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds import kotlin.time.TimeSource import kotlin.time.measureTime @@ -502,21 +503,30 @@ class YamlParserTest { // given — a base prefix, digits and one character outside the base: // a pattern that lets the digits match two ways backtracks // quadratically over them, and every plain scalar is typed - val values = listOf("0x", "0b", "+0x", "-0b").map { prefix -> - prefix + "1".repeat(50_000) + "g" + fun nearMisses(length: Int) = listOf("0x", "0b", "+0x", "-0b").map { prefix -> + prefix + "1".repeat(length) + "g" } - - // when - val elapsed = TimeSource.Monotonic.measureTime { - for (value in values) { - assert(yamlScalarType(value) == null) - assert(!isUnsafePlainScalar(value)) + // the best of a few runs, so a pause of the runtime is not measured + fun timeToResolve(values: List<String>): Duration = List(3) { + TimeSource.Monotonic.measureTime { + for (value in values) { + assert(yamlScalarType(value) == null) + assert(!isUnsafePlainScalar(value)) + } } - } + }.min() + val short = nearMisses(25_000) + val long = nearMisses(100_000) - // then — linear takes milliseconds, quadratic minutes; the bound - // is far from both so a slow runner cannot trip it - assert(elapsed < 20.seconds) + // when + val shortTime = timeToResolve(short) + val longTime = timeToResolve(long) + + // then — four times the input takes about four times as long when + // linear, sixteen times when quadratic; the allowance absorbs the + // timer's resolution when both are only milliseconds, and no absolute + // bound is set, since a Native or JS regex engine is much slower + assert(longTime < shortTime * 8 + 50.milliseconds) } @Test @@ -943,6 +953,31 @@ class YamlParserTest { } } + @Test + fun `DIVERGENCE - should read a hash right after a closed token as a comment`() = runTest { + // given — YAML 1.2 §6.6 wants whitespace before a comment, but the + // front matter readers (PyYAML, Psych, go-yaml v2 — all libyaml's + // scanner) start one at any `#` after a quoted scalar or a flow + // collection, and read these lines as the values below; a strict + // reader (npm `yaml`, snakeyaml-engine) refuses them instead + val textFlow = "a: \"x\"#b\nc: 'y'#d\ntags: [e]#f\ngeo: {g: 1}#h\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a") { +"x" } + "entry"("key" to "c") { +"y" } + "entry"("key" to "tags") { + "item" { +"e" } + } + "entry"("key" to "geo") { + "entry"("key" to "g", "type" to "int") { +"1" } + } + } + } + // --- verbatim fallback (DIVERGENCE) ----------------------------------- @Test diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index 9c963fb..cc4c158 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -186,6 +186,40 @@ class YamlRoundTripTest { rendered sameAs source } + @Test + fun `should quote a one-letter boolean key below the top level`() = runTest { + // given — go-yaml v2 decodes only the top-level keys into strings; a + // nested mapping, in an entry or in an item, is decoded with + // interface{} keys, so a plain `y:` there becomes the key `true` + val events = semanticEvents { + "entry"("key" to "params") { + "entry"("key" to "y") { +"1" } + "entry"("key" to "N") { +"2" } + } + "entry"("key" to "list") { + "item" { + "entry"("key" to "n") { +"3" } + } + } + "entry"("key" to "y") { +"4" } + } + + // when + val rendered = events.renderYaml() + val reparsed = flowOf(rendered).parseYaml() + + // then + rendered sameAs """ + params: + "y": "1" + "N": "2" + list: + - "n": "3" + y: "4" + """.trimIndent() + "\n" + reparsed.mergeAdjacentText() sameAs events + } + @Test fun `should keep YAML 1_1 typed front matter values as written`() = runTest { // given — Jekyll's documented date format, a short date, go-yaml v2 From 067bceb6771d1943774a12157d3ec1d934970e36 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Sat, 26 Sep 2026 15:15:52 +0200 Subject: [PATCH 10/10] Keep lines the YAML readers disagree on verbatim and bound go-yaml numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - keep a flow scalar whose `:` precedes `,` / `[` / `]` / `{` / `}` verbatim: PyYAML reads a mapping, Psych refuses the line and go-yaml v2 reads the colon as content, so no single reading is right - keep a block scalar header followed by `#` with no space verbatim, as PyYAML refuses it - type a number go-yaml v2 alone reads only within its int64 / uint64 / float64 range (`1e999`, `0X` and 17 hex digits stay strings) - match the timestamp pattern once in `isUnsafePlainScalar` - replace the wall-clock linear-time test with an input large enough that quadratic backtracking hangs - name every divergence test `DIVERGENCE - …` - condense the YAML entries in CLAUDE.md and state the lesson / why rule Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 16 +- .../kotlin/FrontMatterRoundTripTest.kt | 2 +- .../src/commonTest/kotlin/FrontMatterTest.kt | 2 +- .../kotlin/HtmlBlockInListItemTest.kt | 6 +- .../src/commonTest/kotlin/HtmlParsingTest.kt | 8 +- .../commonTest/kotlin/gfm/Gfm_06_11_Test.kt | 2 +- markanywhere-yaml/README.md | 4 +- .../src/commonMain/kotlin/YamlParser.kt | 26 ++- .../src/commonMain/kotlin/YamlScalarType.kt | 72 ++++++-- .../src/commonTest/kotlin/YamlParserTest.kt | 163 ++++++++++++++---- .../commonTest/kotlin/YamlRoundTripTest.kt | 25 ++- 11 files changed, 248 insertions(+), 78 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a6570f5..90e1790 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,8 @@ Both developers and AI agents are expected to add entries as they encounter surp - **Add an entry** when you encounter something unexpected: a build quirk, a non-obvious constraint, a dependency gotcha, or any behavior that would surprise the next agent or developer. - **Add an entry** when a developer flags an anti-pattern produced by AI — describe the anti-pattern and the preferred alternative. - **Do not** add codebase overviews, directory listings, or anything discoverable by reading the source. -- Keep entries concise: one line per lesson, grouped under a heading if a theme emerges. +- Keep entries concise: the lesson and its non-obvious *why*, grouped under a heading if a theme emerges. + Name a symbol or test as a pointer, but don't restate what its source comment already says. ## Conventions @@ -412,7 +413,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. - A `commonMain` `Regex` applied to untrusted input must not **repeat a group** (`(?:_?[0-9])*`, `(?::[0-9])+`): `java.util.regex` matches each repetition of a group one stack frame deeper, so a few thousand repetitions throw `StackOverflowError` on the JVM (an `Error`, which nothing catches — it takes the whole pipeline down). - Repeat a character class instead (`[0-9_]*`) and check what the group expressed by hand — see `hasSeparatorsBeforeDigits` / `isBase60Tail` in `YamlScalarType.kt`, pinned by `YamlParserTest` "long run of digits". + Repeat a character class instead (`[0-9_]*`) and check what the group expressed by hand (`YamlScalarType.kt`). - Facts of the HTML standard needed by more than one of `parse` / `render` / `html` — HTML whitespace (`isHtmlWhitespace` / `isHtmlBlank`), void and raw text elements, `Mark.classList` — live in the leaf module `markanywhere-html-spec` (depends only on `api`), never in `markanywhere-api` (which describes only the event model) and never as private copies. Only spec facts go there, not project policy (e.g. which tags render as blocks in Markdown); the parser's `isFlankWhitespace` is deliberately a different set (CommonMark "Unicode whitespace", which includes NBSP). - `MutableMap.putIfAbsent` is **JVM-only** — not in the common stdlib, so it can't be used in `commonMain`. @@ -546,16 +547,14 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' `entry`/`item` are Markdown-native **only inside** a `frontmatter` (the Markdown renderer creates a `YamlWriter` on the mark, routes every following event into `YamlWriter.collect`, and treats the first `unmark` arriving at `depth == 0` as the wrapper's own close; outside one they fall through to the raw-tag path), so they are deliberately NOT in `MARKDOWN_NATIVE_MARK_NAMES`. Any other mark is **transparent** to the writer (children lay out as if it were absent), which is what lets `renderYaml()` accept a `frontmatter`-wrapped stream unchanged. The writer's quoting rules reuse the parser's `yamlScalarType` (`YamlScalarType.kt`, one copy of the reserved literals / int / float / timestamp rules) and are pinned against it by `YamlRoundTripTest` in `markanywhere-yaml`; `FrontMatterRoundTripTest` in `markanywhere-parse` (whose tests depend on render) covers only the fences and the wrapper — extend the yaml one when you touch either side of the codec. - A shape some front matter reader (Psych/Jekyll, PyYAML, go-yaml v2) types must be typed by the **parser**, not merely quoted by the writer — writer-only quoting rewrites the source on round-trip (Jekyll's `date: 2016-01-01 12:00:00 -0500` came back as a quoted string). - The converse holds too: a shape **no** reader types must not be typed, or `type=int` stops meaning a number (`1,` once came back `type=int`). - Check a change against the real readers (PyYAML `safe_load`, Ruby `Psych.unsafe_load`, `gopkg.in/yaml.v2` into `map[string]interface{}`) over generated strings, not by reading their docs — go-yaml v2 decodes a timestamp into a *string*, Psych normalises an out-of-range date *and time* but rejects an out-of-range date alone. - The one sanctioned writer-only quoting is a shape a reader **refuses to load** (`=`, `<<`, `0x_`, `.e+4`, `2024-13-45`): there is no type to report, so the parser keeps it a string and the rewrite is pinned as `DIVERGENCE` in `YamlRoundTripTest`. + Typing follows the front matter readers (Psych/Jekyll, PyYAML, go-yaml v2): a shape some reader types must be typed by the **parser** (writer-only quoting rewrote Jekyll's `date: 2016-01-01 12:00:00 -0500` into a quoted string), a shape none types must not be (or `type=int` stops meaning a number), and a shape they read **differently** from one another (`[a:, b]`: a mapping, an error, the string `"a:"`) stays a verbatim line — picking one reading rewrites the document for the others. + Verify a change by running the readers themselves (PyYAML `safe_load`, Ruby `Psych.unsafe_load`, `gopkg.in/yaml.v2` into `map[string]interface{}`) over generated strings, never from their docs or the spec — reasoning from the spec got each of the three rules above wrong at least once. + The one writer-only quoting is a shape a reader **refuses to load** (`=`, `0x_`, `2024-13-45`), pinned as `DIVERGENCE` in `YamlRoundTripTest`. `renderYaml()` **keeps the trailing newline** (unlike `renderMarkdown()` / `renderHtml()`, which suppress one): a `|+` block scalar at the end of the document needs it, and dropping it broke the round-trip for `z\n\n`. - Front matter **detection** requires line 2 to be a mapping key (identifier or quoted, then `:` + whitespace/EOL — `isYamlKeyLine`, a public function of `markanywhere-yaml`, NOT a private copy in `FrontMatterFilter`); a `# comment` cannot be line 2 (`---` + `# Heading` is a thematic break + heading), and neither can `http://…` or a `- item`. Consequence for the YAML writer: `YamlWriter.renderKey` quotes every key that is not identifier-shaped (`"a b"`, `"42"`, `"og:title"`), even where YAML would not require it, so whatever `simplifyHtml` emits first (a `<meta name>` with a space, say) still re-detects. It also quotes a key that is a reserved literal (`"yes":`) — a YAML 1.1 reader would take a bare `yes:` as a boolean key. - Key typing is its own rule (`isTypedPlainKey`), not the value rule: Psych types `yEs:` / `nULL:` keys, but go-yaml v2 decodes a front matter key into a *string*, so its one-letter booleans `y` / `n` — typed as values — must stay plain as keys (`x: 1\ny: 2` was once rewritten to `"y": 2`). - That holds only at the **top level**: a nested mapping decodes with `interface{}` keys, so go-yaml v2 reads a nested `y:` as the key `true` — the writer quotes `y` / `n` below the top level (`YamlWriter.Node.topLevel`). + Key typing is not value typing: go-yaml v2 decodes a **top-level** front matter key into a string but a nested one as `interface{}`, so `y:` must stay plain at the top level (it was once rewritten to `"y":`) and be quoted below it (`isTypedPlainKey`). The writer's plain-key rule (`isIdentifierKey`) and the detector share their character rules in `YamlKeyLine.kt`, and `YamlWriterTest` "should write every key as a line the front matter detector accepts" pins the two — widen one without the other and every document whose *first* entry has such a key silently loses its front matter on re-parse. **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. @@ -573,6 +572,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - For semantic event flow testing, use the overloaded `sameAs` infix on `Flow<SemanticEvent>` (defined in `markanywhere-test`) against a `semanticEvents { ... }` builder — this is event-stream comparison, unrelated to HTML. - For asserting rendered HTML output (e.g. in `markanywhere-render` tests), use `sameAsHtml` (not the generic string `sameAs`) — provides syntax highlighting in the IDE. - GFM example test naming convention: `example N - <description>` for spec-conformant tests, `example N - DIVERGENCE - <description>` when the parser intentionally diverges from GFM. + Any other test pinning an intentional divergence (from GFM, YAML or a reader's behaviour) starts with `DIVERGENCE - ` (`DIVERGENCE - should keep …`), never `should DIVERGENCE …` or a trailing `… DIVERGENCE`. Keep `DIVERGENCE` in the name even when updating expectations to match the divergent behavior. - When asserting on a boolean expression (counts, equality between two nullables, `contains`, etc.), use `assert(expr)` from `com.xemantic.kotlin.test.assert` — it is power-assert-instrumented in the convention plugin, so a failure prints the full expression with subvalues. Do NOT fall back to `kotlin.test.assertEquals` / `assertTrue` / `assertNotNull` — they discard that information. diff --git a/markanywhere-parse/src/commonTest/kotlin/FrontMatterRoundTripTest.kt b/markanywhere-parse/src/commonTest/kotlin/FrontMatterRoundTripTest.kt index 7341179..ffa85cf 100644 --- a/markanywhere-parse/src/commonTest/kotlin/FrontMatterRoundTripTest.kt +++ b/markanywhere-parse/src/commonTest/kotlin/FrontMatterRoundTripTest.kt @@ -128,7 +128,7 @@ class FrontMatterRoundTripTest { } @Test - fun `should DIVERGENCE not re-detect a root sequence front matter`() = runTest { + fun `DIVERGENCE - should not re-detect a root sequence front matter`() = runTest { // given — YAML allows a root sequence, but front matter detection // requires a mapping key on line 2 (`---` + `- a` is a thematic // break followed by a list in Markdown, and no front matter consumer diff --git a/markanywhere-parse/src/commonTest/kotlin/FrontMatterTest.kt b/markanywhere-parse/src/commonTest/kotlin/FrontMatterTest.kt index 99c5f9a..ef49706 100644 --- a/markanywhere-parse/src/commonTest/kotlin/FrontMatterTest.kt +++ b/markanywhere-parse/src/commonTest/kotlin/FrontMatterTest.kt @@ -168,7 +168,7 @@ class FrontMatterTest { } @Test - fun `should DIVERGENCE auto-close unterminated front matter at EOF`() = runTest { + fun `DIVERGENCE - should auto-close unterminated front matter at EOF`() = runTest { // given — opener but no closer. DIVERGENCE: spec-correct behavior would // require buffering the whole document to detect the missing closer // and fall back to a thematic break + paragraphs. The streaming parser diff --git a/markanywhere-parse/src/commonTest/kotlin/HtmlBlockInListItemTest.kt b/markanywhere-parse/src/commonTest/kotlin/HtmlBlockInListItemTest.kt index 67308ac..b3197d7 100644 --- a/markanywhere-parse/src/commonTest/kotlin/HtmlBlockInListItemTest.kt +++ b/markanywhere-parse/src/commonTest/kotlin/HtmlBlockInListItemTest.kt @@ -98,7 +98,7 @@ class HtmlBlockInListItemTest { } @Test - fun `type 1 pre block inside list item DIVERGENCE`() = runTest { + fun `DIVERGENCE - type 1 pre block inside list item`() = runTest { // given val textFlow = /* language=markdown */ """ - intro @@ -298,7 +298,7 @@ class HtmlBlockInListItemTest { } @Test - fun `blank line inside list-internal html block stays in raw mode DIVERGENCE`() = runTest { + fun `DIVERGENCE - blank line inside list-internal html block stays in raw mode`() = runTest { // given: a blank line inside a `<div>` block. At top level this would // transition the block to sub-parse mode and the `- nested` line would // open a Markdown list. Inside a list item we stay in raw-text mode @@ -444,7 +444,7 @@ class HtmlBlockInListItemTest { } @Test - fun `type 1 pre block with multi-line opener inside list item DIVERGENCE`() = runTest { + fun `DIVERGENCE - type 1 pre block with multi-line opener inside list item`() = runTest { // given: `<pre` on the marker line, attribute on the next line, `>` // closes the opener. Exercises the buffered-opener path in the type-1 // branch of `streamListHtmlBlockLine` — `firstLineBuffer` accumulates diff --git a/markanywhere-parse/src/commonTest/kotlin/HtmlParsingTest.kt b/markanywhere-parse/src/commonTest/kotlin/HtmlParsingTest.kt index 2550d3d..f756625 100644 --- a/markanywhere-parse/src/commonTest/kotlin/HtmlParsingTest.kt +++ b/markanywhere-parse/src/commonTest/kotlin/HtmlParsingTest.kt @@ -401,7 +401,7 @@ class HtmlParsingTest { * over spec-faithful escaping. */ @Test - fun `DIVERGENCE inline disallowed script tag and its body are dropped`() = runTest { + fun `DIVERGENCE - inline disallowed script tag and its body are dropped`() = runTest { // given val src = "before<script>alert(1)</script>after\n" @@ -419,7 +419,7 @@ class HtmlParsingTest { * including HTML-like substrings, is dropped. */ @Test - fun `DIVERGENCE inline disallowed style tag and its body are dropped`() = runTest { + fun `DIVERGENCE - inline disallowed style tag and its body are dropped`() = runTest { // given val src = "x<style>a::before{content:\"<x>\"}</style>y\n" @@ -437,7 +437,7 @@ class HtmlParsingTest { * the final `>` (`</script >`), mirroring the block-1 close rule. */ @Test - fun `DIVERGENCE inline disallowed close tag tolerates trailing whitespace`() = runTest { + fun `DIVERGENCE - inline disallowed close tag tolerates trailing whitespace`() = runTest { // given val src = "a<script>js</script\t >b\n" @@ -539,7 +539,7 @@ class HtmlParsingTest { * always close their `<script>`/`<style>`, so this edge is benign in practice. */ @Test - fun `DIVERGENCE unclosed inline disallowed opener drops only the rest of its line`() = runTest { + fun `DIVERGENCE - unclosed inline disallowed opener drops only the rest of its line`() = runTest { // given val src = "keep<script>dropped to end of line\nnext line\n" diff --git a/markanywhere-parse/src/commonTest/kotlin/gfm/Gfm_06_11_Test.kt b/markanywhere-parse/src/commonTest/kotlin/gfm/Gfm_06_11_Test.kt index 54c40b7..1088118 100644 --- a/markanywhere-parse/src/commonTest/kotlin/gfm/Gfm_06_11_Test.kt +++ b/markanywhere-parse/src/commonTest/kotlin/gfm/Gfm_06_11_Test.kt @@ -36,7 +36,7 @@ import kotlin.test.Test class Gfm_06_11_Test { @Test - fun `example 657 - DIVERGENCE disallowed inline tags are dropped not escaped`() = runTest { + fun `example 657 - DIVERGENCE - disallowed inline tags are dropped not escaped`() = runTest { // given val textFlow = buildString { +"<strong> <title> <style> <em>\n" diff --git a/markanywhere-yaml/README.md b/markanywhere-yaml/README.md index 21b6e63..eb61603 100644 --- a/markanywhere-yaml/README.md +++ b/markanywhere-yaml/README.md @@ -41,7 +41,7 @@ The vocabulary is deliberately small and the key lives in an attribute, so a key - The kind of value follows from the children: text only is a scalar, nested `entry` marks are a mapping, nested `item` marks are a sequence. - `type` is present only on a non-string scalar: `bool`, `int`, `float`, `null`, `timestamp`. A plain scalar is typed exactly when some reader front matter is written for reads it as something other than a string — the YAML 1.2 core schema, plus the YAML 1.1 shapes Psych (Jekyll), PyYAML and go-yaml v2 (Hugo) still type — so a Jekyll / Hugo document round-trips as written. - Beyond the core schema that means booleans and nulls in any letter case and `y` / `n`, numbers with `_` / `,` separators or a binary / octal / upper-case base prefix (`1_000`, `1,000`, `0b101`, `0X1F`), base-60 numbers (`12:30` is an `int`), and timestamps with a one-digit month or day or a colonless offset (`2016-01-01 12:00:00 -0500`); a date only when it is in the calendar. + Beyond the core schema that means booleans and nulls in any letter case and `y` / `n`, numbers with `_` / `,` separators or a binary / octal / upper-case base prefix (`1_000`, `1,000`, `0b101`, `0X1F`), base-60 numbers (`12:30` is an `int`), and timestamps with a one-digit month or day or a colonless offset (`2016-01-01 12:00:00 -0500`); a date only when it is in the calendar, and a number only go-yaml v2 reads only within its 64-bit range (`1e999` stays a string). So `type="int"` / `type="float"` says a reader reads a number, not that the text is a Kotlin literal: reading the value yourself takes that reader's rules (drop the separators, resolve the prefix or the base 60). No text and no type is an empty string; a bare `key:` is `type="null"`; an empty `[]` / `{}` is `type="seq"` / `type="map"` with no children. - All marks are untagged. @@ -53,7 +53,7 @@ Block mappings and sequences (a sequence may sit at its key's own indentation), Streaming: a `key: value` line commits on its newline. A bare `key:` (or `-`) is held for one line to decide between a nested block and a null value; a block scalar is buffered until it closes — never past the enclosing construct. -DIVERGENCE (never throws, never loses content): a line outside that subset — a complex `? key`, a directive, a document marker, a multi-line flow collection or quoted scalar, a plain scalar's continuation line, a `- item` inside a mapping — is emitted **verbatim** (with its `\n`) as a text child of the container it sits in, and the writer writes it back as-is. +DIVERGENCE (never throws, never loses content): a line outside that subset — a complex `? key`, a directive, a document marker, a multi-line flow collection or quoted scalar, a plain scalar's continuation line, a `- item` inside a mapping, a shape the front matter readers read differently from one another (`[draft:, x]`, `{a:[1]}`, `|-#note`) — is emitted **verbatim** (with its `\n`) as a text child of the container it sits in, and the writer writes it back as-is. Anchors, aliases and tags are not resolved: a plain scalar starting with `&`, `*` or `!` is just a string. ## Writing diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt index ff6d46c..001449b 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlParser.kt @@ -52,7 +52,10 @@ import kotlinx.coroutines.flow.FlowCollector * * DIVERGENCE (verbatim fallback): a line the subset does not understand — * complex keys, directives, multi-line flow collections or quoted scalars, - * a plain scalar's continuation line, a `- item` inside a mapping — is + * a plain scalar's continuation line, a `- item` inside a mapping, or a + * shape the front matter readers read differently from one another (a + * flow scalar's `:` before `,` / `[` / `]` / `{` / `}`, a block scalar + * header followed by `#` with no space) — is * emitted **verbatim** (with its `\n`) as a text child of the container it * sits in, so nothing is lost and [YamlWriter] can write it back as-is. * Anchors, aliases and tags are not resolved: a plain scalar starting with @@ -397,7 +400,10 @@ public class YamlParser( } i++ } - if (!isBlankOrComment(s, i)) return null + // PyYAML wants whitespace before a comment here, unlike after a + // quoted scalar or a flow collection (`|-#x` is refused) + val rest = skipSpaces(s, i) + if (rest < s.length && !s.isCommentStartAt(rest)) return null return Value.Block(folded, chomp, digit) } @@ -480,8 +486,8 @@ public class YamlParser( var i = start while (i < s.length) { val c = s[i] - if (s.isMappingColonAt(i) || c == ':' && (s[i + 1] == ',' || s[i + 1] == '}')) break - if (c == ',' || c == '}' || c == ']' || c == '[' || c == '{' || s.isCommentStartAt(i)) return null + if (s.isMappingColonAt(i)) break + if (c in FLOW_INDICATORS || s.isFlowColonAt(i) || s.isCommentStartAt(i)) return null i++ } val key = s.substring(start, i).trim() @@ -496,7 +502,7 @@ public class YamlParser( while (i < s.length) { val c = s[i] if (c == ',' || c == ']' || c == '}') break - if (s.isMappingColonAt(i) || s.isCommentStartAt(i)) return null + if (s.isMappingColonAt(i) || s.isFlowColonAt(i) || s.isCommentStartAt(i)) return null i++ } val text = s.substring(start, i).trim() @@ -530,6 +536,16 @@ private const val ITEM = "item" // supported subset (`-` is handled by the sequence check first). private const val KEY_FORBIDDEN_START = "[]{}&*!|>%@`," +private const val FLOW_INDICATORS = ",[]{}" + +// A `:` followed by a flow indicator inside a flow collection, where the +// readers disagree: PyYAML ends the plain scalar there (`[a:, b]` holds the +// mapping `{a: null}`), Psych refuses the line and go-yaml v2 reads the +// colon as content (`"a:"`). No reading is right for all three, so the +// line is left outside the subset and kept as written. +private fun String.isFlowColonAt(i: Int): Boolean = + this[i] == ':' && i + 1 < length && this[i + 1] in FLOW_INDICATORS + private fun leadingSpaces(s: String): Int { var i = 0 while (i < s.length && s[i] == ' ') i++ diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt index b428ce7..87327f8 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlScalarType.kt @@ -42,8 +42,8 @@ package com.xemantic.markanywhere.yaml // without a colon (`2016-01-01 12:00:00 -0500`, Jekyll's documented // format) — a date only when it is in the calendar, a date and time only // within the ranges Psych's `Time` accepts (it normalises `2023-02-31`). -// Not modelled: a float that overflows (`7e700`), which go-yaml v2 alone -// reads as a string. +// A shape go-yaml v2 alone reads as a number is typed only within the +// range it parses (`1e999`, `0X` and 17 hex digits are strings to it). // Typing such a value keeps a source document as written: the writer writes // a typed scalar bare, so only a *string* of one of these shapes is quoted. @@ -158,8 +158,7 @@ internal fun yamlScalarType(text: String): String? { // a date is the only shape with a `-` after four digits, and a time // follows one, so a timestamp never runs the number patterns if (text.isTimestampShaped()) { - val timestamp = YAML_TIMESTAMP.matchEntire(text) - return if (timestamp != null && isTimestampInRange(timestamp.groupValues)) "timestamp" else null + return if (timestampVerdict(text) == TYPED) "timestamp" else null } // … and base 60 is the only number with a `:` if (':' in text) { @@ -172,15 +171,46 @@ internal fun yamlScalarType(text: String): String? { if (PSYCH_DECIMAL_INT.matches(text) && text.hasSeparatorsBeforeDigits()) return "int" if (PYYAML_FLOAT.matches(text) || PSYCH_FLOAT.matches(text)) return "float" if (first == '.') { - if (GO_YAML_DOT_FLOAT.matches(text) && text.hasUnderscoresBetweenDigits()) return "float" + if (GO_YAML_DOT_FLOAT.matches(text) && text.hasUnderscoresBetweenDigits() && + text.replace("_", "").toDouble().isFinite() + ) { + return "float" + } } else { val plain = if ('_' in text) text.replace("_", "") else text - if (GO_YAML_INT.matches(plain)) return "int" - if (GO_YAML_FLOAT.matches(plain)) return "float" + if (GO_YAML_INT.matches(plain) && plain.fitsGoYamlInt()) return "int" + if (GO_YAML_FLOAT.matches(plain) && plain.toDouble().isFinite()) return "float" } return null } +// Whether go-yaml v2 parses this integer (`_` already deleted): Go's +// `ParseInt` takes a sign and up to 64 bits signed, `ParseUint` no sign and +// up to 64 bits unsigned; its own `0b` prefix hands the rest after it, +// sign included, to the same two. +private fun String.fitsGoYamlInt(): Boolean { + val binarySigned = startsWith("0b-") || startsWith("0b+") + val number = if (binarySigned) substring(2) else this + val sign = number[0].takeIf { it == '-' || it == '+' } + val unsigned = if (sign != null) number.substring(1) else number + val (radix, digits) = when { + binarySigned -> 2 to unsigned + unsigned.length > 1 && unsigned[0] == '0' -> when (unsigned[1]) { + 'x', 'X' -> 16 to unsigned.substring(2) + 'o', 'O' -> 8 to unsigned.substring(2) + 'b', 'B' -> 2 to unsigned.substring(2) + else -> 8 to unsigned.substring(1) + } + else -> 10 to unsigned + } + val magnitude = digits.toULongOrNull(radix) ?: return false + return when (sign) { + '-' -> magnitude <= Long.MIN_VALUE.toULong() + '+' -> magnitude <= Long.MAX_VALUE.toULong() + else -> true + } +} + // Whether the text starts like a [YAML_TIMESTAMP]: four digits, after an // optional `-`, then a `-`. private fun String.isTimestampShaped(): Boolean { @@ -212,6 +242,20 @@ private fun isBase60Int(groups: List<String>): Boolean { return isBase60Tail(tail) && (groups[1] != "0" || tail.count { it == ':' } <= 2) } +private enum class TimestampVerdict { NONE, TYPED, REFUSED } + +// For a timestamp-shaped text: [TimestampVerdict.TYPED] when some reader +// types it, [TimestampVerdict.REFUSED] when PyYAML resolves its shape and +// then refuses the value, [TimestampVerdict.NONE] for a plain string. +private fun timestampVerdict(text: String): TimestampVerdict { + val groups = YAML_TIMESTAMP.matchEntire(text)?.groupValues ?: return NONE + return when { + isTimestampInRange(groups) -> TYPED + isPyYamlTimestamp(groups) -> REFUSED + else -> NONE + } +} + // Whether a [YAML_TIMESTAMP] match is also PyYAML's own shape: no sign on the // year, a two-digit month and day for a date alone, and a colon in an // offset with minutes. @@ -277,16 +321,14 @@ private val YAML_REJECTED = anchored("""=|<<|[-+]?0[bx][_,]+|[-+]?\.[eE][-+][0-9 // Whether a plain scalar would not read back as this string in every // reader: the parser types it, or some reader refuses it. internal fun isUnsafePlainScalar(text: String): Boolean { + // typed or refused, decided by one match of the timestamp pattern + if (text.isTimestampShaped()) return timestampVerdict(text) != NONE if (yamlScalarType(text) != null) return true - // every refused shape starts with one of these, so the patterns only - // run on a string that can match them + // every other refused shape starts with one of these, so the pattern + // only runs on a string that can match it val first = text.firstOrNull() ?: return false - if (first != '=' && first != '<' && first != '-' && first != '+' && first != '.' && first !in '0'..'9') { - return false - } - if (YAML_REJECTED.matches(text)) return true - val timestamp = YAML_TIMESTAMP.matchEntire(text) - return timestamp != null && isPyYamlTimestamp(timestamp.groupValues) + return (first == '=' || first == '<' || first == '0' || first == '+' || first == '-' || first == '.') && + YAML_REJECTED.matches(text) } // Whether a plain identifier-shaped mapping key would be read as something diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt index 4928eae..ef3d56a 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlParserTest.kt @@ -25,10 +25,6 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.test.runTest import com.xemantic.kotlin.test.assert import kotlin.test.Test -import kotlin.time.Duration -import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.TimeSource -import kotlin.time.measureTime /** * Specifies the streaming YAML parser and its event representation. @@ -467,7 +463,64 @@ class YamlParserTest { } @Test - fun `should DIVERGENCE not type a plain scalar a reader refuses to load`() = runTest { + fun `should not type a number go-yaml v2 alone reads and then overflows`() = runTest { + // given — PyYAML and Psych need a `.` and a signed exponent for a + // float and take only lower-case `0x` / `0b` prefixes, so go-yaml v2 + // alone reads these shapes as numbers, and it falls back to a string + // when the value overflows its int64 / uint64 / float64 + val textFlow = """ + a: 1e999 + b: .5e999 + c: 0XFFFFFFFFFFFFFFFFF + d: 0o7777777777777777777777777 + e: +0XFFFFFFFFFFFFFFFF + f: -0X8000000000000001 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a") { +"1e999" } + "entry"("key" to "b") { +".5e999" } + "entry"("key" to "c") { +"0XFFFFFFFFFFFFFFFFF" } + "entry"("key" to "d") { +"0o7777777777777777777777777" } + "entry"("key" to "e") { +"+0XFFFFFFFFFFFFFFFF" } + "entry"("key" to "f") { +"-0X8000000000000001" } + } + } + + @Test + fun `should type a number go-yaml v2 alone reads within its range`() = runTest { + // given — the largest uint64 and the smallest int64 still fit, a + // decimal too large for an integer is read as a float, and an + // underflowing float is zero rather than an error + val textFlow = """ + a: 0XFFFFFFFFFFFFFFFF + b: -0X8000000000000000 + c: 0b-1000000000000000000000000000000000000000000000000000000000000000 + d: -_99999999999999999999 + e: 1e-999 + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "a", "type" to "int") { +"0XFFFFFFFFFFFFFFFF" } + "entry"("key" to "b", "type" to "int") { +"-0X8000000000000000" } + "entry"("key" to "c", "type" to "int") { + +"0b-1000000000000000000000000000000000000000000000000000000000000000" + } + "entry"("key" to "d", "type" to "float") { +"-_99999999999999999999" } + "entry"("key" to "e", "type" to "float") { +"1e-999" } + } + } + + @Test + fun `DIVERGENCE - should not type a plain scalar a reader refuses to load`() = runTest { // given — PyYAML resolves `=` and `<<` to its value / merge tags and // then refuses to construct them; it refuses a date not in the // calendar or a time / offset out of range (which Psych reads as a @@ -499,34 +552,25 @@ class YamlParserTest { } @Test - fun `should resolve a long near-miss of a base prefixed number in linear time`() { - // given — a base prefix, digits and one character outside the base: + fun `should resolve a long near-miss of a base prefixed number`() { + // given — a base prefix, a long digit run and one character outside + // the base but still numeric-shaped, so the number patterns do run: // a pattern that lets the digits match two ways backtracks - // quadratically over them, and every plain scalar is typed - fun nearMisses(length: Int) = listOf("0x", "0b", "+0x", "-0b").map { prefix -> - prefix + "1".repeat(length) + "g" + // quadratically, which at this length is hours instead of + // milliseconds — a regression hangs the test rather than failing an + // assertion, since a wall-clock bound would be flaky across targets + val values = listOf("0x", "+0x").map { it + "1".repeat(500_000) + "." } + + listOf("0b", "-0b").map { it + "1".repeat(500_000) + "2" } + + for (value in values) { + // when + val type = yamlScalarType(value) + val unsafe = isUnsafePlainScalar(value) + + // then + assert(type == null) + assert(!unsafe) } - // the best of a few runs, so a pause of the runtime is not measured - fun timeToResolve(values: List<String>): Duration = List(3) { - TimeSource.Monotonic.measureTime { - for (value in values) { - assert(yamlScalarType(value) == null) - assert(!isUnsafePlainScalar(value)) - } - } - }.min() - val short = nearMisses(25_000) - val long = nearMisses(100_000) - - // when - val shortTime = timeToResolve(short) - val longTime = timeToResolve(long) - - // then — four times the input takes about four times as long when - // linear, sixteen times when quadratic; the allowance absorbs the - // timer's resolution when both are only milliseconds, and no absolute - // bound is set, since a Native or JS regex engine is much slower - assert(longTime < shortTime * 8 + 50.milliseconds) } @Test @@ -1029,7 +1073,56 @@ class YamlParserTest { } @Test - fun `should DIVERGENCE keep an unrecognised line verbatim`() = runTest { + fun `DIVERGENCE - should keep a flow scalar with a colon before a flow indicator verbatim`() = runTest { + // given — the readers disagree on a plain flow scalar whose `:` is + // followed by `,` / `[` / `]` / `{` / `}`: PyYAML reads a mapping + // (`[{draft: null}, x]`), Psych refuses the line, go-yaml v2 reads + // the colon as content (`"draft:"`), so no single reading is right and + // the line is kept as written; a colon followed by anything else is + // content for all three + val textFlow = """ + title: x + tags: [draft:, x] + one: [a:] + geo: {a:[1, 2]} + nested: {a:{b: 1}} + empty: {a:, b: 1} + last: {a:} + time: [a:b] + """.trimIndent().chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + "entry"("key" to "title") { +"x" } + +"tags: [draft:, x]\none: [a:]\ngeo: {a:[1, 2]}\nnested: {a:{b: 1}}\nempty: {a:, b: 1}\nlast: {a:}\n" + "entry"("key" to "time") { + "item" { +"a:b" } + } + } + } + + @Test + fun `should keep a block scalar header with a hash right after it verbatim`() = runTest { + // given — Psych and go-yaml v2 read `|-#x` as a header and a comment, + // but PyYAML refuses it, so the line is outside the subset; with + // whitespace before the `#` every reader takes the comment + val textFlow = "a: |-#x\nb: | #y\n z\n".chunkedRandomly().asFlow() + + // when + val parsed = textFlow.parseYaml() + + // then + parsed.mergeAdjacentText() sameAs semanticEvents { + +"a: |-#x\n" + "entry"("key" to "b") { +"z\n" } + } + } + + @Test + fun `DIVERGENCE - should keep an unrecognised line verbatim`() = runTest { // given — a complex key, a multi-line flow sequence, an unterminated // quote, a `key:value` without a space and a tab-indented line are // outside the subset; each line survives verbatim (with its newline) @@ -1057,7 +1150,7 @@ class YamlParserTest { } @Test - fun `should DIVERGENCE keep a sequence line inside a mapping verbatim`() = runTest { + fun `DIVERGENCE - should keep a sequence line inside a mapping verbatim`() = runTest { // given — a `- item` at the mapping's own indentation is a YAML error val textFlow = """ title: x @@ -1075,7 +1168,7 @@ class YamlParserTest { } @Test - fun `should DIVERGENCE parse anchors aliases and tags as plain strings`() = runTest { + fun `DIVERGENCE - should parse anchors aliases and tags as plain strings`() = runTest { // given val textFlow = """ base: &b value @@ -1095,7 +1188,7 @@ class YamlParserTest { } @Test - fun `should DIVERGENCE accept a mapping indicator inside a plain value`() = runTest { + fun `DIVERGENCE - should accept a mapping indicator inside a plain value`() = runTest { // given — YAML readers (Psych, PyYAML) reject both lines val textFlow = """ note: Note: see diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt index cc4c158..c38a12d 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlRoundTripTest.kt @@ -242,6 +242,24 @@ class YamlRoundTripTest { rendered sameAs source } + @Test + fun `should keep a line the readers disagree on as written`() = runTest { + // given — PyYAML, Psych and go-yaml v2 read a colon before a flow + // indicator three different ways, and PyYAML alone refuses a hash + // right after a block scalar header, so these lines stay verbatim + val source = """ + tags: [draft:, x] + geo: {a:[1, 2]} + text: |-#x + """.trimIndent() + "\n" + + // when + val rendered = flowOf(source).parseYaml().renderYaml() + + // then + rendered sameAs source + } + @Test fun `should quote a string go-yaml v2 reads as a number once underscores are removed`() = runTest { // given @@ -266,6 +284,7 @@ class YamlRoundTripTest { // given val values = listOf( "1,", "1,,2", "._0", ".e5", "2024-2-30", "2024-5-32", "2024-01-01 10:00:00 +530", + "1e999", ".5e999", "0XFFFFFFFFFFFFFFFFF", "0o7777777777777777777777777", ) for (value in values) { val events = semanticEvents { @@ -283,7 +302,7 @@ class YamlRoundTripTest { } @Test - fun `should DIVERGENCE quote a plain scalar a reader refuses to load`() = runTest { + fun `DIVERGENCE - should quote a plain scalar a reader refuses to load`() = runTest { // given — the parser leaves these strings (no reader has a type for // them), but PyYAML or Psych refuse the whole document when they are // plain, so the writer quotes them and the first render rewrites the @@ -366,7 +385,7 @@ class YamlRoundTripTest { } @Test - fun `should DIVERGENCE write a lone surrogate as the replacement character`() = runTest { + fun `DIVERGENCE - should write a lone surrogate as the replacement character`() = runTest { // given — a lone surrogate (a JS string cut mid-pair) has no YAML // representation: raw it is outside the printable set, and Psych and // go-yaml refuse its `\u` escape as an invalid code point, so either @@ -395,7 +414,7 @@ class YamlRoundTripTest { } @Test - fun `should DIVERGENCE write a lone surrogate in a key as the replacement character`() = runTest { + fun `DIVERGENCE - should write a lone surrogate in a key as the replacement character`() = runTest { // given val events = semanticEvents { "entry"("key" to "a\uD800") { +"v" }