From dcd14743eeb827ebbe6a7538064144228891254f Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Mon, 28 Sep 2026 14:07:32 +0200 Subject: [PATCH 01/25] Drop application-state values from front matter; first non-blank title wins (#82) Single-page apps ship serialised state in under framework-private names (LinkedIn's __init, Ember's percent-encoded config/environment, ...), swamping the front matter. simplifyHtml now judges the value: a leading JSON object/array (raw or percent-encoded), a value over 4096 chars, or an HTML-blank value is dropped. Bracketed prefixes like "[Solved] ..." survive. When holds several s, the first non-blank one wins, and it takes precedence over <meta name="title"> in either order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 1 + .../src/commonMain/kotlin/SimplifyHtml.kt | 64 ++++- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 221 ++++++++++++++++++ 3 files changed, 283 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 90e1790..da1a986 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -409,6 +409,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' The operator **requires** the DISPLAY annotation to be accurate (captured dumps); it is suppressed inside `<pre>`/`<code>`/`<textarea>`. - `simplifyHtml` drops **technical-noise `<meta name>`** values from the frontmatter via a denylist (`isNoiseMetaName` in `SimplifyHtml.kt`: `viewport`, `generator`, `theme-color`, `robots`, `msapplication-*`, `apple-*`, `*-verification`, …). A denylist (not an allowlist) is deliberate so unknown-but-useful names (`og:*`, `article:*`, custom) survive; extend the denylist as new noise names appear. + Single-page apps ship application state in `<meta>` under framework-private names no denylist can anticipate (issue #82), so new state shapes belong in the value-based `isApplicationStateMeta`, not in the name list. `renderDumpMarkdown` (Gradle task, `RenderDumpMarkdown.kt` in jvmTest) runs the full `transformHtmlToMarkdown` pipeline over every dump to `build/renderedMarkdown/<name>.md` — the canonical way to regenerate the per-dump golden strings (`dumps/OpenjurTest`, `dumps/HackerNewsTest`) after a pipeline change. - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 94e578e..9dba750 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -18,6 +18,8 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations +import com.xemantic.markanywhere.html.spec.isHtmlBlank +import com.xemantic.markanywhere.html.spec.isHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform import kotlinx.coroutines.flow.Flow @@ -131,7 +133,10 @@ import kotlinx.coroutines.flow.Flow * matter vocabulary — just before `<body>` content streams through. Technical meta * names that carry no content signal (rendering hints, crawler / verification * directives, platform tile metadata — see [isNoiseMetaName]) are dropped so - * they don't inflate the frontmatter. If `<head>` is absent or yields no + * they don't inflate the frontmatter, and so is application state that + * single-page apps ship in `<meta>` (serialised JSON, framework config + * blobs — see [isApplicationStateMeta]). When `<head>` holds several + * `<title>`s, the first non-blank one wins, over a `<meta name="title">` too. If `<head>` is absent or yields no * metadata, no frontmatter mark is emitted. * * Matcher registration is grouped: per-tag explicit matchers come first @@ -165,6 +170,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( val metadata = mutableMapOf<String, String>() val titleText = StringBuilder() + var titleElementSeen = false // Attribute map kept on a preserved element: its own [names] whitelist, the // ARIA name/state keep-set, and any caller-requested [keepAttributes]. An @@ -258,10 +264,17 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("title") { // Capture the title's text into metadata; the mark itself is dropped. + // A page can carry several <title>s (e.g. after a client-side + // navigation): each is collected on its own and the first non-blank + // one wins — unlike `document.title`, which takes the first even when + // blank, since a blank title tells the reader nothing. It also wins + // over a `<meta name="title">` in either order. + titleText.clear() children(mode = "titleText") afterClose { val trimmed = titleText.toString().trim() - if (trimmed.isNotEmpty()) { + if (trimmed.isNotEmpty() && !titleElementSeen) { + titleElementSeen = true metadata["title"] = trimmed } } @@ -270,7 +283,11 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("meta") { event -> val name = event["name"] val content = event["content"] - if (name != null && content != null && !isNoiseMetaName(name)) { + if (name != null && content != null + && !isNoiseMetaName(name) + && !isApplicationStateMeta(content) + && !(name == "title" && titleElementSeen) + ) { metadata[name] = content } } @@ -690,6 +707,47 @@ private fun isNoiseMetaName(name: String): Boolean { || n.startsWith("verify-") } +// Single-page apps use `<meta>` as a transport for application state +// (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded +// `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A +// name denylist cannot keep up with names private to each site's framework, +// so these rules judge the *value*, which is what tells metadata apart from +// state: +// - a value opening a JSON object or array, raw or percent-encoded — a `{` +// elsewhere is fine, and so is a bracketed tag (`[Solved] …`, `[1] …`): +// `[` counts only when followed by what a JSON array can start with and +// the value ends with `]`; +// - a value longer than [MAX_META_VALUE_LENGTH] — the backstop for opaque +// blobs of any other shape (base64, hash lists); +// - an HTML-blank value, which tells the reader nothing. +private fun isApplicationStateMeta(content: String): Boolean { + if (content.isHtmlBlank()) return true + val value = content.trim { it.isHtmlWhitespace() } + return value.length > MAX_META_VALUE_LENGTH + || value.startsWith('{') + || value.startsWith("%7B", ignoreCase = true) + || value.startsWith('[') && value.endsWith(']') + && value.drop(1).trimStart { it.isHtmlWhitespace() }.startsJsonValue() + || value.startsWith("%5B", ignoreCase = true) && value.endsWith("%5D", ignoreCase = true) + && value.drop(3).startsPercentEncodedJsonValue() +} + +// What may follow a JSON array's `[`: a value (object, array, string, number) +// or the `]` of an empty array. +private fun String.startsJsonValue(): Boolean = + firstOrNull()?.let { it in "{[\"]-" || it.isDigit() } ?: false + +private fun String.startsPercentEncodedJsonValue(): Boolean = + PERCENT_ENCODED_JSON_STARTS.any { startsWith(it, ignoreCase = true) } + || firstOrNull()?.let { it == '-' || it.isDigit() } ?: false + +private val PERCENT_ENCODED_JSON_STARTS = listOf("%7B", "%5B", "%22", "%5D") + +// Real metadata is short: `description` / `og:description` rarely exceed 300 +// characters, and the cap still fits a full academic abstract +// (`citation_abstract`, `dc.description`). +private const val MAX_META_VALUE_LENGTH = 4096 + private val NOISE_META_NAMES = setOf( "viewport", "referrer", "generator", "theme-color", "color-scheme", "format-detection", "tdm-reservation", "robots", "googlebot", "bingbot", diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index f711d90..5c77295 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -838,6 +838,227 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop application-state meta values from frontmatter`() = runTest { + // given — single-page-app state serialised into <meta content>, the + // shapes that swamped LinkedIn's front matter (issue #82) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "__init", "content" to "{\"a\":1}") { } + "meta"("name" to "storage-inventory", "content" to " [1,2,3]") { } + "meta"("name" to "como-err", "content" to "[{\"x\":1}]") { } + "meta"("name" to "empty-list", "content" to "[]") { } + "meta"("name" to "feed/config/environment", "content" to "%7B%22x%22%3A1%7D") { } + "meta"("name" to "jam/config/environment", "content" to "%7b%7d") { } + "meta"("name" to "hash-list", "content" to "%5B%22a%22%5D") { } + "meta"("name" to "description", "content" to "Save {50%} today") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then — only a *leading* JSON object/array marks state + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +"Save {50%} today" } + } + "p" { +"text" } + } + } + + @Test + fun `should keep meta values opening with a bracketed tag`() = runTest { + // given — human-readable prefixes, not JSON arrays + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "description", "content" to "[Solved] How to fix X") { } + "meta"("name" to "og:title", "content" to "[PDF] Annual report") { } + "meta"("name" to "abstract", "content" to "[1] Introduction") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +"[Solved] How to fix X" } + "entry"("key" to "og:title") { +"[PDF] Annual report" } + "entry"("key" to "abstract") { +"[1] Introduction" } + } + "p" { +"text" } + } + } + + @Test + fun `should drop meta values longer than the cap from frontmatter`() = runTest { + // given — the cap still fits a long academic abstract + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "blob", "content" to "x".repeat(4097)) { } + "meta"("name" to "citation_abstract", "content" to "y".repeat(4096)) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "citation_abstract") { +"y".repeat(4096) } + } + "p" { +"text" } + } + } + + @Test + fun `should drop blank meta values from frontmatter`() = runTest { + // given — NBSP is HTML content, not whitespace + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to "") { } + "meta"("name" to "abstract", "content" to " \t\n") { } + "meta"("name" to "separator", "content" to "\u00A0") { } + "meta"("name" to "section/topic", "content" to "Politics") { } + "meta"("name" to "description", "content" to "A doc") { } + "meta"("name" to "og:title", "content" to "Hello") { } + "meta"("name" to "author", "content" to "Alice") { } + "meta"("name" to "article:published_time", "content" to "2026-09-27T10:00:00Z") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then — ordinary metadata survives, a `/` in the name included + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "separator") { +"\u00A0" } + "entry"("key" to "section/topic") { +"Politics" } + "entry"("key" to "description") { +"A doc" } + "entry"("key" to "og:title") { +"Hello" } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "article:published_time") { +"2026-09-27T10:00:00Z" } + } + "p" { +"text" } + } + } + + @Test + fun `should keep the first non-blank title when head holds several`() = runTest { + // given — e.g. after a client-side navigation (issue #82) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +" " } + "title" { +"A" } + "title" { +"B" } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then — not "AB", not "B" + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"A" } + } + "p" { +"x" } + } + } + + @Test + fun `should prefer the title element over a title meta preceding it`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "title", "content" to "SEO blurb") { } + "title" { +"Actual Page" } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Actual Page" } + } + "p" { +"x" } + } + } + + @Test + fun `should prefer the title element over a title meta following it`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"Actual Page" } + "meta"("name" to "title", "content" to "SEO blurb") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Actual Page" } + } + "p" { +"x" } + } + } + + @Test + fun `should fall back to a title meta when no title element is present`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "title", "content" to "SEO blurb") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"SEO blurb" } + } + "p" { +"x" } + } + } + @Test fun `should not emit frontmatter when head yields no metadata`() = runTest { // given From dbb1fe0daa45aa7b89344ea4dd3b6e0300557767 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 17:40:57 +0200 Subject: [PATCH 02/25] Judge <meta> application state by parsing JSON; resolve title candidates at </head> (#82) isApplicationStateMeta moves to ApplicationStateMeta.kt and now parses the value as a JSON object/array (raw or percent-decoded) instead of peeking at its first and last char, so bracketed human text like "[2024] Annual report [PDF]" survives. A blank value is dropped on its own, not as "application state". Title candidates are resolved when <head> closes: the first non-blank <title> wins over <meta name="title"> (matched case-insensitively), and the entry keeps the position of the first candidate. Documented which values don't survive a wrapInHtmlDocument round-trip. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 2 +- README.md | 2 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 204 ++++++++++++++++++ .../src/commonMain/kotlin/SimplifyHtml.kt | 95 +++----- .../commonMain/kotlin/WrapInHtmlDocument.kt | 3 +- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 98 ++++++++- .../kotlin/WrapInHtmlDocumentTest.kt | 30 ++- 7 files changed, 367 insertions(+), 67 deletions(-) create mode 100644 markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt diff --git a/CLAUDE.md b/CLAUDE.md index da1a986..4d197e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -409,7 +409,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' The operator **requires** the DISPLAY annotation to be accurate (captured dumps); it is suppressed inside `<pre>`/`<code>`/`<textarea>`. - `simplifyHtml` drops **technical-noise `<meta name>`** values from the frontmatter via a denylist (`isNoiseMetaName` in `SimplifyHtml.kt`: `viewport`, `generator`, `theme-color`, `robots`, `msapplication-*`, `apple-*`, `*-verification`, …). A denylist (not an allowlist) is deliberate so unknown-but-useful names (`og:*`, `article:*`, custom) survive; extend the denylist as new noise names appear. - Single-page apps ship application state in `<meta>` under framework-private names no denylist can anticipate (issue #82), so new state shapes belong in the value-based `isApplicationStateMeta`, not in the name list. + Value-shaped noise (single-page-app state, issue #82) belongs in `isApplicationStateMeta`, not in the name list. `renderDumpMarkdown` (Gradle task, `RenderDumpMarkdown.kt` in jvmTest) runs the full `transformHtmlToMarkdown` pipeline over every dump to `build/renderedMarkdown/<name>.md` — the canonical way to regenerate the per-dump golden strings (`dumps/OpenjurTest`, `dumps/HackerNewsTest`) after a pipeline change. - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. diff --git a/README.md b/README.md index 68148bd..fad43b3 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document -`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip. +`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader: a blank value, and application state such as a JSON object or an over-long blob. ```kotlin val document = """ diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt new file mode 100644 index 0000000..7cf5ec5 --- /dev/null +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -0,0 +1,204 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xemantic.markanywhere.html + +import com.xemantic.markanywhere.html.spec.isHtmlWhitespace + +// Single-page apps use `<meta>` as a transport for application state +// (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded +// `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A +// name denylist cannot keep up with names private to each site's framework, +// so this judges the *value*, which is what tells metadata apart from state: +// - a value that parses as a JSON object or array, raw or percent-encoded. +// Parsing, not a look at the first and last char, is what keeps human text +// that merely starts with a bracket (`[Solved] …`, `{Draft} …`, +// `[2024] Annual report [PDF]`); +// - a value longer than [MAX_META_VALUE_LENGTH] — the backstop for opaque +// blobs of any other shape (base64, hash lists, truncated JSON). +internal fun isApplicationStateMeta(content: String): Boolean { + val value = content.trim { it.isHtmlWhitespace() } + return value.length > MAX_META_VALUE_LENGTH + || value.isJsonContainer() + || value.startsWith('%') && value.percentDecodedOrNull()?.isJsonContainer() == true +} + +// Real metadata is short: `description` / `og:description` rarely exceed 300 +// characters, and the cap still fits a full academic abstract +// (`citation_abstract`, `dc.description`). +private const val MAX_META_VALUE_LENGTH = 4096 + +// Decodes `%XX` escapes as UTF-8, or null when the value is not +// percent-encoded text (a malformed escape, a raw non-ASCII char). +private fun String.percentDecodedOrNull(): String? { + val bytes = ByteArray(length) + var size = 0 + var i = 0 + while (i < length) { + val c = this[i] + if (c == '%') { + val high = getOrNull(i + 1)?.digitToIntOrNull(16) ?: return null + val low = getOrNull(i + 2)?.digitToIntOrNull(16) ?: return null + bytes[size++] = (high * 16 + low).toByte() + i += 3 + } else { + if (c.code > 0x7F) return null + bytes[size++] = c.code.toByte() + i++ + } + } + return bytes.decodeToString(0, size) +} + +private enum class JsonExpect { VALUE, VALUE_OR_END, KEY, KEY_OR_END, COLON, SEPARATOR_OR_END } + +// Whether the whole string (JSON whitespace around it allowed) is one JSON +// object or array (RFC 8259). Iterative with an explicit stack, so nesting +// depth in untrusted input cannot overflow the call stack. +private fun String.isJsonContainer(): Boolean { + var i = skipJsonWhitespace(0) + if (getOrNull(i) != '{' && getOrNull(i) != '[') return false + val open = StringBuilder() + var expect: JsonExpect = VALUE + while (true) { + i = skipJsonWhitespace(i) + val c = getOrNull(i) ?: return false + when (expect) { + VALUE, VALUE_OR_END -> when { + c == ']' && expect == VALUE_OR_END -> { + open.setLength(open.length - 1) + i++ + if (open.isEmpty()) return skipJsonWhitespace(i) == length + expect = SEPARATOR_OR_END + } + c == '{' -> { + open.append(c) + i++ + expect = KEY_OR_END + } + c == '[' -> { + open.append(c) + i++ + expect = VALUE_OR_END + } + else -> { + i = skipJsonScalar(i) ?: return false + expect = SEPARATOR_OR_END + } + } + KEY, KEY_OR_END -> when { + c == '}' && expect == KEY_OR_END -> { + open.setLength(open.length - 1) + i++ + if (open.isEmpty()) return skipJsonWhitespace(i) == length + expect = SEPARATOR_OR_END + } + c == '"' -> { + i = skipJsonString(i) ?: return false + expect = COLON + } + else -> return false + } + COLON -> { + if (c != ':') return false + i++ + expect = VALUE + } + SEPARATOR_OR_END -> { + val container = open.last() + when (c) { + ',' -> expect = if (container == '{') KEY else VALUE + '}', ']' -> { + if (c != if (container == '{') '}' else ']') return false + open.setLength(open.length - 1) + if (open.isEmpty()) return skipJsonWhitespace(i + 1) == length + } + else -> return false + } + i++ + } + } + } +} + +private fun String.skipJsonWhitespace(from: Int): Int { + var i = from + while (i < length && this[i] in " \t\n\r") i++ + return i +} + +// The index past a string, number or literal starting at [from], or null. +private fun String.skipJsonScalar(from: Int): Int? = when (this[from]) { + '"' -> skipJsonString(from) + 't' -> skipJsonLiteral(from, "true") + 'f' -> skipJsonLiteral(from, "false") + 'n' -> skipJsonLiteral(from, "null") + else -> skipJsonNumber(from) +} + +private fun String.skipJsonLiteral(from: Int, literal: String): Int? = + if (startsWith(literal, from)) from + literal.length else null + +private fun String.skipJsonString(from: Int): Int? { + var i = from + 1 + while (i < length) { + val c = this[i] + when { + c == '"' -> return i + 1 + c < ' ' -> return null + c == '\\' -> { + val escaped = getOrNull(i + 1) ?: return null + i += when (escaped) { + in "\"\\/bfnrt" -> 2 + 'u' -> if ( + (i + 2..i + 5).all { getOrNull(it)?.digitToIntOrNull(16) != null } + ) 6 else return null + else -> return null + } + } + else -> i++ + } + } + return null +} + +// `-? (0 | [1-9][0-9]*) (. [0-9]+)? ([eE] [+-]? [0-9]+)?` +private fun String.skipJsonNumber(from: Int): Int? { + var i = from + if (getOrNull(i) == '-') i++ + when (getOrNull(i)) { + '0' -> i++ + in '1'..'9' -> i = skipDigits(i) + else -> return null + } + if (getOrNull(i) == '.') { + if (getOrNull(i + 1) !in '0'..'9') return null + i = skipDigits(i + 1) + } + if (getOrNull(i) == 'e' || getOrNull(i) == 'E') { + i++ + if (getOrNull(i) == '+' || getOrNull(i) == '-') i++ + if (getOrNull(i) !in '0'..'9') return null + i = skipDigits(i) + } + return i +} + +private fun String.skipDigits(from: Int): Int { + var i = from + while (getOrNull(i) in '0'..'9') i++ + return i +} diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 9dba750..a0fa571 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -133,11 +133,13 @@ import kotlinx.coroutines.flow.Flow * matter vocabulary — just before `<body>` content streams through. Technical meta * names that carry no content signal (rendering hints, crawler / verification * directives, platform tile metadata — see [isNoiseMetaName]) are dropped so - * they don't inflate the frontmatter, and so is application state that - * single-page apps ship in `<meta>` (serialised JSON, framework config - * blobs — see [isApplicationStateMeta]). When `<head>` holds several - * `<title>`s, the first non-blank one wins, over a `<meta name="title">` too. If `<head>` is absent or yields no - * metadata, no frontmatter mark is emitted. + * they don't inflate the frontmatter, and so are a blank value and + * application state that single-page apps ship in `<meta>` (serialised JSON, + * framework config blobs — see [isApplicationStateMeta]). Those are the + * values that do not survive a [wrapInHtmlDocument] round-trip. When `<head>` + * holds several `<title>`s, the first non-blank one wins, over a + * `<meta name="title">` (in any letter case) too. If `<head>` is absent or + * yields no metadata, no frontmatter mark is emitted. * * Matcher registration is grouped: per-tag explicit matchers come first * (so they win the `firstOrNull` race), then a small number of @@ -170,7 +172,14 @@ public fun Flow<SemanticEvent>.simplifyHtml( val metadata = mutableMapOf<String, String>() val titleText = StringBuilder() - var titleElementSeen = false + // Title candidates, resolved once when `<head>` closes: the first + // non-blank `<title>` wins, else the first non-blank `<meta name="title">` + // — unlike `document.title`, which takes the first `<title>` even when + // blank, since a blank title tells the reader nothing. The entry lands + // where the first candidate appeared. + var elementTitle: String? = null + var metaTitle: String? = null + var titlePosition = -1 // Attribute map kept on a preserved element: its own [names] whitelist, the // ARIA name/state keep-set, and any caller-requested [keepAttributes]. An @@ -252,9 +261,11 @@ public fun Flow<SemanticEvent>.simplifyHtml( // swallowed (see the mode-scoped matchText below); emit no mark. children(mode = "head") afterClose { - if (metadata.isNotEmpty()) { + val entries = metadata.toList().toMutableList() + (elementTitle ?: metaTitle)?.let { entries.add(titlePosition, "title" to it) } + if (entries.isNotEmpty()) { "frontmatter" { - for ((key, value) in metadata) { + for ((key, value) in entries) { "entry"("key" to key) { +value } } } @@ -263,19 +274,16 @@ public fun Flow<SemanticEvent>.simplifyHtml( } match("title") { - // Capture the title's text into metadata; the mark itself is dropped. + // Capture the title's text as a candidate; the mark itself is dropped. // A page can carry several <title>s (e.g. after a client-side - // navigation): each is collected on its own and the first non-blank - // one wins — unlike `document.title`, which takes the first even when - // blank, since a blank title tells the reader nothing. It also wins - // over a `<meta name="title">` in either order. + // navigation), so each is collected on its own. titleText.clear() children(mode = "titleText") afterClose { - val trimmed = titleText.toString().trim() - if (trimmed.isNotEmpty() && !titleElementSeen) { - titleElementSeen = true - metadata["title"] = trimmed + val trimmed = titleText.toString().trim { it.isHtmlWhitespace() } + if (trimmed.isNotEmpty() && elementTitle == null) { + elementTitle = trimmed + if (titlePosition < 0) titlePosition = metadata.size } } } @@ -284,11 +292,19 @@ public fun Flow<SemanticEvent>.simplifyHtml( val name = event["name"] val content = event["content"] if (name != null && content != null + && !content.isHtmlBlank() && !isNoiseMetaName(name) && !isApplicationStateMeta(content) - && !(name == "title" && titleElementSeen) ) { - metadata[name] = content + // Meta names are ASCII case-insensitive (HTML §4.2.5). + if (name.equals("title", ignoreCase = true)) { + if (metaTitle == null) { + metaTitle = content + if (titlePosition < 0) titlePosition = metadata.size + } + } else { + metadata[name] = content + } } } @@ -707,47 +723,6 @@ private fun isNoiseMetaName(name: String): Boolean { || n.startsWith("verify-") } -// Single-page apps use `<meta>` as a transport for application state -// (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded -// `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A -// name denylist cannot keep up with names private to each site's framework, -// so these rules judge the *value*, which is what tells metadata apart from -// state: -// - a value opening a JSON object or array, raw or percent-encoded — a `{` -// elsewhere is fine, and so is a bracketed tag (`[Solved] …`, `[1] …`): -// `[` counts only when followed by what a JSON array can start with and -// the value ends with `]`; -// - a value longer than [MAX_META_VALUE_LENGTH] — the backstop for opaque -// blobs of any other shape (base64, hash lists); -// - an HTML-blank value, which tells the reader nothing. -private fun isApplicationStateMeta(content: String): Boolean { - if (content.isHtmlBlank()) return true - val value = content.trim { it.isHtmlWhitespace() } - return value.length > MAX_META_VALUE_LENGTH - || value.startsWith('{') - || value.startsWith("%7B", ignoreCase = true) - || value.startsWith('[') && value.endsWith(']') - && value.drop(1).trimStart { it.isHtmlWhitespace() }.startsJsonValue() - || value.startsWith("%5B", ignoreCase = true) && value.endsWith("%5D", ignoreCase = true) - && value.drop(3).startsPercentEncodedJsonValue() -} - -// What may follow a JSON array's `[`: a value (object, array, string, number) -// or the `]` of an empty array. -private fun String.startsJsonValue(): Boolean = - firstOrNull()?.let { it in "{[\"]-" || it.isDigit() } ?: false - -private fun String.startsPercentEncodedJsonValue(): Boolean = - PERCENT_ENCODED_JSON_STARTS.any { startsWith(it, ignoreCase = true) } - || firstOrNull()?.let { it == '-' || it.isDigit() } ?: false - -private val PERCENT_ENCODED_JSON_STARTS = listOf("%7B", "%5B", "%22", "%5D") - -// Real metadata is short: `description` / `og:description` rarely exceed 300 -// characters, and the cap still fits a full academic abstract -// (`citation_abstract`, `dc.description`). -private const val MAX_META_VALUE_LENGTH = 4096 - private val NOISE_META_NAMES = setOf( "viewport", "referrer", "generator", "theme-color", "color-scheme", "format-detection", "tdm-reservation", "robots", "googlebot", "bingbot", diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index de751ff..c126932 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -27,7 +27,8 @@ import kotlinx.coroutines.flow.Flow * A **leading** `frontmatter` block (the untagged mark the parser emits for * YAML `---` front matter, holding `entry` marks) feeds the `head` — the * inverse of [simplifyHtml]'s head-to-frontmatter extraction, so the two - * round-trip: + * round-trip, except for the values [simplifyHtml] discards (a blank value, + * application state such as a JSON object or an over-long blob): * * - the `title` entry becomes `<title>` * - the `lang` entry becomes the `lang` attribute on `<html>` diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 5c77295..b16853e 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -852,6 +852,9 @@ class SimplifyHtmlTest { "meta"("name" to "feed/config/environment", "content" to "%7B%22x%22%3A1%7D") { } "meta"("name" to "jam/config/environment", "content" to "%7b%7d") { } "meta"("name" to "hash-list", "content" to "%5B%22a%22%5D") { } + "meta"("name" to "flags", "content" to "[true,false]") { } + "meta"("name" to "slots", "content" to "[null]") { } + "meta"("name" to "spaced", "content" to "%5B%20%7B%22a%22%3A1%7D%20%5D") { } "meta"("name" to "description", "content" to "Save {50%} today") { } } "body" { "p" { +"text" } } @@ -861,7 +864,7 @@ class SimplifyHtmlTest { // when val output = input.simplifyHtml() - // then — only a *leading* JSON object/array marks state + // then — only a value that *is* a JSON object/array marks state output sameAs semanticEvents { "frontmatter" { "entry"("key" to "description") { +"Save {50%} today" } @@ -871,14 +874,19 @@ class SimplifyHtmlTest { } @Test - fun `should keep meta values opening with a bracketed tag`() = runTest { - // given — human-readable prefixes, not JSON arrays + fun `should keep meta values opening with a bracketed or braced tag`() = runTest { + // given — human-readable prefixes, not JSON objects or arrays, even + // when the value also ends with a bracket val input = semanticEvents(tagged = true) { "html" { "head" { "meta"("name" to "description", "content" to "[Solved] How to fix X") { } "meta"("name" to "og:title", "content" to "[PDF] Annual report") { } "meta"("name" to "abstract", "content" to "[1] Introduction") { } + "meta"("name" to "twitter:title", "content" to "[2024] Annual report [PDF]") { } + "meta"("name" to "summary", "content" to "[1] Intro, see [2]") { } + "meta"("name" to "og:description", "content" to "{Kotlin} Multiplatform guide") { } + "meta"("name" to "subject", "content" to "%5BDraft%5D notes") { } } "body" { "p" { +"text" } } } @@ -893,6 +901,10 @@ class SimplifyHtmlTest { "entry"("key" to "description") { +"[Solved] How to fix X" } "entry"("key" to "og:title") { +"[PDF] Annual report" } "entry"("key" to "abstract") { +"[1] Introduction" } + "entry"("key" to "twitter:title") { +"[2024] Annual report [PDF]" } + "entry"("key" to "summary") { +"[1] Intro, see [2]" } + "entry"("key" to "og:description") { +"{Kotlin} Multiplatform guide" } + "entry"("key" to "subject") { +"%5BDraft%5D notes" } } "p" { +"text" } } @@ -1035,6 +1047,86 @@ class SimplifyHtmlTest { } } + @Test + fun `should prefer the title element over a title meta in any letter case`() = runTest { + // given — meta names are matched case-insensitively (HTML §4.2.5) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "Title", "content" to "SEO blurb") { } + "title" { +"Actual Page" } + "meta"("name" to "TITLE", "content" to "Other blurb") { } + "meta"("name" to "author", "content" to "Alice") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then — a single title, at the position of the first candidate + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Actual Page" } + "entry"("key" to "author") { +"Alice" } + } + "p" { +"x" } + } + } + + @Test + fun `should keep the first non-blank title meta when there is no title element`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "author", "content" to "Alice") { } + "meta"("name" to "title", "content" to "First") { } + "meta"("name" to "Title", "content" to "Second") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { +"First" } + } + "p" { +"x" } + } + } + + @Test + fun `should keep non-breaking spaces in a title`() = runTest { + // given — NBSP is content, not HTML whitespace, like in a meta value + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"  Page \n" } + "title" { +"Later" } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +" Page " } + } + "p" { +"x" } + } + } + @Test fun `should fall back to a title meta when no title element is present`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 35ffade..4370b07 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -33,7 +33,7 @@ import kotlin.test.Test * `entry` marks) feeds the `head`: the `title` entry becomes `<title>`, * `lang` becomes the `<html lang>` attribute, every other top-level scalar * entry becomes a `<meta name content>` — the inverse of `simplifyHtml`'s - * head-to-frontmatter extraction. + * head-to-frontmatter extraction, minus the values it discards. */ class WrapInHtmlDocumentTest { @@ -285,6 +285,34 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should round-trip through simplifyHtml except the values it discards`() = runTest { + // given — simplifyHtml keeps no blank value and no application state + // (a JSON object or array, or an over-long blob), so those entries do + // not come back; text that merely looks bracketed does + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Page" } + "entry"("key" to "summary") { +"{draft} notes" } + "entry"("key" to "keywords") { } + "entry"("key" to "state") { +"{\"a\":1}" } + "entry"("key" to "blob") { +"x".repeat(4097) } + } + "p" { +"Hi" } + } + + // when + val output = input.wrapInHtmlDocument().simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Page" } + "entry"("key" to "summary") { +"{draft} notes" } + } + "p" { +"Hi" } + } + } @Test fun `should use an entry left open when the stream ends inside the frontmatter`() = runTest { From c0aa3b697dc19a5d0483086ec3e875042266a51d Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 20:42:21 +0200 Subject: [PATCH 03/25] Keep flat word/number arrays in <meta>; merge case-variant meta names (#82) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isApplicationStateMeta now parses with kotlinx Json.parseToJsonElement instead of a hand-rolled JSON validator. A JSON object is still dropped, but an array survives when it is a flat list of strings or numbers (keywords, article:tag, citation_volume); empty, nested or flag/null-holding arrays, and percent-encoded arrays, are dropped. Meta names are ASCII case-insensitive (HTML §4.2.5): of several names differing only in letter case, the first spelling and value win, and the title candidate logic folds into the same first-wins map. Documented the extra wrapInHtmlDocument round-trip caveat and the kotlinx Json leniency gotcha in CLAUDE.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 1 + README.md | 2 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 184 ++++-------------- .../src/commonMain/kotlin/SimplifyHtml.kt | 69 ++++--- .../commonMain/kotlin/WrapInHtmlDocument.kt | 3 +- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 176 ++++++++++++++++- .../kotlin/WrapInHtmlDocumentTest.kt | 27 +++ 7 files changed, 284 insertions(+), 178 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4d197e0..b0c6492 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -410,6 +410,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - `simplifyHtml` drops **technical-noise `<meta name>`** values from the frontmatter via a denylist (`isNoiseMetaName` in `SimplifyHtml.kt`: `viewport`, `generator`, `theme-color`, `robots`, `msapplication-*`, `apple-*`, `*-verification`, …). A denylist (not an allowlist) is deliberate so unknown-but-useful names (`og:*`, `article:*`, custom) survive; extend the denylist as new noise names appear. Value-shaped noise (single-page-app state, issue #82) belongs in `isApplicationStateMeta`, not in the name list. + It judges JSON with kotlinx's `Json.parseToJsonElement`, which is **not a strict validator** even with the default (non-lenient) `Json`: an unquoted token in value position parses as a non-string literal (`[PDF]` is an array, `{"a":abc}` an object), and numbers are never checked — so decide on the element's *shape*, never on "it parsed, so it is JSON". `renderDumpMarkdown` (Gradle task, `RenderDumpMarkdown.kt` in jvmTest) runs the full `transformHtmlToMarkdown` pipeline over every dump to `build/renderedMarkdown/<name>.md` — the canonical way to regenerate the per-dump golden strings (`dumps/OpenjurTest`, `dumps/HackerNewsTest`) after a pipeline change. - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. diff --git a/README.md b/README.md index fad43b3..7428e08 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document -`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader: a blank value, and application state such as a JSON object or an over-long blob. +`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader (a blank value, and application state such as a JSON object or an over-long blob), and for keys differing only in letter case, which HTML reads as one `<meta>` name, so the first one wins (a `Title` key is read as the title). ```kotlin val document = """ diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 7cf5ec5..1dafc63 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -17,23 +17,43 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.html.spec.isHtmlWhitespace +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive // Single-page apps use `<meta>` as a transport for application state // (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded // `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A // name denylist cannot keep up with names private to each site's framework, // so this judges the *value*, which is what tells metadata apart from state: -// - a value that parses as a JSON object or array, raw or percent-encoded. -// Parsing, not a look at the first and last char, is what keeps human text -// that merely starts with a bracket (`[Solved] …`, `{Draft} …`, +// - a value that parses as a JSON object, raw or percent-encoded. Parsing, +// not a look at the first and last char, is what keeps human text that +// merely starts with a bracket (`[Solved] …`, `{Draft} …`, // `[2024] Annual report [PDF]`); +// - a JSON array that is percent-encoded, or holds anything but strings and +// numbers (an object, a nested array, a flag, a `null`), or nothing at all. +// A flat list of words or numbers is metadata a person writes +// (`keywords`, `article:tag`, `citation_volume`); // - a value longer than [MAX_META_VALUE_LENGTH] — the backstop for opaque // blobs of any other shape (base64, hash lists, truncated JSON). internal fun isApplicationStateMeta(content: String): Boolean { val value = content.trim { it.isHtmlWhitespace() } - return value.length > MAX_META_VALUE_LENGTH - || value.isJsonContainer() - || value.startsWith('%') && value.percentDecodedOrNull()?.isJsonContainer() == true + if (value.length > MAX_META_VALUE_LENGTH) return true + val first = value.firstOrNull() + return when { + first == '{' || first == '[' -> when (val json = value.parseJsonOrNull()) { + is JsonObject -> true + is JsonArray -> json.isEmpty() || json.any { !it.isWordOrNumber() } + else -> false + } + first == '%' -> value.percentDecodedOrNull()?.parseJsonOrNull() + .let { it is JsonObject || it is JsonArray } + else -> false + } } // Real metadata is short: `description` / `og:description` rarely exceed 300 @@ -41,6 +61,18 @@ internal fun isApplicationStateMeta(content: String): Boolean { // (`citation_abstract`, `dc.description`). private const val MAX_META_VALUE_LENGTH = 4096 +private fun String.parseJsonOrNull(): JsonElement? = try { + Json.parseToJsonElement(this) +} catch (_: SerializationException) { + null +} + +// kotlinx's tree reader takes any unquoted token as a literal (`[PDF]` parses +// as an array), so everything that is not a string, a flag or `null` counts — +// a number, or a word that no JSON writer would have produced. +private fun JsonElement.isWordOrNumber(): Boolean = + this is JsonPrimitive && this !is JsonNull && (isString || content != "true" && content != "false") + // Decodes `%XX` escapes as UTF-8, or null when the value is not // percent-encoded text (a malformed escape, a raw non-ASCII char). private fun String.percentDecodedOrNull(): String? { @@ -62,143 +94,3 @@ private fun String.percentDecodedOrNull(): String? { } return bytes.decodeToString(0, size) } - -private enum class JsonExpect { VALUE, VALUE_OR_END, KEY, KEY_OR_END, COLON, SEPARATOR_OR_END } - -// Whether the whole string (JSON whitespace around it allowed) is one JSON -// object or array (RFC 8259). Iterative with an explicit stack, so nesting -// depth in untrusted input cannot overflow the call stack. -private fun String.isJsonContainer(): Boolean { - var i = skipJsonWhitespace(0) - if (getOrNull(i) != '{' && getOrNull(i) != '[') return false - val open = StringBuilder() - var expect: JsonExpect = VALUE - while (true) { - i = skipJsonWhitespace(i) - val c = getOrNull(i) ?: return false - when (expect) { - VALUE, VALUE_OR_END -> when { - c == ']' && expect == VALUE_OR_END -> { - open.setLength(open.length - 1) - i++ - if (open.isEmpty()) return skipJsonWhitespace(i) == length - expect = SEPARATOR_OR_END - } - c == '{' -> { - open.append(c) - i++ - expect = KEY_OR_END - } - c == '[' -> { - open.append(c) - i++ - expect = VALUE_OR_END - } - else -> { - i = skipJsonScalar(i) ?: return false - expect = SEPARATOR_OR_END - } - } - KEY, KEY_OR_END -> when { - c == '}' && expect == KEY_OR_END -> { - open.setLength(open.length - 1) - i++ - if (open.isEmpty()) return skipJsonWhitespace(i) == length - expect = SEPARATOR_OR_END - } - c == '"' -> { - i = skipJsonString(i) ?: return false - expect = COLON - } - else -> return false - } - COLON -> { - if (c != ':') return false - i++ - expect = VALUE - } - SEPARATOR_OR_END -> { - val container = open.last() - when (c) { - ',' -> expect = if (container == '{') KEY else VALUE - '}', ']' -> { - if (c != if (container == '{') '}' else ']') return false - open.setLength(open.length - 1) - if (open.isEmpty()) return skipJsonWhitespace(i + 1) == length - } - else -> return false - } - i++ - } - } - } -} - -private fun String.skipJsonWhitespace(from: Int): Int { - var i = from - while (i < length && this[i] in " \t\n\r") i++ - return i -} - -// The index past a string, number or literal starting at [from], or null. -private fun String.skipJsonScalar(from: Int): Int? = when (this[from]) { - '"' -> skipJsonString(from) - 't' -> skipJsonLiteral(from, "true") - 'f' -> skipJsonLiteral(from, "false") - 'n' -> skipJsonLiteral(from, "null") - else -> skipJsonNumber(from) -} - -private fun String.skipJsonLiteral(from: Int, literal: String): Int? = - if (startsWith(literal, from)) from + literal.length else null - -private fun String.skipJsonString(from: Int): Int? { - var i = from + 1 - while (i < length) { - val c = this[i] - when { - c == '"' -> return i + 1 - c < ' ' -> return null - c == '\\' -> { - val escaped = getOrNull(i + 1) ?: return null - i += when (escaped) { - in "\"\\/bfnrt" -> 2 - 'u' -> if ( - (i + 2..i + 5).all { getOrNull(it)?.digitToIntOrNull(16) != null } - ) 6 else return null - else -> return null - } - } - else -> i++ - } - } - return null -} - -// `-? (0 | [1-9][0-9]*) (. [0-9]+)? ([eE] [+-]? [0-9]+)?` -private fun String.skipJsonNumber(from: Int): Int? { - var i = from - if (getOrNull(i) == '-') i++ - when (getOrNull(i)) { - '0' -> i++ - in '1'..'9' -> i = skipDigits(i) - else -> return null - } - if (getOrNull(i) == '.') { - if (getOrNull(i + 1) !in '0'..'9') return null - i = skipDigits(i + 1) - } - if (getOrNull(i) == 'e' || getOrNull(i) == 'E') { - i++ - if (getOrNull(i) == '+' || getOrNull(i) == '-') i++ - if (getOrNull(i) !in '0'..'9') return null - i = skipDigits(i) - } - return i -} - -private fun String.skipDigits(from: Int): Int { - var i = from - while (getOrNull(i) in '0'..'9') i++ - return i -} diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index a0fa571..10a2278 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -135,9 +135,11 @@ import kotlinx.coroutines.flow.Flow * directives, platform tile metadata — see [isNoiseMetaName]) are dropped so * they don't inflate the frontmatter, and so are a blank value and * application state that single-page apps ship in `<meta>` (serialised JSON, - * framework config blobs — see [isApplicationStateMeta]). Those are the - * values that do not survive a [wrapInHtmlDocument] round-trip. When `<head>` - * holds several `<title>`s, the first non-blank one wins, over a + * framework config blobs — see [isApplicationStateMeta]). Meta names are + * ASCII case-insensitive, so of several names differing only in letter case + * the first one (spelling and value) wins. Those discarded values and merged + * names are what does not survive a [wrapInHtmlDocument] round-trip. When + * `<head>` holds several `<title>`s, the first non-blank one wins, over a * `<meta name="title">` (in any letter case) too. If `<head>` is absent or * yields no metadata, no frontmatter mark is emitted. * @@ -170,16 +172,20 @@ public fun Flow<SemanticEvent>.simplifyHtml( svgMode: SvgMode = SvgMode.RESOLVE, ): Flow<SemanticEvent> = transform { + // Keyed by the first spelling of each name; insertion order is the + // front matter order. val metadata = mutableMapOf<String, String>() + // Meta names are ASCII case-insensitive (HTML §4.2.5): the first + // occurrence of a name, in any letter case, wins. + val metadataNames = mutableSetOf<String>() val titleText = StringBuilder() - // Title candidates, resolved once when `<head>` closes: the first - // non-blank `<title>` wins, else the first non-blank `<meta name="title">` - // — unlike `document.title`, which takes the first `<title>` even when - // blank, since a blank title tells the reader nothing. The entry lands - // where the first candidate appeared. - var elementTitle: String? = null - var metaTitle: String? = null - var titlePosition = -1 + // The first non-blank `<title>` wins, over a `<meta name="title">` too — + // unlike `document.title`, which takes the first `<title>` even when + // blank, since a blank title tells the reader nothing. Overwriting a meta + // title keeps the entry where the first candidate appeared. Blank is + // judged as `ensureFrontmatterTitle` judges it (NBSP included), so a + // title kept here is never replaced there. + var titleFromElement = false // Attribute map kept on a preserved element: its own [names] whitelist, the // ARIA name/state keep-set, and any caller-requested [keepAttributes]. An @@ -252,7 +258,10 @@ public fun Flow<SemanticEvent>.simplifyHtml( // --- metadata extraction (explicit per-tag) ------------------------- match("html") { event -> - event["lang"]?.let { metadata["lang"] = it } + event["lang"]?.let { + metadata["lang"] = it + metadataNames += "lang" + } children() } @@ -261,11 +270,9 @@ public fun Flow<SemanticEvent>.simplifyHtml( // swallowed (see the mode-scoped matchText below); emit no mark. children(mode = "head") afterClose { - val entries = metadata.toList().toMutableList() - (elementTitle ?: metaTitle)?.let { entries.add(titlePosition, "title" to it) } - if (entries.isNotEmpty()) { + if (metadata.isNotEmpty()) { "frontmatter" { - for ((key, value) in entries) { + for ((key, value) in metadata) { "entry"("key" to key) { +value } } } @@ -281,9 +288,10 @@ public fun Flow<SemanticEvent>.simplifyHtml( children(mode = "titleText") afterClose { val trimmed = titleText.toString().trim { it.isHtmlWhitespace() } - if (trimmed.isNotEmpty() && elementTitle == null) { - elementTitle = trimmed - if (titlePosition < 0) titlePosition = metadata.size + if (trimmed.isNotBlank() && !titleFromElement) { + metadata["title"] = trimmed + metadataNames += "title" + titleFromElement = true } } } @@ -296,13 +304,13 @@ public fun Flow<SemanticEvent>.simplifyHtml( && !isNoiseMetaName(name) && !isApplicationStateMeta(content) ) { - // Meta names are ASCII case-insensitive (HTML §4.2.5). - if (name.equals("title", ignoreCase = true)) { - if (metaTitle == null) { - metaTitle = content - if (titlePosition < 0) titlePosition = metadata.size + val normalizedName = name.asciiLowercase() + if (normalizedName == "title") { + val trimmed = content.trim { it.isHtmlWhitespace() } + if (trimmed.isNotBlank() && metadataNames.add("title")) { + metadata["title"] = trimmed } - } else { + } else if (metadataNames.add(normalizedName)) { metadata[name] = content } } @@ -709,13 +717,22 @@ private val ARIA_KEEP = arrayOf( "aria-modal", ) +// HTML's "ASCII lowercase": unlike [String.lowercase] or +// `equals(ignoreCase = true)`, it folds only `A`–`Z`, so `tıtle` (dotless ı) +// does not match `title`. +private fun String.asciiLowercase(): String = + if (none { it in 'A'..'Z' }) this else String(CharArray(length) { + val c = this[it] + if (c in 'A'..'Z') c + ('a' - 'A') else c + }) + // Technical `<meta name>` values that carry no content signal for an LLM and // only inflate the frontmatter: rendering hints, crawler / verification // directives, and platform tile metadata. Dropped from the extracted metadata. // A denylist (rather than an allowlist) keeps unknown-but-possibly-useful names // — `description`, `keywords`, `author`, `og:*`, `article:*`, … — by default. private fun isNoiseMetaName(name: String): Boolean { - val n = name.lowercase() + val n = name.asciiLowercase() return n in NOISE_META_NAMES || NOISE_META_PREFIXES.any { n.startsWith(it) } || n.endsWith("-verification") diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index c126932..291e75c 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -28,7 +28,8 @@ import kotlinx.coroutines.flow.Flow * YAML `---` front matter, holding `entry` marks) feeds the `head` — the * inverse of [simplifyHtml]'s head-to-frontmatter extraction, so the two * round-trip, except for the values [simplifyHtml] discards (a blank value, - * application state such as a JSON object or an over-long blob): + * application state such as a JSON object or an over-long blob) and for keys + * differing only in letter case, which HTML reads as one `<meta>` name: * * - the `title` entry becomes `<title>` * - the `lang` entry becomes the `lang` attribute on `<html>` diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index b16853e..80640d9 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -846,7 +846,7 @@ class SimplifyHtmlTest { "html" { "head" { "meta"("name" to "__init", "content" to "{\"a\":1}") { } - "meta"("name" to "storage-inventory", "content" to " [1,2,3]") { } + "meta"("name" to "storage-inventory", "content" to " [[1,2],[3]]") { } "meta"("name" to "como-err", "content" to "[{\"x\":1}]") { } "meta"("name" to "empty-list", "content" to "[]") { } "meta"("name" to "feed/config/environment", "content" to "%7B%22x%22%3A1%7D") { } @@ -864,7 +864,8 @@ class SimplifyHtmlTest { // when val output = input.simplifyHtml() - // then — only a value that *is* a JSON object/array marks state + // then — only a value that *is* a JSON object, or an array that is + // not a flat list of words or numbers, marks state output sameAs semanticEvents { "frontmatter" { "entry"("key" to "description") { +"Save {50%} today" } @@ -1108,7 +1109,7 @@ class SimplifyHtmlTest { val input = semanticEvents(tagged = true) { "html" { "head" { - "title" { +"  Page \n" } + "title" { +" \u00A0Page\u00A0\n" } "title" { +"Later" } } "body" { "p" { +"x" } } @@ -1121,7 +1122,174 @@ class SimplifyHtmlTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +" Page " } + "entry"("key" to "title") { +"\u00A0Page\u00A0" } + } + "p" { +"x" } + } + } + + @Test + fun `should keep a flat JSON list of words or numbers in frontmatter`() = runTest { + // given — a list a person writes, not serialised state + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to "[\"Kotlin\",\"Multiplatform\"]") { } + "meta"("name" to "article:tag", "content" to "[\"politics\"]") { } + "meta"("name" to "citation_volume", "content" to "[2024]") { } + "meta"("name" to "chapters", "content" to "[1, 2, 3]") { } + "meta"("name" to "label", "content" to "[PDF]") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "keywords") { +"[\"Kotlin\",\"Multiplatform\"]" } + "entry"("key" to "article:tag") { +"[\"politics\"]" } + "entry"("key" to "citation_volume") { +"[2024]" } + "entry"("key" to "chapters") { +"[1, 2, 3]" } + "entry"("key" to "label") { +"[PDF]" } + } + "p" { +"text" } + } + } + + @Test + fun `should drop a malformed-JSON-looking value only when it parses`() = runTest { + // given — not JSON, so kept as text: trailing commas, crossed brackets + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "a", "content" to "[1,]") { } + "meta"("name" to "b", "content" to "{\"a\":1,}") { } + "meta"("name" to "c", "content" to "[}") { } + "meta"("name" to "d", "content" to "{draft}") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "a") { +"[1,]" } + "entry"("key" to "b") { +"{\"a\":1,}" } + "entry"("key" to "c") { +"[}" } + "entry"("key" to "d") { +"{draft}" } + } + "p" { +"text" } + } + } + + @Test + fun `should skip a title of non-breaking spaces only`() = runTest { + // given — blank as ensureFrontmatterTitle judges it, which would + // otherwise replace it and lose the later title + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"\u00A0\u00A0" } + "title" { +"Real Page" } + "meta"("name" to "title", "content" to "\u00A0") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real Page" } + } + "p" { +"x" } + } + } + + @Test + fun `should trim a title meta like a title element`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "title", "content" to " SEO blurb\n") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"SEO blurb" } + } + "p" { +"x" } + } + } + + @Test + fun `should keep the first of meta names differing only in letter case`() = runTest { + // given — meta names are ASCII case-insensitive (HTML §4.2.5) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "Description", "content" to "First") { } + "meta"("name" to "author", "content" to "Alice") { } + "meta"("name" to "description", "content" to "Second") { } + "meta"("name" to "AUTHOR", "content" to "Bob") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Description") { +"First" } + "entry"("key" to "author") { +"Alice" } + } + "p" { +"x" } + } + } + + @Test + fun `should fold only ASCII letter case in meta names`() = runTest { + // given — a dotless ı is not an i, whatever Unicode case folding says + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"Page" } + "meta"("name" to "t\u0131tle", "content" to "Other") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Page" } + "entry"("key" to "t\u0131tle") { +"Other" } } "p" { +"x" } } diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 4370b07..6fa426c 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -314,6 +314,33 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should merge keys differing only in letter case on a round-trip through simplifyHtml`() = runTest { + // given — HTML reads meta names case-insensitively, so the first + // spelling wins, and any `title` spelling is the title + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Page" } + "entry"("key" to "Title") { +"Subtitle" } + "entry"("key" to "Author") { +"Alice" } + "entry"("key" to "author") { +"Bob" } + } + "p" { +"Hi" } + } + + // when + val output = input.wrapInHtmlDocument().simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Page" } + "entry"("key" to "Author") { +"Alice" } + } + "p" { +"Hi" } + } + } + @Test fun `should use an entry left open when the stream ends inside the frontmatter`() = runTest { // given — a broken upstream contract: neither the entry nor the From ed1727cb33d5d78c1f213e5e7d0cd0a55153ebaf Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 21:37:27 +0200 Subject: [PATCH 04/25] Read <meta> names ASCII case-insensitively everywhere; keep long prose meta values (#82) Move HTML's ASCII lowercase into markanywhere-html-spec (asciiLowercase) and use it in simplifyHtml, wrapInHtmlDocument, ensureFrontmatterTitle and AutolinkCollector instead of private copies. wrapInHtmlDocument and ensureFrontmatterTitle now read front matter keys the way HTML reads <meta> names, so a Title key is the title. isApplicationStateMeta judges a percent-encoded value by what it decodes to, keeps flat arrays (an empty one included), and drops an over-long value only when it does not read as text, so a long prose abstract survives. JSON parse failures of any kind are treated as "not JSON". A blank (NBSP-only included) <html lang> is skipped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 1 - README.md | 3 +- .../api/markanywhere-html-spec.api | 5 + .../src/commonMain/kotlin/AsciiCase.kt | 38 ++++++++ .../src/commonTest/kotlin/AsciiCaseTest.kt | 46 +++++++++ .../commonMain/kotlin/ApplicationStateMeta.kt | 79 +++++++++------ .../kotlin/EnsureFrontmatterTitle.kt | 6 +- .../src/commonMain/kotlin/SimplifyHtml.kt | 84 ++++++++-------- .../commonMain/kotlin/WrapInHtmlDocument.kt | 30 +++--- .../kotlin/EnsureFrontmatterTitleTest.kt | 46 +++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 95 +++++++++++++++++-- .../kotlin/WrapInHtmlDocumentTest.kt | 45 +++++++-- .../commonMain/kotlin/AutolinkCollector.kt | 6 +- 13 files changed, 371 insertions(+), 113 deletions(-) create mode 100644 markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt create mode 100644 markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt diff --git a/CLAUDE.md b/CLAUDE.md index b0c6492..4d197e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -410,7 +410,6 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - `simplifyHtml` drops **technical-noise `<meta name>`** values from the frontmatter via a denylist (`isNoiseMetaName` in `SimplifyHtml.kt`: `viewport`, `generator`, `theme-color`, `robots`, `msapplication-*`, `apple-*`, `*-verification`, …). A denylist (not an allowlist) is deliberate so unknown-but-useful names (`og:*`, `article:*`, custom) survive; extend the denylist as new noise names appear. Value-shaped noise (single-page-app state, issue #82) belongs in `isApplicationStateMeta`, not in the name list. - It judges JSON with kotlinx's `Json.parseToJsonElement`, which is **not a strict validator** even with the default (non-lenient) `Json`: an unquoted token in value position parses as a non-string literal (`[PDF]` is an array, `{"a":abc}` an object), and numbers are never checked — so decide on the element's *shape*, never on "it parsed, so it is JSON". `renderDumpMarkdown` (Gradle task, `RenderDumpMarkdown.kt` in jvmTest) runs the full `transformHtmlToMarkdown` pipeline over every dump to `build/renderedMarkdown/<name>.md` — the canonical way to regenerate the per-dump golden strings (`dumps/OpenjurTest`, `dumps/HackerNewsTest`) after a pipeline change. - A `Regex` used with `matches()` in `commonMain` must be **explicitly anchored** (`^(?:a|b)$`) when it contains a top-level alternation: Kotlin/JS resolves `matches` through the leftmost `find`, so the first branch wins on a *prefix* and the whole-input check fails (`0x1F` matched `[-+]?[0-9]+|0x[0-9a-fA-F]+` as `0`; a full timestamp matched the date-only branch) — JVM backtracks across the branches and never shows it. The YAML scalar typing in `markanywhere-yaml` hit exactly this: green on `jvmTest`, red on `jsBrowserTest`. Dev builds run the JS tests for `yaml`/`parse`/`render`/`html`, so a JVM-only test run is not enough evidence for a regex change. diff --git a/README.md b/README.md index 7428e08..1cf8651 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,8 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document -`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader (a blank value, and application state such as a JSON object or an over-long blob), and for keys differing only in letter case, which HTML reads as one `<meta>` name, so the first one wins (a `Title` key is read as the title). +`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader (a blank value, and application state such as a JSON object or an opaque over-long blob). +Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the later one wins, as for an exact duplicate. ```kotlin val document = """ diff --git a/markanywhere-html-spec/api/markanywhere-html-spec.api b/markanywhere-html-spec/api/markanywhere-html-spec.api index b778a16..cb2eb51 100644 --- a/markanywhere-html-spec/api/markanywhere-html-spec.api +++ b/markanywhere-html-spec/api/markanywhere-html-spec.api @@ -1,3 +1,8 @@ +public final class com/xemantic/markanywhere/html/spec/AsciiCaseKt { + public static final fun asciiLowercase (C)C + public static final fun asciiLowercase (Ljava/lang/String;)Ljava/lang/String; +} + public final class com/xemantic/markanywhere/html/spec/HtmlElementsKt { public static final fun getHTML_RAW_TEXT_ELEMENTS ()Ljava/util/Set; public static final fun getHTML_VOID_ELEMENTS ()Ljava/util/Set; diff --git a/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt new file mode 100644 index 0000000..cdb9b49 --- /dev/null +++ b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.xemantic.markanywhere.html.spec + +/** + * This character with an ASCII upper alpha (`A`–`Z`) replaced by its lowercase + * counterpart — the WHATWG Infra "ASCII lowercase", the folding HTML applies + * wherever it matches names "ASCII case-insensitively" (element names, + * `<meta name>`, URL schemes). + * + * Deliberately narrower than [Char.lowercaseChar]: every other character, + * including non-ASCII letters, is returned unchanged. + */ +public fun Char.asciiLowercase(): Char = + if (this in 'A'..'Z') this + ('a' - 'A') else this + +/** + * This string with every ASCII upper alpha replaced by its lowercase + * counterpart (see [Char.asciiLowercase]) — unlike [String.lowercase], `tıtle` + * (a dotless ı) stays distinct from `title`. + */ +public fun String.asciiLowercase(): String = + if (none { it in 'A'..'Z' }) this else CharArray(length) { this[it].asciiLowercase() }.concatToString() diff --git a/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt new file mode 100644 index 0000000..1634f59 --- /dev/null +++ b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.xemantic.markanywhere.html.spec + +import com.xemantic.kotlin.test.assert +import kotlin.test.Test + +class AsciiCaseTest { + + @Test + fun `should lowercase ASCII upper alphas only`() { + // when + val lowered = "Og:TITLE-1_x".asciiLowercase() + + // then + assert(lowered == "og:title-1_x") + } + + @Test + fun `should leave non-ASCII letters unchanged`() { + // when — Unicode case folding would map these + val dotless = "TıTLE".asciiLowercase() + val umlaut = "ÄRGER".asciiLowercase() + val kelvin = 'K'.asciiLowercase() + + // then + assert(dotless == "tıtle") + assert(umlaut == "Ärger") + assert(kelvin == 'K') + } +} diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 1dafc63..26cd320 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -17,7 +17,6 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.html.spec.isHtmlWhitespace -import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -29,46 +28,68 @@ import kotlinx.serialization.json.JsonPrimitive // (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded // `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A // name denylist cannot keep up with names private to each site's framework, -// so this judges the *value*, which is what tells metadata apart from state: -// - a value that parses as a JSON object, raw or percent-encoded. Parsing, -// not a look at the first and last char, is what keeps human text that -// merely starts with a bracket (`[Solved] …`, `{Draft} …`, -// `[2024] Annual report [PDF]`); -// - a JSON array that is percent-encoded, or holds anything but strings and -// numbers (an object, a nested array, a flag, a `null`), or nothing at all. -// A flat list of words or numbers is metadata a person writes -// (`keywords`, `article:tag`, `citation_volume`); -// - a value longer than [MAX_META_VALUE_LENGTH] — the backstop for opaque -// blobs of any other shape (base64, hash lists, truncated JSON). +// so this judges the *value*, which is what tells metadata apart from state. +// A percent-encoded value is judged by what it decodes to, so the verdict +// never depends on the encoding: +// - a value that parses as a JSON object. Parsing, not a look at the first +// and last char, is what keeps human text that merely starts with a +// bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); +// - a JSON array holding anything but strings and numbers (an object, a +// nested array, a flag, a `null`). A flat list of words or numbers — +// an empty one included — is metadata a person writes (`keywords`, +// `article:tag`, `citation_volume`); +// - a value longer than [MAX_META_VALUE_LENGTH] that does not read as text — +// the backstop for opaque blobs of any other shape (base64, hash lists, +// truncated JSON), which a long abstract in prose is not. internal fun isApplicationStateMeta(content: String): Boolean { val value = content.trim { it.isHtmlWhitespace() } - if (value.length > MAX_META_VALUE_LENGTH) return true - val first = value.firstOrNull() - return when { - first == '{' || first == '[' -> when (val json = value.parseJsonOrNull()) { - is JsonObject -> true - is JsonArray -> json.isEmpty() || json.any { !it.isWordOrNumber() } - else -> false - } - first == '%' -> value.percentDecodedOrNull()?.parseJsonOrNull() - .let { it is JsonObject || it is JsonArray } - else -> false - } + if (value.length > MAX_META_VALUE_LENGTH && !value.readsAsText()) return true + return value.isJsonState() || value.firstOrNull() == '%' && + value.percentDecodedOrNull()?.trim { it.isHtmlWhitespace() }?.isJsonState() == true } // Real metadata is short: `description` / `og:description` rarely exceed 300 -// characters, and the cap still fits a full academic abstract -// (`citation_abstract`, `dc.description`). +// characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 +private fun String.isJsonState(): Boolean { + val first = firstOrNull() + if (first != '{' && first != '[') return false + return when (val json = parseJsonOrNull()) { + is JsonObject -> true + is JsonArray -> json.any { !it.isWordOrNumber() } + else -> false + } +} + +// Prose breaks into words: at least one char in ten is whitespace, or a +// letter of a script written without spaces (anything past ASCII — base64 +// and hex never contain one), while the punctuation of serialised data +// (quotes, brackets, `=`, `;`, `|`, `\`) stays rare. +private fun String.readsAsText(): Boolean { + var wordBreaks = 0 + var dataPunctuation = 0 + for (c in this) { + if (c.isWhitespace() || c.code > 0x7F && c.isLetter()) wordBreaks++ + else if (c in DATA_PUNCTUATION) dataPunctuation++ + } + return wordBreaks * 10 >= length && dataPunctuation * 20 < length +} + +private const val DATA_PUNCTUATION = "{}[]\"<>=;|\\" + +// Never throws: the value is page-controlled, and a malformed one is simply +// not JSON, whatever the parser reports it with. private fun String.parseJsonOrNull(): JsonElement? = try { Json.parseToJsonElement(this) -} catch (_: SerializationException) { +} catch (_: Exception) { null } -// kotlinx's tree reader takes any unquoted token as a literal (`[PDF]` parses -// as an array), so everything that is not a string, a flag or `null` counts — +// kotlinx's tree reader is no strict validator: it takes any unquoted token +// as a literal (`[PDF]` parses as an array, `{"a":abc}` as an object) and +// never checks a number, so the verdict rests on the element's shape, never +// on "it parsed". Everything that is not a string, a flag or `null` counts — // a number, or a word that no JSON writer would have produced. private fun JsonElement.isWordOrNumber(): Boolean = this is JsonPrimitive && this !is JsonNull && (isString || content != "true" && content != "false") diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 3276735..9f453fe 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -18,6 +18,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents +import com.xemantic.markanywhere.html.spec.asciiLowercase import kotlinx.coroutines.flow.Flow /** @@ -25,7 +26,8 @@ import kotlinx.coroutines.flow.Flow * deriving a missing title from the first `h1`. * * A stream whose leading `frontmatter` already holds a usable top-level - * `entry` with `key="title"` — one with non-blank text, or one holding a + * `entry` with `key="title"` (in any ASCII letter case, as + * [wrapInHtmlDocument] reads it) — one with non-blank text, or one holding a * nested structure — passes through untouched. Otherwise the frontmatter is * held back and the title is derived from the very first `h1` following it * (only blank text may intervene): the `h1` subtree's flattened text — @@ -161,7 +163,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterDepth++ if (frontmatterDepth == 2) { // a top-level entry (a direct child) - inTitleEntry = event.name == "entry" && event["key"] == "title" + inTitleEntry = event.name == "entry" && event["key"]?.asciiLowercase() == "title" if (inTitleEntry) { titleHasChildren = false titleText.clear() diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 10a2278..c511368 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -18,7 +18,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations -import com.xemantic.markanywhere.html.spec.isHtmlBlank +import com.xemantic.markanywhere.html.spec.asciiLowercase import com.xemantic.markanywhere.html.spec.isHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform @@ -133,7 +133,8 @@ import kotlinx.coroutines.flow.Flow * matter vocabulary — just before `<body>` content streams through. Technical meta * names that carry no content signal (rendering hints, crawler / verification * directives, platform tile metadata — see [isNoiseMetaName]) are dropped so - * they don't inflate the frontmatter, and so are a blank value and + * they don't inflate the frontmatter, and so are a blank value (NBSP-only + * included, `<html lang>` too) and * application state that single-page apps ship in `<meta>` (serialised JSON, * framework config blobs — see [isApplicationStateMeta]). Meta names are * ASCII case-insensitive, so of several names differing only in letter case @@ -172,19 +173,16 @@ public fun Flow<SemanticEvent>.simplifyHtml( svgMode: SvgMode = SvgMode.RESOLVE, ): Flow<SemanticEvent> = transform { - // Keyed by the first spelling of each name; insertion order is the - // front matter order. - val metadata = mutableMapOf<String, String>() - // Meta names are ASCII case-insensitive (HTML §4.2.5): the first - // occurrence of a name, in any letter case, wins. - val metadataNames = mutableSetOf<String>() + // Meta names are ASCII case-insensitive (HTML §4.2.5), so entries are + // keyed by the folded name, holding the first spelling and its value: + // the first occurrence of a name, in any letter case, wins. Insertion + // order is the front matter order. + val metadata = mutableMapOf<String, MetadataEntry>() val titleText = StringBuilder() // The first non-blank `<title>` wins, over a `<meta name="title">` too — // unlike `document.title`, which takes the first `<title>` even when // blank, since a blank title tells the reader nothing. Overwriting a meta - // title keeps the entry where the first candidate appeared. Blank is - // judged as `ensureFrontmatterTitle` judges it (NBSP included), so a - // title kept here is never replaced there. + // title keeps the entry where the first candidate appeared. var titleFromElement = false // Attribute map kept on a preserved element: its own [names] whitelist, the @@ -259,8 +257,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("html") { event -> event["lang"]?.let { - metadata["lang"] = it - metadataNames += "lang" + if (!it.isBlankMetadata()) metadata["lang"] = MetadataEntry("lang", it) } children() } @@ -272,7 +269,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( afterClose { if (metadata.isNotEmpty()) { "frontmatter" { - for ((key, value) in metadata) { + for ((key, value) in metadata.values) { "entry"("key" to key) { +value } } } @@ -287,10 +284,8 @@ public fun Flow<SemanticEvent>.simplifyHtml( titleText.clear() children(mode = "titleText") afterClose { - val trimmed = titleText.toString().trim { it.isHtmlWhitespace() } - if (trimmed.isNotBlank() && !titleFromElement) { - metadata["title"] = trimmed - metadataNames += "title" + if (!titleFromElement && !titleText.isBlankMetadata()) { + metadata["title"] = MetadataEntry("title", titleText.trim { it.isHtmlWhitespace() }.toString()) titleFromElement = true } } @@ -299,19 +294,19 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("meta") { event -> val name = event["name"] val content = event["content"] - if (name != null && content != null - && !content.isHtmlBlank() - && !isNoiseMetaName(name) - && !isApplicationStateMeta(content) - ) { + if (name != null && content != null && !content.isBlankMetadata()) { + // cheapest checks first: the JSON parse runs only for a name + // that would otherwise be kept val normalizedName = name.asciiLowercase() - if (normalizedName == "title") { - val trimmed = content.trim { it.isHtmlWhitespace() } - if (trimmed.isNotBlank() && metadataNames.add("title")) { - metadata["title"] = trimmed + if (normalizedName !in metadata + && !isNoiseMetaName(normalizedName) + && !isApplicationStateMeta(content) + ) { + metadata[normalizedName] = if (normalizedName == "title") { + MetadataEntry("title", content.trim { it.isHtmlWhitespace() }) + } else { + MetadataEntry(name, content) } - } else if (metadataNames.add(normalizedName)) { - metadata[name] = content } } } @@ -717,28 +712,27 @@ private val ARIA_KEEP = arrayOf( "aria-modal", ) -// HTML's "ASCII lowercase": unlike [String.lowercase] or -// `equals(ignoreCase = true)`, it folds only `A`–`Z`, so `tıtle` (dotless ı) -// does not match `title`. -private fun String.asciiLowercase(): String = - if (none { it in 'A'..'Z' }) this else String(CharArray(length) { - val c = this[it] - if (c in 'A'..'Z') c + ('a' - 'A') else c - }) +// A front matter entry: the name as first spelled, and its value. +private data class MetadataEntry(val key: String, val value: String) + +// A metadata value that tells a reader nothing — judged with Unicode +// whitespace (NBSP included), not HTML's: rendered, an NBSP-only value is as +// empty as a blank one. `ensureFrontmatterTitle` judges a title the same way, +// so a title kept here is never replaced there. +private fun CharSequence.isBlankMetadata(): Boolean = isBlank() // Technical `<meta name>` values that carry no content signal for an LLM and // only inflate the frontmatter: rendering hints, crawler / verification // directives, and platform tile metadata. Dropped from the extracted metadata. // A denylist (rather than an allowlist) keeps unknown-but-possibly-useful names // — `description`, `keywords`, `author`, `og:*`, `article:*`, … — by default. -private fun isNoiseMetaName(name: String): Boolean { - val n = name.asciiLowercase() - return n in NOISE_META_NAMES - || NOISE_META_PREFIXES.any { n.startsWith(it) } - || n.endsWith("-verification") - || n.endsWith("-verify") - || n.startsWith("verify-") -} +// Takes the name already ASCII-lowercased. +private fun isNoiseMetaName(normalizedName: String): Boolean = + normalizedName in NOISE_META_NAMES + || NOISE_META_PREFIXES.any { normalizedName.startsWith(it) } + || normalizedName.endsWith("-verification") + || normalizedName.endsWith("-verify") + || normalizedName.startsWith("verify-") private val NOISE_META_NAMES = setOf( "viewport", "referrer", "generator", "theme-color", "color-scheme", diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 291e75c..0700a26 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -18,6 +18,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents +import com.xemantic.markanywhere.html.spec.asciiLowercase import kotlinx.coroutines.flow.Flow /** @@ -28,8 +29,7 @@ import kotlinx.coroutines.flow.Flow * YAML `---` front matter, holding `entry` marks) feeds the `head` — the * inverse of [simplifyHtml]'s head-to-frontmatter extraction, so the two * round-trip, except for the values [simplifyHtml] discards (a blank value, - * application state such as a JSON object or an over-long blob) and for keys - * differing only in letter case, which HTML reads as one `<meta>` name: + * application state such as a JSON object or an opaque over-long blob): * * - the `title` entry becomes `<title>` * - the `lang` entry becomes the `lang` attribute on `<html>` @@ -37,7 +37,10 @@ import kotlinx.coroutines.flow.Flow * * Only top-level scalar entries are interpreted — exactly the shape * [simplifyHtml] produces. A nested mapping or sequence, a null value, and - * verbatim text are skipped (never an error); a later duplicate key wins. + * verbatim text are skipped (never an error). Keys are read the way HTML + * reads `<meta>` names, ASCII case-insensitively — a `Title` entry is the + * title — and a later duplicate key, in any letter case, wins (value and + * spelling) at the position of the first. * A `frontmatter` mark appearing anywhere past the first event is ordinary * content and flows into `body` verbatim. * @@ -56,7 +59,12 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // the top-level entry being read, null when it is not a scalar to keep var entryKey: String? = null val entryText = StringBuilder() - val metadata = LinkedHashMap<String, String>() + // keyed by the ASCII-lowercased key: the entry's spelling and its value + val metadata = LinkedHashMap<String, Pair<String, String>>() + + fun putEntry(key: String) { + metadata[key.asciiLowercase()] = key to entryText.toString() + } // `head` and its subtree are lexically scoped, so the paired `"name" { }` // builder fits; `html` and `body` close only at end-of-stream, so their @@ -67,18 +75,18 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman mark( "html", attributes = metadata["lang"] - ?.let { mapOf("lang" to it) } + ?.let { (_, lang) -> mapOf("lang" to lang) } ?: emptyMap() ) "head" { - metadata["title"]?.let { title -> + metadata["title"]?.let { (_, title) -> "title" { +title } } - for ((key, value) in metadata) { - if (key == "title" || key == "lang") continue - "meta"("name" to key, "content" to value) {} + for ((normalizedKey, entry) in metadata) { + if (normalizedKey == "title" || normalizedKey == "lang") continue + "meta"("name" to entry.first, "content" to entry.second) {} } } mark("body") @@ -104,7 +112,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman openDocument() } else { if (depth == 1) { - entryKey?.let { metadata[it] = entryText.toString() } + entryKey?.let { putEntry(it) } entryKey = null } depth-- @@ -126,7 +134,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // still used, including an entry left open — its text is complete by // then; an empty stream yields the bare skeleton. if (collectingFrontmatter && depth == 1) { - entryKey?.let { metadata[it] = entryText.toString() } + entryKey?.let { putEntry(it) } } if (!opened) openDocument() unmark("body") diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 967a81c..3419e60 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -532,6 +532,52 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should pass through a title entry in any letter case`() = runTest { + // given — wrapInHtmlDocument reads a `Title` key as the title + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { +"My Page" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { +"My Page" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should replace a blank title entry in any letter case with the derived title`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "TITLE") { +" " } + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Hello" } + } + } + @Test fun `should keep a title entry holding a nested structure`() = runTest { // given — not a usable title, but replacing it would lose content diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 80640d9..b5704ea 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -848,10 +848,10 @@ class SimplifyHtmlTest { "meta"("name" to "__init", "content" to "{\"a\":1}") { } "meta"("name" to "storage-inventory", "content" to " [[1,2],[3]]") { } "meta"("name" to "como-err", "content" to "[{\"x\":1}]") { } - "meta"("name" to "empty-list", "content" to "[]") { } + "meta"("name" to "empty-object", "content" to "{}") { } "meta"("name" to "feed/config/environment", "content" to "%7B%22x%22%3A1%7D") { } "meta"("name" to "jam/config/environment", "content" to "%7b%7d") { } - "meta"("name" to "hash-list", "content" to "%5B%22a%22%5D") { } + "meta"("name" to "encoded-flags", "content" to "%5Btrue%5D") { } "meta"("name" to "flags", "content" to "[true,false]") { } "meta"("name" to "slots", "content" to "[null]") { } "meta"("name" to "spaced", "content" to "%5B%20%7B%22a%22%3A1%7D%20%5D") { } @@ -865,7 +865,8 @@ class SimplifyHtmlTest { val output = input.simplifyHtml() // then — only a value that *is* a JSON object, or an array that is - // not a flat list of words or numbers, marks state + // not a flat list of words or numbers, raw or percent-encoded, marks + // state output sameAs semanticEvents { "frontmatter" { "entry"("key" to "description") { +"Save {50%} today" } @@ -912,13 +913,20 @@ class SimplifyHtmlTest { } @Test - fun `should drop meta values longer than the cap from frontmatter`() = runTest { - // given — the cap still fits a long academic abstract + fun `should drop meta values longer than the cap unless they read as text`() = runTest { + // given — past the cap only prose survives: a long abstract, in a + // script written with spaces or without + val abstract = "Background: we study the effect of X on Y. ".repeat(110) + val cjkAbstract = "本研究では大規模言語モデルの挙動を分析した。".repeat(200) val input = semanticEvents(tagged = true) { "html" { "head" { "meta"("name" to "blob", "content" to "x".repeat(4097)) { } - "meta"("name" to "citation_abstract", "content" to "y".repeat(4096)) { } + "meta"("name" to "hashes", "content" to "0123456789abcdef, ".repeat(300)) { } + "meta"("name" to "truncated-json", "content" to "{\"id\": 1, \"tags\": [\"a\", \"b\"], ".repeat(150)) { } + "meta"("name" to "short-blob", "content" to "y".repeat(4096)) { } + "meta"("name" to "citation_abstract", "content" to abstract) { } + "meta"("name" to "dc.description", "content" to cjkAbstract) { } } "body" { "p" { +"text" } } } @@ -930,7 +938,44 @@ class SimplifyHtmlTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "citation_abstract") { +"y".repeat(4096) } + "entry"("key" to "short-blob") { +"y".repeat(4096) } + "entry"("key" to "citation_abstract") { +abstract } + "entry"("key" to "dc.description") { +cjkAbstract } + } + "p" { +"text" } + } + } + + @Test + fun `should keep malformed JSON-looking meta values as text without throwing`() = runTest { + // given — page-controlled input the JSON parser rejects in any way + val malformed = listOf( + "[\"\\u12\"]", + "{\"a\":\"\\", + "[1e99999999999999999999]", + "[".repeat(2000) + "]".repeat(1999), + "{\"a\":\"\u0000\"", + ) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + malformed.forEachIndexed { i, value -> + "meta"("name" to "m$i", "content" to value) { } + } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + malformed.forEachIndexed { i, value -> + "entry"("key" to "m$i") { +value } + } } "p" { +"text" } } @@ -938,7 +983,7 @@ class SimplifyHtmlTest { @Test fun `should drop blank meta values from frontmatter`() = runTest { - // given — NBSP is HTML content, not whitespace + // given — an NBSP-only value tells a reader as little as a blank one val input = semanticEvents(tagged = true) { "html" { "head" { @@ -961,7 +1006,6 @@ class SimplifyHtmlTest { // then — ordinary metadata survives, a `/` in the name included output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "separator") { +"\u00A0" } "entry"("key" to "section/topic") { +"Politics" } "entry"("key" to "description") { +"A doc" } "entry"("key" to "og:title") { +"Hello" } @@ -972,6 +1016,30 @@ class SimplifyHtmlTest { } } + @Test + fun `should skip a blank html lang in favour of a lang meta`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html"("lang" to " ") { + "head" { + "meta"("name" to "lang", "content" to "de") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"de" } + } + "p" { +"x" } + } + } + @Test fun `should keep the first non-blank title when head holds several`() = runTest { // given — e.g. after a client-side navigation (issue #82) @@ -1130,7 +1198,8 @@ class SimplifyHtmlTest { @Test fun `should keep a flat JSON list of words or numbers in frontmatter`() = runTest { - // given — a list a person writes, not serialised state + // given — a list a person writes, not serialised state — an empty + // one, or one percent-encoded, included val input = semanticEvents(tagged = true) { "html" { "head" { @@ -1139,6 +1208,9 @@ class SimplifyHtmlTest { "meta"("name" to "citation_volume", "content" to "[2024]") { } "meta"("name" to "chapters", "content" to "[1, 2, 3]") { } "meta"("name" to "label", "content" to "[PDF]") { } + "meta"("name" to "none", "content" to "[]") { } + "meta"("name" to "none-spaced", "content" to "[ ]") { } + "meta"("name" to "encoded", "content" to "%5B%22a%22%5D") { } } "body" { "p" { +"text" } } } @@ -1155,6 +1227,9 @@ class SimplifyHtmlTest { "entry"("key" to "citation_volume") { +"[2024]" } "entry"("key" to "chapters") { +"[1, 2, 3]" } "entry"("key" to "label") { +"[PDF]" } + "entry"("key" to "none") { +"[]" } + "entry"("key" to "none-spaced") { +"[ ]" } + "entry"("key" to "encoded") { +"%5B%22a%22%5D" } } "p" { +"text" } } diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 6fa426c..60713ba 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -315,29 +315,54 @@ class WrapInHtmlDocumentTest { } @Test - fun `should merge keys differing only in letter case on a round-trip through simplifyHtml`() = runTest { - // given — HTML reads meta names case-insensitively, so the first - // spelling wins, and any `title` spelling is the title + fun `should read keys ASCII case-insensitively like meta names`() = runTest { + // given — a later key differing only in letter case is a duplicate, + // so it wins (spelling and value) at the position of the first val input = semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Page" } - "entry"("key" to "Title") { +"Subtitle" } + "entry"("key" to "Title") { +"My Page" } + "entry"("key" to "LANG") { +"de" } "entry"("key" to "Author") { +"Alice" } + "entry"("key" to "description") { +"A doc" } "entry"("key" to "author") { +"Bob" } } - "p" { +"Hi" } } // when - val output = input.wrapInHtmlDocument().simplifyHtml() + val output = input.wrapInHtmlDocument() // then output sameAs semanticEvents { + "html"("lang" to "de") { + "head" { + "title" { +"My Page" } + "meta"("name" to "author", "content" to "Bob") { } + "meta"("name" to "description", "content" to "A doc") { } + } + "body" { } + } + } + } + + @Test + fun `should keep a title key in any letter case on a round-trip through ensureFrontmatterTitle and simplifyHtml`() = runTest { + // given — the H1 must not displace the `Title` entry + val input = semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Page" } - "entry"("key" to "Author") { +"Alice" } + "entry"("key" to "Title") { +"My Page" } } - "p" { +"Hi" } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument().simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"My Page" } + } + "h1" { +"Heading" } } } diff --git a/markanywhere-parse/src/commonMain/kotlin/AutolinkCollector.kt b/markanywhere-parse/src/commonMain/kotlin/AutolinkCollector.kt index 714d80e..809346b 100644 --- a/markanywhere-parse/src/commonMain/kotlin/AutolinkCollector.kt +++ b/markanywhere-parse/src/commonMain/kotlin/AutolinkCollector.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.parse import com.xemantic.markanywhere.SemanticEvent +import com.xemantic.markanywhere.html.spec.asciiLowercase import com.xemantic.markanywhere.html.spec.isHtmlWhitespace import com.xemantic.markanywhere.parse.AutolinkCollector.Companion.SUPPRESS_NAMES import kotlinx.coroutines.flow.FlowCollector @@ -616,11 +617,8 @@ private fun String.regionMatchesAsciiCi( val a = this[thisOffset + i] val b = other[otherOffset + i] if (a == b) continue - if (a.foldAsciiLower() == b.foldAsciiLower()) continue + if (a.asciiLowercase() == b.asciiLowercase()) continue return false } return true } - -private fun Char.foldAsciiLower(): Char = - if (this in 'A'..'Z') (this.code or 0x20).toChar() else this From ce5d90be914a5113378a5b7643da95dd561dcbbb Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 22:01:53 +0200 Subject: [PATCH 05/25] Share head metadata rules between simplifyHtml and wrapInHtmlDocument (#82) - Extract HeadMetadata: first of case-variant duplicate keys wins in both simplifyHtml and wrapInHtmlDocument (wrapInHtmlDocument used last-wins). - ensureFrontmatterTitle judges only the first title entry in any case, so a usable title can no longer hide a blank variant that blanks <title>. - Strip and collapse <title> whitespace as document.title does (stripAndCollapseHtmlWhitespace in markanywhere-html-spec). - Spell a lang meta as `lang`, yielding to <html lang>. - Keep long comma/semicolon-separated keyword lists as text; decode percent escapes with ASCII hex digits only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 2 +- README.md | 2 +- markanywhere-html-spec/README.md | 2 + .../api/markanywhere-html-spec.api | 1 + .../src/commonMain/kotlin/AsciiCase.kt | 1 - .../src/commonMain/kotlin/HtmlWhitespace.kt | 19 +++ .../commonTest/kotlin/HtmlWhitespaceTest.kt | 17 ++- markanywhere-html/build.gradle.kts | 1 + .../commonMain/kotlin/ApplicationStateMeta.kt | 31 ++-- .../kotlin/EnsureFrontmatterTitle.kt | 15 +- .../src/commonMain/kotlin/HeadMetadata.kt | 53 +++++++ .../src/commonMain/kotlin/SimplifyHtml.kt | 46 +++--- .../commonMain/kotlin/WrapInHtmlDocument.kt | 30 ++-- .../kotlin/EnsureFrontmatterTitleTest.kt | 49 +++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 135 ++++++++++++++++++ .../kotlin/WrapInHtmlDocumentTest.kt | 64 ++++++++- .../kotlin/dumps/W3cValidatorNoRefsTest.kt | 2 +- .../kotlin/dumps/W3cValidatorRefsTest.kt | 2 +- 18 files changed, 408 insertions(+), 64 deletions(-) create mode 100644 markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt diff --git a/CLAUDE.md b/CLAUDE.md index 4d197e0..7cbf0b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,7 +560,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. - The two must agree on what a *usable* title is, because `wrapInHtmlDocument` skips a `type=null` entry and emits an empty `<title>` for a blank one: `ensureFrontmatterTitle` therefore treats a top-level `title` with blank text or `type=null` (a bare `title:` line) as missing and **replaces it in place** with the derived one (a `title` holding nested marks is left alone — replacing it would lose content), and it holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. + The two must agree on what a *usable* title is, because `wrapInHtmlDocument` skips a `type=null` entry and emits an empty `<title>` for a blank one: `ensureFrontmatterTitle` therefore treats a top-level `title` with blank text or `type=null` (a bare `title:` line) as missing and **replaces it in place** with the derived one (a `title` holding nested marks is left alone — replacing it would lose content), it judges **only the first** top-level title entry in any ASCII letter case, because `wrapInHtmlDocument` (like `simplifyHtml`, via the shared `HeadMetadata`) lets the first of case-variant duplicate keys win — judging any later variant let a usable `title` hide a blank `Title` that then blanked the `<title>`; and it holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. ## Test conventions diff --git a/README.md b/README.md index 1cf8651..c8210bb 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document `wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader (a blank value, and application state such as a JSON object or an opaque over-long blob). -Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the later one wins, as for an exact duplicate. +Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the first one wins, as for an exact duplicate — the same rule `simplifyHtml` applies to `<meta>` names. ```kotlin val document = """ diff --git a/markanywhere-html-spec/README.md b/markanywhere-html-spec/README.md index 36075d2..7c4d66a 100644 --- a/markanywhere-html-spec/README.md +++ b/markanywhere-html-spec/README.md @@ -11,6 +11,8 @@ so code that reads a semantic event stream carrying HTML attributes can use it w |---------------------------------------------------|------------------------------------------------------------------------------------------------| | `HTML_WHITESPACE_CHARS`, `Char.isHtmlWhitespace()` | HTML "ASCII whitespace": TAB, LF, FF, CR, SPACE — not NBSP, which HTML treats as content | | `String.isHtmlBlank()` | Empty or HTML whitespace only | +| `String.stripAndCollapseHtmlWhitespace()` | Trimmed of HTML whitespace, inner runs collapsed to one space, as `document.title` reads it | +| `Char.asciiLowercase()`, `String.asciiLowercase()` | ASCII lowercase: only `A`–`Z` fold, as HTML compares names "ASCII case-insensitively" | | `HTML_VOID_ELEMENTS` | Elements with no content and no closing tag, including the obsolete `keygen` and `param` | | `HTML_RAW_TEXT_ELEMENTS` | `script` and `style`, whose content is neither escaped nor parsed as markup | | `SemanticEvent.Mark.classList` | The distinct class names of a mark's `class` attribute, like the DOM's `classList` | diff --git a/markanywhere-html-spec/api/markanywhere-html-spec.api b/markanywhere-html-spec/api/markanywhere-html-spec.api index cb2eb51..9cb4fbe 100644 --- a/markanywhere-html-spec/api/markanywhere-html-spec.api +++ b/markanywhere-html-spec/api/markanywhere-html-spec.api @@ -11,6 +11,7 @@ public final class com/xemantic/markanywhere/html/spec/HtmlElementsKt { public final class com/xemantic/markanywhere/html/spec/HtmlWhitespaceKt { public static final fun isHtmlBlank (Ljava/lang/String;)Z public static final fun isHtmlWhitespace (C)Z + public static final fun stripAndCollapseHtmlWhitespace (Ljava/lang/String;)Ljava/lang/String; } public final class com/xemantic/markanywhere/html/spec/MarkClassListKt { diff --git a/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt index cdb9b49..a6cc6d2 100644 --- a/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt +++ b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt @@ -14,7 +14,6 @@ * limitations under the License. */ - package com.xemantic.markanywhere.html.spec /** diff --git a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt index b5b0a80..33a1b97 100644 --- a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt +++ b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt @@ -41,3 +41,22 @@ public fun Char.isHtmlWhitespace(): Boolean = this in HTML_WHITESPACE_CHARS * whitespace. */ public fun String.isHtmlBlank(): Boolean = all { it.isHtmlWhitespace() } + +/** + * This string with leading and trailing [HTML whitespace][isHtmlWhitespace] + * removed and every inner run of it replaced by a single space — the WHATWG + * Infra "strip and collapse ASCII whitespace", the normalisation behind + * `document.title`. NBSP is content and stays. + */ +public fun String.stripAndCollapseHtmlWhitespace(): String = buildString(length) { + var pendingSpace = false + for (c in this@stripAndCollapseHtmlWhitespace) { + if (c.isHtmlWhitespace()) { + pendingSpace = isNotEmpty() + } else { + if (pendingSpace) append(' ') + pendingSpace = false + append(c) + } + } +} diff --git a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt index f50bf3a..c1ee51f 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt @@ -70,4 +70,19 @@ class HtmlWhitespaceTest { assert(!"text".isHtmlBlank()) } -} \ No newline at end of file + @Test + fun `should strip and collapse HTML whitespace`() { + assert("\n Foo\t\r\n Bar \u000C".stripAndCollapseHtmlWhitespace() == "Foo Bar") + } + + @Test + fun `should keep NBSP when stripping and collapsing HTML whitespace`() { + assert("\u00A0 a \u00A0b ".stripAndCollapseHtmlWhitespace() == "\u00A0 a \u00A0b") + } + + @Test + fun `should strip and collapse a blank string to empty`() { + assert(" \t\n".stripAndCollapseHtmlWhitespace() == "") + } + +} diff --git a/markanywhere-html/build.gradle.kts b/markanywhere-html/build.gradle.kts index e59d11d..51664b5 100644 --- a/markanywhere-html/build.gradle.kts +++ b/markanywhere-html/build.gradle.kts @@ -42,6 +42,7 @@ kotlin { implementation(project(":markanywhere-dump")) implementation(project(":markanywhere-html-spec")) api(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) implementation(libs.xemantic.kotlin.core) } } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 26cd320..014fe76 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -62,21 +62,26 @@ private fun String.isJsonState(): Boolean { } } -// Prose breaks into words: at least one char in ten is whitespace, or a -// letter of a script written without spaces (anything past ASCII — base64 -// and hex never contain one), while the punctuation of serialised data -// (quotes, brackets, `=`, `;`, `|`, `\`) stays rare. +// Text is made of words: at least half the chars are letters (hex and +// number lists are mostly digits), and at least one char in sixteen breaks a +// word — whitespace, a list separator (`,` `;`, so `a,b,c` keywords count), +// or a letter of a script written without spaces (anything past ASCII — +// base64 and hex never contain one) — sparse enough for a list of long +// compound words, while the punctuation of serialised data (quotes, +// brackets, `=`, `|`, `\`) stays rare. private fun String.readsAsText(): Boolean { + var letters = 0 var wordBreaks = 0 var dataPunctuation = 0 for (c in this) { - if (c.isWhitespace() || c.code > 0x7F && c.isLetter()) wordBreaks++ + if (c.isLetter()) letters++ + if (c.isWhitespace() || c == ',' || c == ';' || c.code > 0x7F && c.isLetter()) wordBreaks++ else if (c in DATA_PUNCTUATION) dataPunctuation++ } - return wordBreaks * 10 >= length && dataPunctuation * 20 < length + return letters * 2 >= length && wordBreaks * 16 >= length && dataPunctuation * 20 < length } -private const val DATA_PUNCTUATION = "{}[]\"<>=;|\\" +private const val DATA_PUNCTUATION = "{}[]\"<>=|\\" // Never throws: the value is page-controlled, and a malformed one is simply // not JSON, whatever the parser reports it with. @@ -103,8 +108,8 @@ private fun String.percentDecodedOrNull(): String? { while (i < length) { val c = this[i] if (c == '%') { - val high = getOrNull(i + 1)?.digitToIntOrNull(16) ?: return null - val low = getOrNull(i + 2)?.digitToIntOrNull(16) ?: return null + val high = getOrNull(i + 1)?.asciiHexDigitOrNull() ?: return null + val low = getOrNull(i + 2)?.asciiHexDigitOrNull() ?: return null bytes[size++] = (high * 16 + low).toByte() i += 3 } else { @@ -115,3 +120,11 @@ private fun String.percentDecodedOrNull(): String? { } return bytes.decodeToString(0, size) } + +// Unlike [Char.digitToIntOrNull], which also takes non-ASCII Unicode digits. +private fun Char.asciiHexDigitOrNull(): Int? = when (this) { + in '0'..'9' -> this - '0' + in 'a'..'f' -> this - 'a' + 10 + in 'A'..'F' -> this - 'A' + 10 + else -> null +} diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 9f453fe..eda0935 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -26,9 +26,9 @@ import kotlinx.coroutines.flow.Flow * deriving a missing title from the first `h1`. * * A stream whose leading `frontmatter` already holds a usable top-level - * `entry` with `key="title"` (in any ASCII letter case, as - * [wrapInHtmlDocument] reads it) — one with non-blank text, or one holding a - * nested structure — passes through untouched. Otherwise the frontmatter is + * `entry` with `key="title"` (in any ASCII letter case, and only the first + * such entry counts, as [wrapInHtmlDocument] reads it) — one with non-blank + * text, or one holding a nested structure — passes through untouched. Otherwise the frontmatter is * held back and the title is derived from the very first `h1` following it * (only blank text may intervene): the `h1` subtree's flattened text — * its text events plus the `alt` of every `img` mark, in document order, @@ -66,7 +66,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s var titleStart = -1 var titleEnd = -1 // the top-level title entry currently open: its text, and whether it - // holds nested marks (then it is kept as is, whatever its text) + // holds nested marks (then it is kept as is, whatever its text); only + // the first title entry is judged, the one wrapInHtmlDocument reads + var titleSeen = false var inTitleEntry = false var titleHasChildren = false val titleText = StringBuilder() @@ -144,6 +146,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterEvents = mutableListOf(event) frontmatterDepth = 1 hasTitle = false + titleSeen = false titleStart = -1 state = InFrontmatter } @@ -163,8 +166,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterDepth++ if (frontmatterDepth == 2) { // a top-level entry (a direct child) - inTitleEntry = event.name == "entry" && event["key"]?.asciiLowercase() == "title" + inTitleEntry = !titleSeen && event.name == "entry" && + event["key"]?.asciiLowercase() == "title" if (inTitleEntry) { + titleSeen = true titleHasChildren = false titleText.clear() titleStart = events.lastIndex diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt new file mode 100644 index 0000000..9d534b4 --- /dev/null +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xemantic.markanywhere.html + +import com.xemantic.markanywhere.html.spec.asciiLowercase + +// A front matter entry: the name as first spelled, and its value. +internal data class MetadataEntry(val key: String, val value: String) + +// The `<head>` metadata a front matter holds, in insertion order and keyed +// the way HTML reads `<meta>` names: ASCII case-insensitively (HTML §4.2.5). +// The one duplicate policy [simplifyHtml] and [wrapInHtmlDocument] share, so +// the two stay inverses whichever way a document travels: the first +// occurrence of a name, in any letter case, wins — spelling and value. +internal class HeadMetadata { + + private val entries = LinkedHashMap<String, MetadataEntry>() + + val values: Collection<MetadataEntry> get() = entries.values + + fun isNotEmpty(): Boolean = entries.isNotEmpty() + + operator fun contains(name: String): Boolean = name.asciiLowercase() in entries + + operator fun get(name: String): MetadataEntry? = entries[name.asciiLowercase()] + + // Adds the entry unless its name is already present. + fun add(key: String, value: String) { + val name = key.asciiLowercase() + if (name !in entries) entries[name] = MetadataEntry(key, value) + } + + // Sets the entry whether or not its name is present, keeping the position + // of an existing one. + operator fun set(key: String, value: String) { + entries[key.asciiLowercase()] = MetadataEntry(key, value) + } + +} diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index c511368..94872a7 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -20,6 +20,7 @@ import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations import com.xemantic.markanywhere.html.spec.asciiLowercase import com.xemantic.markanywhere.html.spec.isHtmlWhitespace +import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform import kotlinx.coroutines.flow.Flow @@ -138,10 +139,13 @@ import kotlinx.coroutines.flow.Flow * application state that single-page apps ship in `<meta>` (serialised JSON, * framework config blobs — see [isApplicationStateMeta]). Meta names are * ASCII case-insensitive, so of several names differing only in letter case - * the first one (spelling and value) wins. Those discarded values and merged + * the first one (spelling and value) wins, as in [wrapInHtmlDocument]; a + * `lang` meta is spelled `lang` and yields to `<html lang>`, the document's + * actual language. Those discarded values and merged * names are what does not survive a [wrapInHtmlDocument] round-trip. When * `<head>` holds several `<title>`s, the first non-blank one wins, over a - * `<meta name="title">` (in any letter case) too. If `<head>` is absent or + * `<meta name="title">` (in any letter case) too, with its whitespace + * stripped and collapsed as `document.title` does. If `<head>` is absent or * yields no metadata, no frontmatter mark is emitted. * * Matcher registration is grouped: per-tag explicit matchers come first @@ -173,11 +177,11 @@ public fun Flow<SemanticEvent>.simplifyHtml( svgMode: SvgMode = SvgMode.RESOLVE, ): Flow<SemanticEvent> = transform { - // Meta names are ASCII case-insensitive (HTML §4.2.5), so entries are - // keyed by the folded name, holding the first spelling and its value: - // the first occurrence of a name, in any letter case, wins. Insertion - // order is the front matter order. - val metadata = mutableMapOf<String, MetadataEntry>() + // A value that tells a reader nothing is never added — blank judged with + // Unicode whitespace (NBSP included), not HTML's: rendered, an NBSP-only + // value is as empty as a blank one. `ensureFrontmatterTitle` judges a + // title the same way, so a title kept here is never replaced there. + val metadata = HeadMetadata() val titleText = StringBuilder() // The first non-blank `<title>` wins, over a `<meta name="title">` too — // unlike `document.title`, which takes the first `<title>` even when @@ -257,7 +261,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("html") { event -> event["lang"]?.let { - if (!it.isBlankMetadata()) metadata["lang"] = MetadataEntry("lang", it) + if (it.isNotBlank()) metadata.add("lang", it) } children() } @@ -284,8 +288,9 @@ public fun Flow<SemanticEvent>.simplifyHtml( titleText.clear() children(mode = "titleText") afterClose { - if (!titleFromElement && !titleText.isBlankMetadata()) { - metadata["title"] = MetadataEntry("title", titleText.trim { it.isHtmlWhitespace() }.toString()) + if (!titleFromElement && titleText.isNotBlank()) { + // as `document.title` reads it + metadata["title"] = titleText.toString().stripAndCollapseHtmlWhitespace() titleFromElement = true } } @@ -294,7 +299,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("meta") { event -> val name = event["name"] val content = event["content"] - if (name != null && content != null && !content.isBlankMetadata()) { + if (name != null && !content.isNullOrBlank()) { // cheapest checks first: the JSON parse runs only for a name // that would otherwise be kept val normalizedName = name.asciiLowercase() @@ -302,10 +307,12 @@ public fun Flow<SemanticEvent>.simplifyHtml( && !isNoiseMetaName(normalizedName) && !isApplicationStateMeta(content) ) { - metadata[normalizedName] = if (normalizedName == "title") { - MetadataEntry("title", content.trim { it.isHtmlWhitespace() }) - } else { - MetadataEntry(name, content) + when (normalizedName) { + // the two keys wrapInHtmlDocument turns back into <title> + // and <html lang>, spelled as it reads them + "title" -> metadata.add("title", content.trim { it.isHtmlWhitespace() }) + "lang" -> metadata.add("lang", content) + else -> metadata.add(name, content) } } } @@ -712,15 +719,6 @@ private val ARIA_KEEP = arrayOf( "aria-modal", ) -// A front matter entry: the name as first spelled, and its value. -private data class MetadataEntry(val key: String, val value: String) - -// A metadata value that tells a reader nothing — judged with Unicode -// whitespace (NBSP included), not HTML's: rendered, an NBSP-only value is as -// empty as a blank one. `ensureFrontmatterTitle` judges a title the same way, -// so a title kept here is never replaced there. -private fun CharSequence.isBlankMetadata(): Boolean = isBlank() - // Technical `<meta name>` values that carry no content signal for an LLM and // only inflate the frontmatter: rendering hints, crawler / verification // directives, and platform tile metadata. Dropped from the extracted metadata. diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 0700a26..65bbf2f 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -39,8 +39,8 @@ import kotlinx.coroutines.flow.Flow * [simplifyHtml] produces. A nested mapping or sequence, a null value, and * verbatim text are skipped (never an error). Keys are read the way HTML * reads `<meta>` names, ASCII case-insensitively — a `Title` entry is the - * title — and a later duplicate key, in any letter case, wins (value and - * spelling) at the position of the first. + * title — and of duplicate keys, in any letter case, the first one wins + * (spelling and value), as in [simplifyHtml]. * A `frontmatter` mark appearing anywhere past the first event is ordinary * content and flows into `body` verbatim. * @@ -59,12 +59,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // the top-level entry being read, null when it is not a scalar to keep var entryKey: String? = null val entryText = StringBuilder() - // keyed by the ASCII-lowercased key: the entry's spelling and its value - val metadata = LinkedHashMap<String, Pair<String, String>>() - - fun putEntry(key: String) { - metadata[key.asciiLowercase()] = key to entryText.toString() - } + val metadata = HeadMetadata() // `head` and its subtree are lexically scoped, so the paired `"name" { }` // builder fits; `html` and `body` close only at end-of-stream, so their @@ -75,18 +70,18 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman mark( "html", attributes = metadata["lang"] - ?.let { (_, lang) -> mapOf("lang" to lang) } + ?.let { mapOf("lang" to it.value) } ?: emptyMap() ) "head" { - metadata["title"]?.let { (_, title) -> + metadata["title"]?.let { "title" { - +title + +it.value } } - for ((normalizedKey, entry) in metadata) { - if (normalizedKey == "title" || normalizedKey == "lang") continue - "meta"("name" to entry.first, "content" to entry.second) {} + for ((key, value) in metadata.values) { + if (key.asciiLowercase() in HEAD_KEYS) continue + "meta"("name" to key, "content" to value) {} } } mark("body") @@ -112,7 +107,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman openDocument() } else { if (depth == 1) { - entryKey?.let { putEntry(it) } + entryKey?.let { metadata.add(it, entryText.toString()) } entryKey = null } depth-- @@ -134,7 +129,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // still used, including an entry left open — its text is complete by // then; an empty stream yields the bare skeleton. if (collectingFrontmatter && depth == 1) { - entryKey?.let { putEntry(it) } + entryKey?.let { metadata.add(it, entryText.toString()) } } if (!opened) openDocument() unmark("body") @@ -143,4 +138,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // Scalar `type`s whose text is meaningful as a `<meta content>` (`null` and // the empty collections are not). +// The keys that become `<title>` and `<html lang>` rather than a `<meta>`. +private val HEAD_KEYS = setOf("title", "lang") + private val SCALAR_TYPES = setOf("bool", "int", "float", "timestamp") diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 3419e60..3776f2d 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -578,6 +578,55 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should judge only the first title entry in any letter case`() = runTest { + // given — wrapInHtmlDocument takes the first title variant, so a + // later usable one does not make a blank first one usable + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +" " } + "entry"("key" to "Title") { +"Later" } + } + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + "entry"("key" to "Title") { +"Later" } + } + "h1" { +"Hello" } + } + } + + @Test + fun `should pass through a usable first title entry followed by a blank variant`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Foo" } + "entry"("key" to "TITLE") { } + } + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Foo" } + "entry"("key" to "TITLE") { } + } + "h1" { +"Hello" } + } + } + @Test fun `should keep a title entry holding a nested structure`() = runTest { // given — not a usable title, but replacing it would lose content diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index b5704ea..a88babe 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -946,6 +946,66 @@ class SimplifyHtmlTest { } } + @Test + fun `should keep long comma-separated keyword lists`() = runTest { + // given — list separators break words as spaces do, and long compound + // words keep a list sparse in breaks; a list of hashes stays dropped + val compact = (1..600).joinToString(",") { "tag$it" } + val german = List(100) { "Bundesverfassungsgericht, Rechtsprechung" }.joinToString(", ") + val semicolons = List(160) { "Verwaltungsgerichtsbarkeit" }.joinToString("; ") + val hashes = (1..300).joinToString(",") { "0123456789abcdef" } + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to compact) { } + "meta"("name" to "news_keywords", "content" to german) { } + "meta"("name" to "subject", "content" to semicolons) { } + "meta"("name" to "hashes", "content" to hashes) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "keywords") { +compact } + "entry"("key" to "news_keywords") { +german } + "entry"("key" to "subject") { +semicolons } + } + "p" { +"text" } + } + } + + @Test + fun `should not decode a percent escape made of non-ASCII digits`() = runTest { + // given — an Arabic-Indic seven is no hex digit, so this is not a + // percent-encoded JSON object but text + val value = "%\u0667B%22a%22%3A1%7D" + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "config", "content" to value) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "config") { +value } + } + "p" { +"text" } + } + } + @Test fun `should keep malformed JSON-looking meta values as text without throwing`() = runTest { // given — page-controlled input the JSON parser rejects in any way @@ -1040,6 +1100,56 @@ class SimplifyHtmlTest { } } + @Test + fun `should prefer html lang over a lang meta in any letter case`() = runTest { + // given — the document language is the root element's attribute + val input = semanticEvents(tagged = true) { + "html"("lang" to "en") { + "head" { + "meta"("name" to "Lang", "content" to "de") { } + "meta"("name" to "lang", "content" to "fr") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"en" } + } + "p" { +"x" } + } + } + + @Test + fun `should spell a lang meta in any letter case as the lang key`() = runTest { + // given — without <html lang> the meta is the only language there is, + // and the front matter says so in the key wrapInHtmlDocument reads + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "LANG", "content" to "de") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"de" } + } + "p" { +"x" } + } + } + @Test fun `should keep the first non-blank title when head holds several`() = runTest { // given — e.g. after a client-side navigation (issue #82) @@ -1466,6 +1576,31 @@ class SimplifyHtmlTest { } } + @Test + fun `should collapse whitespace inside a pretty-printed title`() = runTest { + // given — like document.title, which strips and collapses ASCII + // whitespace; the NBSP is content and stays + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"\n Foo\n Bar\u00A0\u00A0Baz\n " } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Foo Bar\u00A0\u00A0Baz" } + } + "p" { +"x" } + } + } + @Test fun `should drop marks nested inside title and keep only their text`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 60713ba..cd98abc 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -167,7 +167,7 @@ class WrapInHtmlDocumentTest { } @Test - fun `should let a later duplicate key win`() = runTest { + fun `should let the first of duplicate keys win`() = runTest { // given val input = semanticEvents { "frontmatter" { @@ -183,7 +183,7 @@ class WrapInHtmlDocumentTest { output sameAs semanticEvents { "html" { "head" { - "meta"("name" to "author", "content" to "Bob") { } + "meta"("name" to "author", "content" to "Alice") { } } "body" { } } @@ -317,7 +317,7 @@ class WrapInHtmlDocumentTest { @Test fun `should read keys ASCII case-insensitively like meta names`() = runTest { // given — a later key differing only in letter case is a duplicate, - // so it wins (spelling and value) at the position of the first + // so the first one wins, spelling and value, as in simplifyHtml val input = semanticEvents { "frontmatter" { "entry"("key" to "Title") { +"My Page" } @@ -336,7 +336,7 @@ class WrapInHtmlDocumentTest { "html"("lang" to "de") { "head" { "title" { +"My Page" } - "meta"("name" to "author", "content" to "Bob") { } + "meta"("name" to "Author", "content" to "Alice") { } "meta"("name" to "description", "content" to "A doc") { } } "body" { } @@ -366,6 +366,62 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should agree with ensureFrontmatterTitle when a later title variant is blank`() = runTest { + // given — the usable first title is kept by ensureFrontmatterTitle, + // so the blank variant after it must not blank the <title> + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Foo" } + "entry"("key" to "Title") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Foo" } + } + "body" { + "h1" { +"Heading" } + } + } + } + } + + @Test + fun `should agree with ensureFrontmatterTitle when the first title variant is blank`() = runTest { + // given — the blank first title is the one both judge, so it is + // replaced by the heading rather than shadowed by the later variant + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { } + "entry"("key" to "Title") { +"Foo" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Heading" } + } + "body" { + "h1" { +"Heading" } + } + } + } + } + @Test fun `should use an entry left open when the stream ends inside the frontmatter`() = runTest { // given — a broken upstream contract: neither the entry nor the diff --git a/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorNoRefsTest.kt b/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorNoRefsTest.kt index aac5bf5..33b4e23 100644 --- a/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorNoRefsTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorNoRefsTest.kt @@ -51,7 +51,7 @@ class W3cValidatorNoRefsTest { markdown sameAsMarkdown """ --- lang: en - title: Ready to check - Nu Html Checker + title: Ready to check - Nu Html Checker --- # [Nu Html Checker](.) diff --git a/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorRefsTest.kt b/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorRefsTest.kt index 878b978..9188872 100644 --- a/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorRefsTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/dumps/W3cValidatorRefsTest.kt @@ -53,7 +53,7 @@ class W3cValidatorRefsTest { markdown sameAsMarkdown """ --- lang: en - title: Ready to check - Nu Html Checker + title: Ready to check - Nu Html Checker --- # [Nu Html Checker](ref:1:.) From b4bef682e7c120cf9cc1b4facf427644b2b5c275 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 22:27:23 +0200 Subject: [PATCH 06/25] Align title judging between ensureFrontmatterTitle and wrapInHtmlDocument; tidy head metadata (#82) - ensureFrontmatterTitle judges the first scalar title entry, the one wrapInHtmlDocument reads, instead of the first title entry of any type; a replaced entry keeps its key spelling - strip and collapse a title meta like a <title> element; trim lang - detect JSON state serialised into a JSON string - count code points when judging whether a long value reads as text - share HEAD_KEYS / SCALAR_ENTRY_TYPES via HeadMetadata.kt Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 5 +- .../commonTest/kotlin/HtmlWhitespaceTest.kt | 18 ++- .../commonMain/kotlin/ApplicationStateMeta.kt | 28 +++- .../kotlin/EnsureFrontmatterTitle.kt | 120 ++++++++++----- .../src/commonMain/kotlin/HeadMetadata.kt | 9 ++ .../src/commonMain/kotlin/SimplifyHtml.kt | 17 ++- .../commonMain/kotlin/WrapInHtmlDocument.kt | 9 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 137 +++++++++++++++++- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 107 +++++++++++++- 9 files changed, 386 insertions(+), 64 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7cbf0b7..c2cd46d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,7 +560,10 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. - The two must agree on what a *usable* title is, because `wrapInHtmlDocument` skips a `type=null` entry and emits an empty `<title>` for a blank one: `ensureFrontmatterTitle` therefore treats a top-level `title` with blank text or `type=null` (a bare `title:` line) as missing and **replaces it in place** with the derived one (a `title` holding nested marks is left alone — replacing it would lose content), it judges **only the first** top-level title entry in any ASCII letter case, because `wrapInHtmlDocument` (like `simplifyHtml`, via the shared `HeadMetadata`) lets the first of case-variant duplicate keys win — judging any later variant let a usable `title` hide a blank `Title` that then blanked the `<title>`; and it holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. + The two must agree on **which** title entry is read and what a *usable* one is, because `wrapInHtmlDocument` skips a `type=null`, nested or empty-collection (`type=seq`/`map`) entry — letting a later case variant (`Title:`) be its title — and emits an empty `<title>` for a blank one. + So `ensureFrontmatterTitle` judges only the first top-level **scalar** title entry in any ASCII letter case (the first of case-variant duplicates wins in both, via the shared `HeadMetadata`; `SCALAR_ENTRY_TYPES` is shared too): non-blank passes through, blank is **replaced in place** keeping its key spelling; with no scalar one it replaces the first `type=null` entry (a bare `title:` line), and a title holding nested marks or an empty collection is left alone — replacing it would lose content, and adding a second `title` would duplicate a YAML key. + Judging the first title entry *of any type* once let a leading `title:` hide a later `Title: Real`, which the derived heading then displaced. + It also holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. ## Test conventions diff --git a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt index c1ee51f..43f2e65 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt @@ -72,17 +72,29 @@ class HtmlWhitespaceTest { @Test fun `should strip and collapse HTML whitespace`() { - assert("\n Foo\t\r\n Bar \u000C".stripAndCollapseHtmlWhitespace() == "Foo Bar") + // when + val result = "\n Foo\t\r\n Bar \u000C".stripAndCollapseHtmlWhitespace() + + // then + assert(result == "Foo Bar") } @Test fun `should keep NBSP when stripping and collapsing HTML whitespace`() { - assert("\u00A0 a \u00A0b ".stripAndCollapseHtmlWhitespace() == "\u00A0 a \u00A0b") + // when + val result = "\u00A0 a \u00A0b ".stripAndCollapseHtmlWhitespace() + + // then + assert(result == "\u00A0 a \u00A0b") } @Test fun `should strip and collapse a blank string to empty`() { - assert(" \t\n".stripAndCollapseHtmlWhitespace() == "") + // when + val result = " \t\n".stripAndCollapseHtmlWhitespace() + + // then + assert(result == "") } } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 014fe76..99f2c52 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -38,6 +38,8 @@ import kotlinx.serialization.json.JsonPrimitive // nested array, a flag, a `null`). A flat list of words or numbers — // an empty one included — is metadata a person writes (`keywords`, // `article:tag`, `citation_volume`); +// - a JSON string whose content is itself state by these rules — state +// serialised twice, a common single-page-app double encoding; // - a value longer than [MAX_META_VALUE_LENGTH] that does not read as text — // the backstop for opaque blobs of any other shape (base64, hash lists, // truncated JSON), which a long abstract in prose is not. @@ -52,12 +54,17 @@ internal fun isApplicationStateMeta(content: String): Boolean { // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 +// Recursion unwraps one JSON string per level, and each level's escaping at +// least doubles the backslashes before a quote, so the depth stays +// logarithmic in the value's length. private fun String.isJsonState(): Boolean { val first = firstOrNull() - if (first != '{' && first != '[') return false + if (first != '{' && first != '[' && first != '"') return false return when (val json = parseJsonOrNull()) { is JsonObject -> true is JsonArray -> json.any { !it.isWordOrNumber() } + is JsonPrimitive -> json.isString && + json.content.trim { it.isHtmlWhitespace() }.isJsonState() else -> false } } @@ -69,16 +76,31 @@ private fun String.isJsonState(): Boolean { // base64 and hex never contain one) — sparse enough for a list of long // compound words, while the punctuation of serialised data (quotes, // brackets, `=`, `|`, `\`) stays rare. +// Chars are counted as code points: a surrogate pair — a letter of a +// supplementary-plane script (CJK Extension B, historic scripts) or an emoji, +// which the common stdlib cannot classify — counts once, as a letter of a +// script without spaces; serialised data never holds one. private fun String.readsAsText(): Boolean { + var chars = 0 var letters = 0 var wordBreaks = 0 var dataPunctuation = 0 - for (c in this) { + var i = 0 + while (i < length) { + val c = this[i] + chars++ + if (c.isHighSurrogate() && getOrNull(i + 1)?.isLowSurrogate() == true) { + letters++ + wordBreaks++ + i += 2 + continue + } if (c.isLetter()) letters++ if (c.isWhitespace() || c == ',' || c == ';' || c.code > 0x7F && c.isLetter()) wordBreaks++ else if (c in DATA_PUNCTUATION) dataPunctuation++ + i++ } - return letters * 2 >= length && wordBreaks * 16 >= length && dataPunctuation * 20 < length + return letters * 2 >= chars && wordBreaks * 16 >= chars && dataPunctuation * 20 < chars } private const val DATA_PUNCTUATION = "{}[]\"<>=|\\" diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index eda0935..a653ca9 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -25,19 +25,23 @@ import kotlinx.coroutines.flow.Flow * Ensures the stream starts with a `frontmatter` mark defining a `title`, * deriving a missing title from the first `h1`. * - * A stream whose leading `frontmatter` already holds a usable top-level - * `entry` with `key="title"` (in any ASCII letter case, and only the first - * such entry counts, as [wrapInHtmlDocument] reads it) — one with non-blank - * text, or one holding a nested structure — passes through untouched. Otherwise the frontmatter is - * held back and the title is derived from the very first `h1` following it - * (only blank text may intervene): the `h1` subtree's flattened text — - * its text events plus the `alt` of every `img` mark, in document order, - * the way an accessible name is computed from content — trimmed, internal - * whitespace collapsed to single spaces — becomes the `title`, injected as - * the first `entry` of the frontmatter (or put in - * place of a `title` entry that is blank or `null`, such as a bare `title:` - * line), and only then the frontmatter and the buffered `h1` are emitted, - * in source order. When no frontmatter exists at all, one carrying just the + * The title entry judged is the one [wrapInHtmlDocument] reads: the first + * top-level `entry` with `key="title"` (in any ASCII letter case) holding a + * scalar — entries holding a nested structure, an empty collection or `null` + * are skipped, as there. A stream whose leading `frontmatter` holds such an + * entry with non-blank text passes through untouched. Otherwise the + * frontmatter is held back and the title is derived from the very first `h1` + * following it (only blank text may intervene): the `h1` subtree's flattened + * text — its text events plus the `alt` of every `img` mark, in document + * order, the way an accessible name is computed from content — trimmed, + * internal whitespace collapsed to single spaces — becomes the `title`, and + * only then the frontmatter and the buffered `h1` are emitted, in source + * order. The derived title replaces, in place and keeping its key's spelling, + * the blank scalar title entry, else the first `null` one (a bare `title:` + * line); without either it is injected as the first `entry` of the + * frontmatter — unless a title entry holding a nested structure or an empty + * collection is present, which is then left to stand alone: replacing it + * would lose content, and a second entry would duplicate its key. When no frontmatter exists at all, one carrying just the * derived `title` is synthesized as the **first** event — ahead of any * blank text that preceded the `h1`, since [wrapInHtmlDocument] reads only * a frontmatter that opens the stream. @@ -60,16 +64,27 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s var frontmatterDepth = 0 var hasTitle = false - // the span of a top-level `title` entry that carries no usable title - // (blank text, `type=null`) within `frontmatterEvents`, replaced by the - // derived entry on commit; `titleStart` < 0 when there is none + // the span (and key) of the top-level `title` entry within + // `frontmatterEvents` replaced by the derived entry on commit; + // `titleStart` < 0 when there is none var titleStart = -1 var titleEnd = -1 - // the top-level title entry currently open: its text, and whether it - // holds nested marks (then it is kept as is, whatever its text); only - // the first title entry is judged, the one wrapInHtmlDocument reads - var titleSeen = false + var titleKey = "title" + // the first scalar title entry — the one wrapInHtmlDocument reads — has + // been judged + var scalarTitleSeen = false + // the first `null` title entry, the fallback slot for the derived one + var nullTitleStart = -1 + var nullTitleEnd = -1 + var nullTitleKey = "title" + // a title entry holding a nested structure or an empty collection + var unreadableTitleSeen = false + // the top-level title entry currently open: where it starts, its key, + // `type` and text, and whether it holds nested marks var inTitleEntry = false + var titleEntryStart = -1 + var titleEntryKey = "title" + var titleEntryType: String? = null var titleHasChildren = false val titleText = StringBuilder() @@ -94,8 +109,8 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushBlanks() } - suspend fun emitTitleEntry(title: String) { - "entry"("key" to "title") { +title } + suspend fun emitTitleEntry(title: String, key: String = "title") { + "entry"("key" to key) { +title } } suspend fun commitHeading() { @@ -113,7 +128,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // entry in place of the unusable title entry frontmatterEvents = null emit(held.subList(0, titleStart)) - emitTitleEntry(title) + emitTitleEntry(title, titleKey) emit(held.subList(titleEnd + 1, held.size)) flushBlanks() } else { @@ -146,8 +161,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterEvents = mutableListOf(event) frontmatterDepth = 1 hasTitle = false - titleSeen = false titleStart = -1 + scalarTitleSeen = false + nullTitleStart = -1 + unreadableTitleSeen = false state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) @@ -166,13 +183,15 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterDepth++ if (frontmatterDepth == 2) { // a top-level entry (a direct child) - inTitleEntry = !titleSeen && event.name == "entry" && - event["key"]?.asciiLowercase() == "title" + val key = event["key"] + inTitleEntry = event.name == "entry" && + key?.asciiLowercase() == "title" if (inTitleEntry) { - titleSeen = true + titleEntryStart = events.lastIndex + titleEntryKey = key!! + titleEntryType = event["type"] titleHasChildren = false titleText.clear() - titleStart = events.lastIndex } } else if (inTitleEntry) { titleHasChildren = true @@ -182,18 +201,43 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s is Unmark -> when (--frontmatterDepth) { 1 -> if (inTitleEntry) { inTitleEntry = false - if (titleHasChildren || titleText.isNotBlank()) { - hasTitle = true - titleStart = -1 - } else { - titleEnd = events.lastIndex + val type = titleEntryType + when { + titleHasChildren || type != null && type != "null" && + type !in SCALAR_ENTRY_TYPES -> unreadableTitleSeen = true + type == "null" -> if (nullTitleStart < 0) { + nullTitleStart = titleEntryStart + nullTitleEnd = events.lastIndex + nullTitleKey = titleEntryKey + } + !scalarTitleSeen -> { + scalarTitleSeen = true + if (titleText.isNotBlank()) { + hasTitle = true + } else { + titleStart = titleEntryStart + titleEnd = events.lastIndex + titleKey = titleEntryKey + } + } } } - 0 -> if (hasTitle) { - flushHeld() - state = PassThrough - } else { - state = AwaitingHeading + 0 -> { + if (!scalarTitleSeen) { + if (nullTitleStart >= 0) { + titleStart = nullTitleStart + titleEnd = nullTitleEnd + titleKey = nullTitleKey + } else if (unreadableTitleSeen) { + hasTitle = true + } + } + if (hasTitle) { + flushHeld() + state = PassThrough + } else { + state = AwaitingHeading + } } } } diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 9d534b4..0a02712 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -18,6 +18,15 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.html.spec.asciiLowercase +// The keys wrapInHtmlDocument turns into `<title>` and `<html lang>` rather +// than a `<meta>`, spelled as it reads them. +internal val HEAD_KEYS = setOf("title", "lang") + +// The front matter `entry` types whose text is meaningful as head metadata +// (`null` and the empty collections are not); an entry without a `type` is a +// string. +internal val SCALAR_ENTRY_TYPES = setOf("bool", "int", "float", "timestamp") + // A front matter entry: the name as first spelled, and its value. internal data class MetadataEntry(val key: String, val value: String) diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 94872a7..d0dbda6 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -260,7 +260,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // --- metadata extraction (explicit per-tag) ------------------------- match("html") { event -> - event["lang"]?.let { + event["lang"]?.trim { it.isHtmlWhitespace() }?.let { if (it.isNotBlank()) metadata.add("lang", it) } children() @@ -307,13 +307,16 @@ public fun Flow<SemanticEvent>.simplifyHtml( && !isNoiseMetaName(normalizedName) && !isApplicationStateMeta(content) ) { - when (normalizedName) { - // the two keys wrapInHtmlDocument turns back into <title> - // and <html lang>, spelled as it reads them - "title" -> metadata.add("title", content.trim { it.isHtmlWhitespace() }) - "lang" -> metadata.add("lang", content) - else -> metadata.add(name, content) + val value = when (normalizedName) { + // as a <title> element's text reads (document.title) + "title" -> content.stripAndCollapseHtmlWhitespace() + // as <html lang> is read above + "lang" -> content.trim { it.isHtmlWhitespace() } + else -> content } + // the keys wrapInHtmlDocument turns back into <title> and + // <html lang> are spelled as it reads them + metadata.add(if (normalizedName in HEAD_KEYS) normalizedName else name, value) } } } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 65bbf2f..14f80d3 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -95,7 +95,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman if (depth == 1) { val type = event["type"] entryKey = if ( - event.name == "entry" && (type == null || type in SCALAR_TYPES) + event.name == "entry" && (type == null || type in SCALAR_ENTRY_TYPES) ) event["key"] else null entryText.clear() } else { @@ -135,10 +135,3 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman unmark("body") unmark("html") } - -// Scalar `type`s whose text is meaningful as a `<meta content>` (`null` and -// the empty collections are not). -// The keys that become `<title>` and `<html lang>` rather than a `<meta>`. -private val HEAD_KEYS = setOf("title", "lang") - -private val SCALAR_TYPES = setOf("bool", "int", "float", "timestamp") diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 3776f2d..2fdeafe 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -568,10 +568,10 @@ class EnsureFrontmatterTitleTest { // when val output = input.ensureFrontmatterTitle() - // then + // then — replaced in place, spelled as it was output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Hello" } + "entry"("key" to "TITLE") { +"Hello" } "entry"("key" to "author") { +"Alice" } } "h1" { +"Hello" } @@ -653,6 +653,139 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should pass through a title variant following a null title entry`() = runTest { + // given — wrapInHtmlDocument skips the null entry and reads the + // variant after it as the title + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "Title") { +"Real" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "Title") { +"Real" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should keep a title variant following a null title entry in the head`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "Title") { +"Real" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Real" } + } + "body" { + "h1" { +"Heading" } + } + } + } + } + + @Test + fun `should replace a blank title variant following a nested title entry`() = runTest { + // given — wrapInHtmlDocument skips the nested entry, so the blank + // variant after it is the title it reads + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "Title") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Heading" } + } + "body" { + "h1" { +"Heading" } + } + } + } + } + + @Test + fun `should replace a null title entry following a nested title entry`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "Title", "type" to "null") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "Title") { +"Heading" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should keep a title entry holding an empty collection`() = runTest { + // given — `title: []`, unreadable as a title but not replaceable + // without losing it + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "seq") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "seq") { } + } + "h1" { +"Heading" } + } + } + @Test fun `should derive the head title for parsed Markdown with a bare title key`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index a88babe..fc60f8e 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -946,6 +946,63 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop JSON state serialised into a JSON string`() = runTest { + // given — a single-page-app double encoding, and one nested deeper; + // a quoted phrase and a JSON string of words stay metadata + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "app-state", "content" to "\"{\\\"user\\\":{\\\"id\\\":1}}\"") { } + "meta"("name" to "app-state-2", "content" to "\"\\\"[{\\\\\\\"id\\\\\\\":1}]\\\"\"") { } + "meta"("name" to "description", "content" to "\"Hello\" world") { } + "meta"("name" to "subject", "content" to "\"{Draft} notes\"") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +"\"Hello\" world" } + "entry"("key" to "subject") { +"\"{Draft} notes\"" } + } + "p" { +"text" } + } + } + + @Test + fun `should keep long text written in supplementary-plane letters or emoji`() = runTest { + // given — each of these letters and emoji is a UTF-16 surrogate pair + val extensionB = "\uD840\uDC00\uD840\uDC01\uD840\uDC02。".repeat(700) + val emoji = "Party \uD83C\uDF89\uD83C\uDF89\uD83C\uDF89\uD83C\uDF89 time! ".repeat(200) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "description", "content" to extensionB) { } + "meta"("name" to "og:description", "content" to emoji) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +extensionB } + "entry"("key" to "og:description") { +emoji } + } + "p" { +"text" } + } + } + @Test fun `should keep long comma-separated keyword lists`() = runTest { // given — list separators break words as spaces do, and long compound @@ -1100,6 +1157,52 @@ class SimplifyHtmlTest { } } + @Test + fun `should trim HTML whitespace around the html lang`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html"("lang" to " en\n") { + "head" { } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"en" } + } + "p" { +"x" } + } + } + + @Test + fun `should trim HTML whitespace around a lang meta`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "LANG", "content" to "\tde\n") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"de" } + } + "p" { +"x" } + } + } + @Test fun `should prefer html lang over a lang meta in any letter case`() = runTest { // given — the document language is the root element's attribute @@ -1403,12 +1506,12 @@ class SimplifyHtmlTest { } @Test - fun `should trim a title meta like a title element`() = runTest { + fun `should strip and collapse a title meta like a title element`() = runTest { // given val input = semanticEvents(tagged = true) { "html" { "head" { - "meta"("name" to "title", "content" to " SEO blurb\n") { } + "meta"("name" to "title", "content" to " SEO\n blurb\n") { } } "body" { "p" { +"x" } } } From 124cac8b4939520940dec03d05099ef009a9efdd Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 22:45:33 +0200 Subject: [PATCH 07/25] Skip blank <meta>/front matter values in head metadata; respell title keys (#82) - HeadMetadata.add ignores blank values, so a blank key never shadows a later case variant in simplifyHtml or wrapInHtmlDocument - ensureFrontmatterTitle judges the first non-blank scalar title entry, respells a case-variant key to `title`, and collapses the derived title's whitespace as document.title does - treat double-encoded JSON state inside arrays as application state; count combining marks as letters when judging prose - drop unused kotlinx-serialization-json dependency declaration Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 5 +- README.md | 5 +- markanywhere-html/build.gradle.kts | 1 - .../commonMain/kotlin/ApplicationStateMeta.kt | 21 ++- .../kotlin/EnsureFrontmatterTitle.kt | 172 +++++++++--------- .../src/commonMain/kotlin/HeadMetadata.kt | 7 +- .../commonMain/kotlin/WrapInHtmlDocument.kt | 23 ++- .../kotlin/EnsureFrontmatterTitleTest.kt | 56 +++++- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 55 ++++++ .../kotlin/WrapInHtmlDocumentTest.kt | 63 ++++++- 10 files changed, 280 insertions(+), 128 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c2cd46d..b6dba46 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,9 +560,8 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. - The two must agree on **which** title entry is read and what a *usable* one is, because `wrapInHtmlDocument` skips a `type=null`, nested or empty-collection (`type=seq`/`map`) entry — letting a later case variant (`Title:`) be its title — and emits an empty `<title>` for a blank one. - So `ensureFrontmatterTitle` judges only the first top-level **scalar** title entry in any ASCII letter case (the first of case-variant duplicates wins in both, via the shared `HeadMetadata`; `SCALAR_ENTRY_TYPES` is shared too): non-blank passes through, blank is **replaced in place** keeping its key spelling; with no scalar one it replaces the first `type=null` entry (a bare `title:` line), and a title holding nested marks or an empty collection is left alone — replacing it would lose content, and adding a second `title` would duplicate a YAML key. - Judging the first title entry *of any type* once let a leading `title:` hide a later `Title: Real`, which the derived heading then displaced. + The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. + Blank values are skipped by the shared `HeadMetadata.add`, so `simplifyHtml` and `wrapInHtmlDocument` also agree that a blank key never shadows a later case variant. It also holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. ## Test conventions diff --git a/README.md b/README.md index c8210bb..6e1acd7 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,9 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document -`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the exact inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction, so the two round-trip — except for the values `simplifyHtml` discards as carrying nothing for a reader (a blank value, and application state such as a JSON object or an opaque over-long blob). -Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the first one wins, as for an exact duplicate — the same rule `simplifyHtml` applies to `<meta>` names. +`wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction. +Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the first non-blank one wins, as for an exact duplicate — the same rule `simplifyHtml` applies to `<meta>` names. +The two round-trip up to what they normalise or discard as carrying nothing for a reader: blank, nested and null values, technical noise names (`viewport`, `robots`, …), and application state (a JSON object, an opaque over-long blob) are dropped, case-variant keys are merged, `title` and `lang` come back spelled in lowercase, the title's whitespace is collapsed as `document.title` reads it, and a typed scalar comes back as a string. ```kotlin val document = """ diff --git a/markanywhere-html/build.gradle.kts b/markanywhere-html/build.gradle.kts index 51664b5..e59d11d 100644 --- a/markanywhere-html/build.gradle.kts +++ b/markanywhere-html/build.gradle.kts @@ -42,7 +42,6 @@ kotlin { implementation(project(":markanywhere-dump")) implementation(project(":markanywhere-html-spec")) api(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) implementation(libs.xemantic.kotlin.core) } } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 99f2c52..ef2f835 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -39,7 +39,8 @@ import kotlinx.serialization.json.JsonPrimitive // an empty one included — is metadata a person writes (`keywords`, // `article:tag`, `citation_volume`); // - a JSON string whose content is itself state by these rules — state -// serialised twice, a common single-page-app double encoding; +// serialised twice, a common single-page-app double encoding — whether +// it stands alone or as an element of an array; // - a value longer than [MAX_META_VALUE_LENGTH] that does not read as text — // the backstop for opaque blobs of any other shape (base64, hash lists, // truncated JSON), which a long abstract in prose is not. @@ -62,15 +63,18 @@ private fun String.isJsonState(): Boolean { if (first != '{' && first != '[' && first != '"') return false return when (val json = parseJsonOrNull()) { is JsonObject -> true - is JsonArray -> json.any { !it.isWordOrNumber() } - is JsonPrimitive -> json.isString && - json.content.trim { it.isHtmlWhitespace() }.isJsonState() + is JsonArray -> json.any { !it.isWordOrNumber() || it.isEncodedState() } + is JsonPrimitive -> json.isEncodedState() else -> false } } +private fun JsonElement.isEncodedState(): Boolean = + this is JsonPrimitive && isString && content.trim { it.isHtmlWhitespace() }.isJsonState() + // Text is made of words: at least half the chars are letters (hex and -// number lists are mostly digits), and at least one char in sixteen breaks a +// number lists are mostly digits) — a combining mark counting as one, since +// scripts like Devanagari and vowel-marked Arabic write vowels as marks —, and at least one char in sixteen breaks a // word — whitespace, a list separator (`,` `;`, so `a,b,c` keywords count), // or a letter of a script written without spaces (anything past ASCII — // base64 and hex never contain one) — sparse enough for a list of long @@ -95,7 +99,7 @@ private fun String.readsAsText(): Boolean { i += 2 continue } - if (c.isLetter()) letters++ + if (c.isLetter() || c.category in COMBINING_MARKS) letters++ if (c.isWhitespace() || c == ',' || c == ';' || c.code > 0x7F && c.isLetter()) wordBreaks++ else if (c in DATA_PUNCTUATION) dataPunctuation++ i++ @@ -103,6 +107,11 @@ private fun String.readsAsText(): Boolean { return letters * 2 >= chars && wordBreaks * 16 >= chars && dataPunctuation * 20 < chars } +private val COMBINING_MARKS = setOf( + CharCategory.NON_SPACING_MARK, + CharCategory.COMBINING_SPACING_MARK, +) + private const val DATA_PUNCTUATION = "{}[]\"<>=|\\" // Never throws: the value is page-controlled, and a malformed one is simply diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index a653ca9..cdf19c7 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -19,6 +19,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase +import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import kotlinx.coroutines.flow.Flow /** @@ -26,23 +27,28 @@ import kotlinx.coroutines.flow.Flow * deriving a missing title from the first `h1`. * * The title entry judged is the one [wrapInHtmlDocument] reads: the first - * top-level `entry` with `key="title"` (in any ASCII letter case) holding a - * scalar — entries holding a nested structure, an empty collection or `null` - * are skipped, as there. A stream whose leading `frontmatter` holds such an - * entry with non-blank text passes through untouched. Otherwise the - * frontmatter is held back and the title is derived from the very first `h1` - * following it (only blank text may intervene): the `h1` subtree's flattened - * text — its text events plus the `alt` of every `img` mark, in document - * order, the way an accessible name is computed from content — trimmed, - * internal whitespace collapsed to single spaces — becomes the `title`, and - * only then the frontmatter and the buffered `h1` are emitted, in source - * order. The derived title replaces, in place and keeping its key's spelling, - * the blank scalar title entry, else the first `null` one (a bare `title:` - * line); without either it is injected as the first `entry` of the - * frontmatter — unless a title entry holding a nested structure or an empty - * collection is present, which is then left to stand alone: replacing it - * would lose content, and a second entry would duplicate its key. When no frontmatter exists at all, one carrying just the - * derived `title` is synthesized as the **first** event — ahead of any + * top-level `entry` with `key="title"` (in any ASCII letter case) holding + * non-blank scalar text — entries holding a nested structure, an empty + * collection, `null` or blank text are skipped, as there. A stream whose + * leading `frontmatter` holds such an entry passes through untouched, except + * that a key spelled in another letter case (`Title`) is respelled `title`, + * the one spelling every front matter reader recognises — unless an entry + * spelled `title` is also present, which the respelling would duplicate. + * Otherwise the frontmatter is held back and the title is derived from the + * very first `h1` following it (only blank text may intervene): the `h1` + * subtree's flattened text — its text events plus the `alt` of every `img` + * mark, in document order, the way an accessible name is computed from + * content — with its HTML whitespace stripped and collapsed, as + * `document.title` reads a `<title>`, becomes the `title`, and only then the + * frontmatter and the buffered `h1` are emitted, in source order. The + * derived title replaces, in place, the first blank scalar title entry, else + * the first `null` one (a bare `title:` line), spelling its key `title` on + * the same terms as above; without either it is injected as the first + * `entry` of the frontmatter — unless a title entry holding a nested + * structure or an empty collection is present, which is then left to stand + * alone: replacing it would lose content, and a second entry would + * duplicate its key. When no frontmatter exists at all, one carrying just + * the derived `title` is synthesized as the **first** event — ahead of any * blank text that preceded the `h1`, since [wrapInHtmlDocument] reads only * a frontmatter that opens the stream. * @@ -62,31 +68,27 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // no frontmatter was present and a default one is to be synthesized var frontmatterEvents: MutableList<SemanticEvent>? = null var frontmatterDepth = 0 - var hasTitle = false - // the span (and key) of the top-level `title` entry within - // `frontmatterEvents` replaced by the derived entry on commit; - // `titleStart` < 0 when there is none - var titleStart = -1 - var titleEnd = -1 - var titleKey = "title" - // the first scalar title entry — the one wrapInHtmlDocument reads — has - // been judged - var scalarTitleSeen = false - // the first `null` title entry, the fallback slot for the derived one - var nullTitleStart = -1 - var nullTitleEnd = -1 - var nullTitleKey = "title" - // a title entry holding a nested structure or an empty collection + // top-level title entries (in any letter case) within `frontmatterEvents` + // the first non-blank scalar one — the one wrapInHtmlDocument reads + var usableTitle: TitleSlot? = null + // the first blank scalar one, and the first `null` one, the slots the + // derived title replaces, in this order + var blankTitle: TitleSlot? = null + var nullTitle: TitleSlot? = null + // one holding a nested structure or an empty collection var unreadableTitleSeen = false - // the top-level title entry currently open: where it starts, its key, - // `type` and text, and whether it holds nested marks - var inTitleEntry = false - var titleEntryStart = -1 - var titleEntryKey = "title" - var titleEntryType: String? = null - var titleHasChildren = false - val titleText = StringBuilder() + // an entry spelled exactly `title`, which a respelled key would duplicate + var titleKeyTaken = false + // the slot replaced by the derived entry on commit + var replacedTitle: TitleSlot? = null + + // the top-level title entry currently open (its `end` not yet known), + // its `type` and text, and whether it holds nested marks + var openTitle: TitleSlot? = null + var openTitleType: String? = null + var openTitleHasChildren = false + val openTitleText = StringBuilder() // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit @@ -113,23 +115,28 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s "entry"("key" to key) { +title } } + // the key a title entry is written under + fun titleKey(slot: TitleSlot?): String = + if (slot == null || !titleKeyTaken) "title" else slot.key + suspend fun commitHeading() { - val title = headingText.toString().trim().replace(WHITESPACE_RUN, " ") + val title = headingText.toString().stripAndCollapseHtmlWhitespace() val held = frontmatterEvents - if (title.isEmpty()) { + val replaced = replacedTitle + if (title.isBlank()) { flushHeld() } else if (held == null) { "frontmatter" { emitTitleEntry(title) } flushBlanks() - } else if (titleStart >= 0) { + } else if (replaced != null) { // replay the original frontmatter verbatim, with the derived // entry in place of the unusable title entry frontmatterEvents = null - emit(held.subList(0, titleStart)) - emitTitleEntry(title, titleKey) - emit(held.subList(titleEnd + 1, held.size)) + emit(held.subList(0, replaced.start)) + emitTitleEntry(title, titleKey(replaced)) + emit(held.subList(replaced.end + 1, held.size)) flushBlanks() } else { // replay the original frontmatter verbatim, with a single title @@ -160,11 +167,6 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushBlanks() frontmatterEvents = mutableListOf(event) frontmatterDepth = 1 - hasTitle = false - titleStart = -1 - scalarTitleSeen = false - nullTitleStart = -1 - unreadableTitleSeen = false state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) @@ -184,55 +186,41 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s if (frontmatterDepth == 2) { // a top-level entry (a direct child) val key = event["key"] - inTitleEntry = event.name == "entry" && - key?.asciiLowercase() == "title" - if (inTitleEntry) { - titleEntryStart = events.lastIndex - titleEntryKey = key!! - titleEntryType = event["type"] - titleHasChildren = false - titleText.clear() + if (event.name == "entry" && key?.asciiLowercase() == "title") { + if (key == "title") titleKeyTaken = true + openTitle = TitleSlot(events.lastIndex, -1, key) + openTitleType = event["type"] + openTitleHasChildren = false + openTitleText.clear() } - } else if (inTitleEntry) { - titleHasChildren = true + } else if (openTitle != null) { + openTitleHasChildren = true } } - is Text -> if (inTitleEntry && frontmatterDepth == 2) titleText.append(event.text) + is Text -> if (openTitle != null && frontmatterDepth == 2) openTitleText.append(event.text) is Unmark -> when (--frontmatterDepth) { - 1 -> if (inTitleEntry) { - inTitleEntry = false - val type = titleEntryType + 1 -> openTitle?.let { open -> + openTitle = null + val slot = open.copy(end = events.lastIndex) + val type = openTitleType when { - titleHasChildren || type != null && type != "null" && + openTitleHasChildren || type != null && type != "null" && type !in SCALAR_ENTRY_TYPES -> unreadableTitleSeen = true - type == "null" -> if (nullTitleStart < 0) { - nullTitleStart = titleEntryStart - nullTitleEnd = events.lastIndex - nullTitleKey = titleEntryKey - } - !scalarTitleSeen -> { - scalarTitleSeen = true - if (titleText.isNotBlank()) { - hasTitle = true - } else { - titleStart = titleEntryStart - titleEnd = events.lastIndex - titleKey = titleEntryKey - } - } + type == "null" -> if (nullTitle == null) nullTitle = slot + openTitleText.isNotBlank() -> if (usableTitle == null) usableTitle = slot + else -> if (blankTitle == null) blankTitle = slot } } 0 -> { - if (!scalarTitleSeen) { - if (nullTitleStart >= 0) { - titleStart = nullTitleStart - titleEnd = nullTitleEnd - titleKey = nullTitleKey - } else if (unreadableTitleSeen) { - hasTitle = true - } + val usable = usableTitle + if (usable != null && usable.key != titleKey(usable)) { + val mark = events[usable.start] as Mark + events[usable.start] = mark.copy( + attributes = mark.attributes + ("key" to "title") + ) } - if (hasTitle) { + replacedTitle = blankTitle ?: nullTitle + if (usable != null || replacedTitle == null && unreadableTitleSeen) { flushHeld() state = PassThrough } else { @@ -286,4 +274,6 @@ private enum class State { AtStart, InFrontmatter, AwaitingHeading, InHeading, PassThrough } -private val WHITESPACE_RUN = Regex("""\s+""") +// A top-level title entry: the span of its events within the held +// frontmatter, and its key as spelled. +private data class TitleSlot(val start: Int, val end: Int, val key: String) diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 0a02712..d943eee 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -34,7 +34,9 @@ internal data class MetadataEntry(val key: String, val value: String) // the way HTML reads `<meta>` names: ASCII case-insensitively (HTML §4.2.5). // The one duplicate policy [simplifyHtml] and [wrapInHtmlDocument] share, so // the two stay inverses whichever way a document travels: the first -// occurrence of a name, in any letter case, wins — spelling and value. +// occurrence of a name, in any letter case, with a non-blank value wins — +// spelling and value. A blank value carries nothing for a reader, so it is +// never added, and cannot shadow a later non-blank variant. internal class HeadMetadata { private val entries = LinkedHashMap<String, MetadataEntry>() @@ -47,8 +49,9 @@ internal class HeadMetadata { operator fun get(name: String): MetadataEntry? = entries[name.asciiLowercase()] - // Adds the entry unless its name is already present. + // Adds the entry unless its value is blank or its name already present. fun add(key: String, value: String) { + if (value.isBlank()) return val name = key.asciiLowercase() if (name !in entries) entries[name] = MetadataEntry(key, value) } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 14f80d3..8673a8a 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -27,20 +27,27 @@ import kotlinx.coroutines.flow.Flow * * A **leading** `frontmatter` block (the untagged mark the parser emits for * YAML `---` front matter, holding `entry` marks) feeds the `head` — the - * inverse of [simplifyHtml]'s head-to-frontmatter extraction, so the two - * round-trip, except for the values [simplifyHtml] discards (a blank value, - * application state such as a JSON object or an opaque over-long blob): + * inverse of [simplifyHtml]'s head-to-frontmatter extraction: * * - the `title` entry becomes `<title>` * - the `lang` entry becomes the `lang` attribute on `<html>` * - every other top-level scalar entry becomes a void `<meta name content>` * * Only top-level scalar entries are interpreted — exactly the shape - * [simplifyHtml] produces. A nested mapping or sequence, a null value, and - * verbatim text are skipped (never an error). Keys are read the way HTML - * reads `<meta>` names, ASCII case-insensitively — a `Title` entry is the - * title — and of duplicate keys, in any letter case, the first one wins - * (spelling and value), as in [simplifyHtml]. + * [simplifyHtml] produces. A nested mapping or sequence, a null value, a + * blank value and verbatim text are skipped (never an error). Keys are read + * the way HTML reads `<meta>` names, ASCII case-insensitively — a `Title` + * entry is the title — and of duplicate keys, in any letter case, the first + * one wins (spelling and value), as in [simplifyHtml]. + * + * Passing the result back through [simplifyHtml] restores the front matter + * except for what either side normalises or discards: skipped entries + * (above) and values [simplifyHtml] drops (noise names such as `viewport`, + * application state such as a JSON object or an opaque over-long blob) are + * gone, case-variant duplicates are merged, the `title` and `lang` keys come + * back spelled in lowercase, the title with its whitespace stripped and + * collapsed (as `document.title` reads it) and `lang` trimmed, and a typed + * scalar comes back as a string. * A `frontmatter` mark appearing anywhere past the first event is ordinary * content and flows into `body` verbatim. * diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 2fdeafe..5f0d43b 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -216,6 +216,42 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should keep a non-breaking space inside the derived title`() = runTest { + // given — NBSP is content, not HTML whitespace, as for a <title> + // read by simplifyHtml; an NBSP-only heading still yields no title + val input = semanticEvents { + "h1" { +"\u00A0Foo\u00A0Bar\n" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\u00A0Foo\u00A0Bar" } + } + "h1" { +"\u00A0Foo\u00A0Bar\n" } + } + } + + @Test + fun `should not derive a title from a heading of non-breaking spaces`() = runTest { + // given + val input = semanticEvents { + "h1" { +"\u00A0\u00A0" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "h1" { +"\u00A0\u00A0" } + } + } + @Test fun `should not mistake a nested title entry for a top-level one`() = runTest { // given — only a direct child `entry key=title` counts @@ -533,8 +569,9 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should pass through a title entry in any letter case`() = runTest { - // given — wrapInHtmlDocument reads a `Title` key as the title + fun `should respell a title entry in another letter case as title`() = runTest { + // given — wrapInHtmlDocument reads a `Title` key as the title, but + // front matter readers matching keys case-sensitively (Jekyll) do not val input = semanticEvents { "frontmatter" { "entry"("key" to "Title") { +"My Page" } @@ -548,7 +585,7 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "Title") { +"My Page" } + "entry"("key" to "title") { +"My Page" } } "h1" { +"Heading" } } @@ -568,10 +605,10 @@ class EnsureFrontmatterTitleTest { // when val output = input.ensureFrontmatterTitle() - // then — replaced in place, spelled as it was + // then — replaced in place, spelled as every reader reads it output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "TITLE") { +"Hello" } + "entry"("key" to "title") { +"Hello" } "entry"("key" to "author") { +"Alice" } } "h1" { +"Hello" } @@ -579,9 +616,10 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should judge only the first title entry in any letter case`() = runTest { - // given — wrapInHtmlDocument takes the first title variant, so a - // later usable one does not make a blank first one usable + fun `should pass through a usable title variant following a blank title entry`() = runTest { + // given — wrapInHtmlDocument skips a blank entry, as simplifyHtml + // never extracts one, and reads the later variant; the `title` key + // is taken, so the variant keeps its spelling val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { +" " } @@ -596,7 +634,7 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Hello" } + "entry"("key" to "title") { +" " } "entry"("key" to "Title") { +"Later" } } "h1" { +"Hello" } diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index fc60f8e..2d5870b 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1003,6 +1003,61 @@ class SimplifyHtmlTest { } } + @Test + fun `should keep long text in scripts writing vowels as combining marks`() = runTest { + // given — Devanagari matras and viramas, and Arabic harakat, are marks + // (Mn / Mc), not letters: under half of these chars are letters + val hindi = "किन्तु प्रत्येक व्यक्ति की स्वतन्त्रता सुनिश्चित है। ".repeat(100) + val arabic = "بِسْمِ ٱللَّٰهِ ٱلرَّحْمَٰنِ ٱلرَّحِيمِ ".repeat(150) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "description", "content" to hindi) { } + "meta"("name" to "og:description", "content" to arabic) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +hindi } + "entry"("key" to "og:description") { +arabic } + } + "p" { +"text" } + } + } + + @Test + fun `should drop a JSON array of JSON-encoded state`() = runTest { + // given — each element is state serialised into a string, as the + // same payload unwrapped to a bare JSON string would be + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "app-state", "content" to "[\"{\\\"id\\\":1,\\\"token\\\":\\\"x\\\"}\", \"{\\\"id\\\":2}\"]") { } + "meta"("name" to "keywords", "content" to "[\"{Draft}\", \"notes\"]") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "keywords") { +"[\"{Draft}\", \"notes\"]" } + } + "p" { +"text" } + } + } + @Test fun `should keep long comma-separated keyword lists`() = runTest { // given — list separators break words as spaces do, and long compound diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index cd98abc..abe9dc0 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -130,8 +130,8 @@ class WrapInHtmlDocumentTest { } @Test - fun `should include typed scalars and skip nested null and verbatim content`() = runTest { - // given — only a top-level scalar is representable as head metadata + fun `should include typed scalars and skip nested null blank and verbatim content`() = runTest { + // given — only a top-level non-blank scalar is head metadata val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { +"Hello" } @@ -159,7 +159,6 @@ class WrapInHtmlDocumentTest { "head" { "title" { +"Hello" } "meta"("name" to "year", "content" to "2026") { } - "meta"("name" to "blank", "content" to "") { } } "body" { } } @@ -344,6 +343,58 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should not let a blank key hide a later variant differing in letter case`() = runTest { + // given — simplifyHtml never extracts a blank value, so a blank key + // must not win the duplicate policy either, or the round-trip loses + // the real value + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +"" } + "entry"("key" to "Description") { +"A real summary" } + "entry"("key" to "lang") { +" " } + "entry"("key" to "Lang") { +"de" } + "entry"("key" to "title") { } + "entry"("key" to "Title") { +"Page" } + } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html"("lang" to "de") { + "head" { + "title" { +"Page" } + "meta"("name" to "Description", "content" to "A real summary") { } + } + "body" { } + } + } + } + + @Test + fun `should keep a later non-blank variant of a blank key on a round-trip through simplifyHtml`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +"" } + "entry"("key" to "Description") { +"A real summary" } + } + } + + // when + val output = input.wrapInHtmlDocument().simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Description") { +"A real summary" } + } + } + } + @Test fun `should keep a title key in any letter case on a round-trip through ensureFrontmatterTitle and simplifyHtml`() = runTest { // given — the H1 must not displace the `Title` entry @@ -396,8 +447,8 @@ class WrapInHtmlDocumentTest { @Test fun `should agree with ensureFrontmatterTitle when the first title variant is blank`() = runTest { - // given — the blank first title is the one both judge, so it is - // replaced by the heading rather than shadowed by the later variant + // given — both skip the blank first title, so the later usable + // variant is the title and the heading does not displace it val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { } @@ -413,7 +464,7 @@ class WrapInHtmlDocumentTest { output sameAs semanticEvents { "html" { "head" { - "title" { +"Heading" } + "title" { +"Foo" } } "body" { "h1" { +"Heading" } From 5582f9561386599e99077d673957506af414254f Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 22:58:19 +0200 Subject: [PATCH 08/25] Fix review findings on meta state detection and title derivation (#82) - judge a percent-encoded meta value, including its length, by what it decodes to, and a flat JSON array by its words, so long keyword lists survive the over-long backstop - replace the empty title slot spelled `title` first, inject a title next to an unreadable variant in another spelling, and treat a frontmatter that does not open the stream (or a tagged one) as content, as wrapInHtmlDocument does - share isScalarEntryType / isMetadataValue between wrapInHtmlDocument and ensureFrontmatterTitle; drop the redundant lang blank check Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 3 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 44 +++++---- .../kotlin/EnsureFrontmatterTitle.kt | 59 +++++++----- .../src/commonMain/kotlin/HeadMetadata.kt | 14 ++- .../src/commonMain/kotlin/SimplifyHtml.kt | 6 +- .../commonMain/kotlin/WrapInHtmlDocument.kt | 2 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 96 +++++++++++++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 61 ++++++++++++ 8 files changed, 236 insertions(+), 49 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b6dba46..2295cf9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,8 +560,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. - The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. - Blank values are skipped by the shared `HeadMetadata.add`, so `simplifyHtml` and `wrapInHtmlDocument` also agree that a blank key never shadows a later case variant. + The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc; the shared rules are `isScalarEntryType` / `isMetadataValue`), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. It also holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. ## Test conventions diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index ef2f835..027e0ce 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -23,14 +23,15 @@ import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonPrimitive // Single-page apps use `<meta>` as a transport for application state // (LinkedIn: `__init`, `spark/hash-includes`, Ember's percent-encoded // `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A // name denylist cannot keep up with names private to each site's framework, // so this judges the *value*, which is what tells metadata apart from state. -// A percent-encoded value is judged by what it decodes to, so the verdict -// never depends on the encoding: +// A percent-encoded value is judged by what it decodes to, by every rule +// below, so the verdict never depends on the encoding: // - a value that parses as a JSON object. Parsing, not a look at the first // and last char, is what keeps human text that merely starts with a // bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); @@ -43,34 +44,43 @@ import kotlinx.serialization.json.JsonPrimitive // it stands alone or as an element of an array; // - a value longer than [MAX_META_VALUE_LENGTH] that does not read as text — // the backstop for opaque blobs of any other shape (base64, hash lists, -// truncated JSON), which a long abstract in prose is not. +// truncated JSON), which a long abstract in prose is not. A flat JSON +// array is read by its elements, not the quotes and commas serialising +// them, so a long list of words is kept while a long list of hashes is not. internal fun isApplicationStateMeta(content: String): Boolean { - val value = content.trim { it.isHtmlWhitespace() } - if (value.length > MAX_META_VALUE_LENGTH && !value.readsAsText()) return true - return value.isJsonState() || value.firstOrNull() == '%' && - value.percentDecodedOrNull()?.trim { it.isHtmlWhitespace() }?.isJsonState() == true + val value = content.trimHtmlWhitespace().let { + if (it.firstOrNull() == '%') it.percentDecodedOrNull()?.trimHtmlWhitespace() ?: it else it + } + val json = value.parseJsonCandidateOrNull() + if (json?.isState() == true) return true + val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else value + return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() } // Real metadata is short: `description` / `og:description` rarely exceed 300 // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 +private fun String.trimHtmlWhitespace(): String = trim { it.isHtmlWhitespace() } + +// Only a value opening like a JSON object, array or string is parsed. +private fun String.parseJsonCandidateOrNull(): JsonElement? { + val first = firstOrNull() + return if (first == '{' || first == '[' || first == '"') parseJsonOrNull() else null +} + // Recursion unwraps one JSON string per level, and each level's escaping at // least doubles the backslashes before a quote, so the depth stays // logarithmic in the value's length. -private fun String.isJsonState(): Boolean { - val first = firstOrNull() - if (first != '{' && first != '[' && first != '"') return false - return when (val json = parseJsonOrNull()) { - is JsonObject -> true - is JsonArray -> json.any { !it.isWordOrNumber() || it.isEncodedState() } - is JsonPrimitive -> json.isEncodedState() - else -> false - } +private fun JsonElement.isState(): Boolean = when (this) { + is JsonObject -> true + is JsonArray -> any { !it.isWordOrNumber() || it.isEncodedState() } + is JsonPrimitive -> isEncodedState() } private fun JsonElement.isEncodedState(): Boolean = - this is JsonPrimitive && isString && content.trim { it.isHtmlWhitespace() }.isJsonState() + this is JsonPrimitive && isString && + content.trimHtmlWhitespace().parseJsonCandidateOrNull()?.isState() == true // Text is made of words: at least half the chars are letters (hex and // number lists are mostly digits) — a combining mark counting as one, since diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index cdf19c7..4956e94 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -26,7 +26,9 @@ import kotlinx.coroutines.flow.Flow * Ensures the stream starts with a `frontmatter` mark defining a `title`, * deriving a missing title from the first `h1`. * - * The title entry judged is the one [wrapInHtmlDocument] reads: the first + * The frontmatter and title entry judged are the ones [wrapInHtmlDocument] + * reads: an untagged `frontmatter` mark that is the very first event — one + * anywhere else, or a tagged one, is ordinary content — and in it the first * top-level `entry` with `key="title"` (in any ASCII letter case) holding * non-blank scalar text — entries holding a nested structure, an empty * collection, `null` or blank text are skipped, as there. A stream whose @@ -41,12 +43,13 @@ import kotlinx.coroutines.flow.Flow * content — with its HTML whitespace stripped and collapsed, as * `document.title` reads a `<title>`, becomes the `title`, and only then the * frontmatter and the buffered `h1` are emitted, in source order. The - * derived title replaces, in place, the first blank scalar title entry, else - * the first `null` one (a bare `title:` line), spelling its key `title` on - * the same terms as above; without either it is injected as the first - * `entry` of the frontmatter — unless a title entry holding a nested - * structure or an empty collection is present, which is then left to stand - * alone: replacing it would lose content, and a second entry would + * derived title replaces, in place, the first blank or `null` (a bare + * `title:` line) entry spelled `title`, else the first blank scalar title + * entry in another spelling, else the first `null` one — spelling its key + * `title` on the same terms as above; without any it is injected as the + * first `entry` of the frontmatter — unless an entry spelled `title` holding + * a nested structure or an empty collection is present, which is then left + * to stand alone: replacing it would lose content, and a second entry would * duplicate its key. When no frontmatter exists at all, one carrying just * the derived `title` is synthesized as the **first** event — ahead of any * blank text that preceded the `h1`, since [wrapInHtmlDocument] reads only @@ -72,13 +75,14 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // top-level title entries (in any letter case) within `frontmatterEvents` // the first non-blank scalar one — the one wrapInHtmlDocument reads var usableTitle: TitleSlot? = null - // the first blank scalar one, and the first `null` one, the slots the - // derived title replaces, in this order + // the first blank or `null` one spelled exactly `title`, the first blank + // scalar one and the first `null` one, the slots the derived title + // replaces, in this order + var exactTitle: TitleSlot? = null var blankTitle: TitleSlot? = null var nullTitle: TitleSlot? = null - // one holding a nested structure or an empty collection - var unreadableTitleSeen = false - // an entry spelled exactly `title`, which a respelled key would duplicate + // an entry spelled exactly `title`, which a respelled key or an injected + // entry would duplicate var titleKeyTaken = false // the slot replaced by the derived entry on commit var replacedTitle: TitleSlot? = null @@ -161,10 +165,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s collect { event -> when (state) { AtStart -> when { - event is Mark && event.name == "frontmatter" -> { - // a frontmatter is only ever the first event, but keep - // whatever preceded it in source order - flushBlanks() + // only a frontmatter opening the stream is the page's + // metadata; anywhere else it is content, as for wrapInHtmlDocument + event is Mark && !event.isTagged && event.name == "frontmatter" && blanks.isEmpty() -> { frontmatterEvents = mutableListOf(event) frontmatterDepth = 1 state = InFrontmatter @@ -204,11 +207,18 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s val slot = open.copy(end = events.lastIndex) val type = openTitleType when { - openTitleHasChildren || type != null && type != "null" && - type !in SCALAR_ENTRY_TYPES -> unreadableTitleSeen = true - type == "null" -> if (nullTitle == null) nullTitle = slot - openTitleText.isNotBlank() -> if (usableTitle == null) usableTitle = slot - else -> if (blankTitle == null) blankTitle = slot + // unreadable: kept, never replaced + openTitleHasChildren || type != "null" && !isScalarEntryType(type) -> {} + type == "null" -> { + if (nullTitle == null) nullTitle = slot + if (exactTitle == null && slot.key == "title") exactTitle = slot + } + isMetadataValue(openTitleText.toString()) -> + if (usableTitle == null) usableTitle = slot + else -> { + if (blankTitle == null) blankTitle = slot + if (exactTitle == null && slot.key == "title") exactTitle = slot + } } } 0 -> { @@ -219,8 +229,11 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s attributes = mark.attributes + ("key" to "title") ) } - replacedTitle = blankTitle ?: nullTitle - if (usable != null || replacedTitle == null && unreadableTitleSeen) { + replacedTitle = exactTitle ?: blankTitle ?: nullTitle + // with no slot to replace, an entry spelled + // `title` left is unreadable: injecting would + // duplicate its key + if (usable != null || replacedTitle == null && titleKeyTaken) { flushHeld() state = PassThrough } else { diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index d943eee..44d0359 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -25,7 +25,16 @@ internal val HEAD_KEYS = setOf("title", "lang") // The front matter `entry` types whose text is meaningful as head metadata // (`null` and the empty collections are not); an entry without a `type` is a // string. -internal val SCALAR_ENTRY_TYPES = setOf("bool", "int", "float", "timestamp") +private val SCALAR_ENTRY_TYPES = setOf("bool", "int", "float", "timestamp") + +// Whether an `entry` of this `type` holds a scalar that is head metadata — +// the rule wrapInHtmlDocument reads entries by and ensureFrontmatterTitle +// judges title entries by. +internal fun isScalarEntryType(type: String?): Boolean = + type == null || type in SCALAR_ENTRY_TYPES + +// Whether a value carries anything for a reader — a blank one does not. +internal fun isMetadataValue(value: String): Boolean = value.isNotBlank() // A front matter entry: the name as first spelled, and its value. internal data class MetadataEntry(val key: String, val value: String) @@ -45,13 +54,14 @@ internal class HeadMetadata { fun isNotEmpty(): Boolean = entries.isNotEmpty() + // Every lookup takes a name as spelled; folding it is this class's job. operator fun contains(name: String): Boolean = name.asciiLowercase() in entries operator fun get(name: String): MetadataEntry? = entries[name.asciiLowercase()] // Adds the entry unless its value is blank or its name already present. fun add(key: String, value: String) { - if (value.isBlank()) return + if (!isMetadataValue(value)) return val name = key.asciiLowercase() if (name !in entries) entries[name] = MetadataEntry(key, value) } diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index d0dbda6..1024ecb 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -260,9 +260,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // --- metadata extraction (explicit per-tag) ------------------------- match("html") { event -> - event["lang"]?.trim { it.isHtmlWhitespace() }?.let { - if (it.isNotBlank()) metadata.add("lang", it) - } + event["lang"]?.let { metadata.add("lang", it.trim { c -> c.isHtmlWhitespace() }) } children() } @@ -303,7 +301,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // cheapest checks first: the JSON parse runs only for a name // that would otherwise be kept val normalizedName = name.asciiLowercase() - if (normalizedName !in metadata + if (name !in metadata && !isNoiseMetaName(normalizedName) && !isApplicationStateMeta(content) ) { diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 8673a8a..932e334 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -102,7 +102,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman if (depth == 1) { val type = event["type"] entryKey = if ( - event.name == "entry" && (type == null || type in SCALAR_ENTRY_TYPES) + event.name == "entry" && isScalarEntryType(type) ) event["key"] else null entryText.clear() } else { diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 5f0d43b..5c28fb2 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -824,6 +824,102 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should replace the slot spelled title over a blank variant preceding it`() = runTest { + // given — `Title: ""` then `title:`; a case-sensitive reader + // (Jekyll, Hugo) reads only the latter + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { } + "entry"("key" to "title", "type" to "null") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { } + "entry"("key" to "title") { +"Heading" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should inject a title next to an unreadable title variant`() = runTest { + // given — `Title: []`, which wrapInHtmlDocument skips; a `title` entry + // collides with nothing + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "Title", "type" to "seq") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } + "entry"("key" to "Title", "type" to "seq") { } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should treat a frontmatter after leading blank text as content`() = runTest { + // given — wrapInHtmlDocument reads only a frontmatter opening the + // stream, so this one is body content, not the page's metadata + val input = semanticEvents { + +"\n" + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + +"\n" + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should treat a tagged frontmatter as content`() = runTest { + // given — a literal `<frontmatter>` tag, not front matter + val input = semanticEvents { + tag("frontmatter") { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + tag("frontmatter") { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + } + @Test fun `should derive the head title for parsed Markdown with a bare title key`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 2d5870b..c970c95 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1092,6 +1092,62 @@ class SimplifyHtmlTest { } } + @Test + fun `should judge a long flat JSON array by its words, not its quotes and commas`() = runTest { + // given — both past the cap; the quotes and commas serialising the + // words are not the punctuation of state, a list of hashes still is + val words = (1..600).joinToString(",", "[", "]") { "\"tag$it\"" } + val hashes = (1..300).joinToString(",", "[", "]") { "\"0123456789abcdef\"" } + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to words) { } + "meta"("name" to "hashes", "content" to hashes) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "keywords") { +words } + } + "p" { +"text" } + } + } + + @Test + fun `should judge the length of a percent-encoded value by what it decodes to`() = runTest { + // given — prose whose encoding is past the cap while its text is not, + // and an encoded blob past it either way + val prose = "本研究では大規模言語モデルの挙動を分析した。".repeat(30).percentEncoded() + val blob = ("A" + "x".repeat(5000)).percentEncoded() + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "description", "content" to prose) { } + "meta"("name" to "blob", "content" to blob) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +prose } + } + "p" { +"text" } + } + } + @Test fun `should not decode a percent escape made of non-ASCII digits`() = runTest { // given — an Arabic-Indic seven is no hex digit, so this is not a @@ -2380,3 +2436,8 @@ class SimplifyHtmlTest { } } + +// Every byte as a `%XX` escape of its UTF-8 encoding. +private fun String.percentEncoded(): String = encodeToByteArray().joinToString("") { + "%" + (it.toInt() and 0xFF).toString(16).uppercase().padStart(2, '0') +} From 424475c5b623095ccf34d4e6c0d4dcbf14ceed82 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Mon, 28 Sep 2026 23:34:43 +0200 Subject: [PATCH 09/25] =?UTF-8?q?Add=20review-fix-loop=20script=20automati?= =?UTF-8?q?ng=20the=20code-review=20=E2=86=92=20fix=20=E2=86=92=20commit?= =?UTF-8?q?=20cycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each step runs as a separate headless `claude -p` process, so every review starts from a fresh context. The loop stops when no critical/major correctness finding remains, when the fixer commits nothing, when a round leaves the build red, or at the round limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- scripts/review-fix-loop.sh | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100755 scripts/review-fix-loop.sh diff --git a/scripts/review-fix-loop.sh b/scripts/review-fix-loop.sh new file mode 100755 index 0000000..2deb062 --- /dev/null +++ b/scripts/review-fix-loop.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Automates the "/code-review" → "fix + commit" cycle on a feature branch. +# +# Each step is a separate headless `claude -p` process, so every review starts +# from a fresh context and never sees the reasoning behind the previous fixes. +# The loop stops when the fixer finds no critical/major correctness finding in a +# review, when a round leaves uncommitted changes (build red), or at MAX_ROUNDS. +# +# Usage: scripts/review-fix-loop.sh [base-branch] [max-rounds] +# env: PERMISSION_MODE (default acceptEdits), LOG_DIR (default: a temp dir) + +set -euo pipefail + +base=${1:-main} +max_rounds=${2:-5} +permission_mode=${PERMISSION_MODE:-acceptEdits} +log_dir=${LOG_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/review-fix-loop.XXXXXX")} +sentinel=NO_CRITICAL_FINDINGS + +cd "$(git rev-parse --show-toplevel)" + +if ! git diff --quiet HEAD; then + echo "working tree has uncommitted changes — commit or stash them first" >&2 + exit 1 +fi + +read_tools=( + "Bash(git diff:*)" "Bash(git log:*)" "Bash(git show:*)" "Bash(git status:*)" + "Bash(git rev-parse:*)" "Bash(git merge-base:*)" + "Bash(grep:*)" "Bash(find:*)" "Bash(ls:*)" "Bash(cat:*)" "Bash(head:*)" "Bash(tail:*)" + "Bash(sed -n:*)" "Bash(wc:*)" +) +fix_tools=("${read_tools[@]}" "Bash(./gradlew:*)" "Bash(git add:*)" "Bash(git commit:*)") + +echo "logs: $log_dir" + +for round in $(seq 1 "$max_rounds"); do + review="$log_dir/review-$round.md" + fix_log="$log_dir/fix-$round.log" + + echo "== round $round: review" + claude -p "/code-review high review the current branch against $base" \ + --permission-mode "$permission_mode" \ + --allowedTools "${read_tools[@]}" \ + > "$review" + + if [[ ! -s "$review" ]]; then + echo "round $round: the review produced no output — stopping" >&2 + exit 1 + fi + + echo "== round $round: fix" + head_before=$(git rev-parse HEAD) + claude -p "Below is a code review of the current branch against $base. + +Fix only the findings that are real correctness bugs of critical or major severity caused by this branch: +not cleanups (reuse, simplification, efficiency, altitude, conventions), not contrived edge cases, +not intentional divergences documented in CLAUDE.md. Check each one against the code before acting on it. + +For each bug you fix, follow the TDD rule in CLAUDE.md: add a permanent, named regression test first, +watch it fail, then fix. Drop a finding whose test does not fail. +Run jvmTest for every module you touched, the JS tests if you changed a commonMain Regex, +and apiCheck if you changed public API. If everything is green, create ONE commit (never stage .claude/) +matching the subject style of \`git log -5 --format=%s\`, and end its message with: +Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> +If you cannot get the build green, do not commit. + +If no finding qualifies, change nothing and print $sentinel as the very last line of your answer. + +--- review --- +$(cat "$review")" \ + --permission-mode "$permission_mode" \ + --allowedTools "${fix_tools[@]}" \ + > "$fix_log" + + if [[ "$(tail -n 1 "$fix_log" | tr -d '[:space:]')" == "$sentinel" ]]; then + echo "done: no critical/major findings left after $round round(s)" + exit 0 + fi + + if ! git diff --quiet HEAD; then + echo "round $round left uncommitted changes (build red?) — stopping, see $fix_log" >&2 + exit 1 + fi + + if [[ "$(git rev-parse HEAD)" == "$head_before" ]]; then + echo "round $round: the fixer committed nothing (all findings dropped?) — stopping, see $fix_log" + exit 0 + fi + + echo "round $round: $(git log -1 --format='%h %s')" +done + +echo "stopped: reached $max_rounds rounds" From 5e0bf39f05e6f29eebc0bb2c8771d328b103c3e2 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 00:28:54 +0200 Subject: [PATCH 10/25] Fix Native-illegal test name and untracked-file check in review-fix-loop (#82) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt | 2 +- scripts/review-fix-loop.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index c970c95..442f893 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1093,7 +1093,7 @@ class SimplifyHtmlTest { } @Test - fun `should judge a long flat JSON array by its words, not its quotes and commas`() = runTest { + fun `should judge a long flat JSON array by its words and not its quotes and commas`() = runTest { // given — both past the cap; the quotes and commas serialising the // words are not the punctuation of state, a list of hashes still is val words = (1..600).joinToString(",", "[", "]") { "\"tag$it\"" } diff --git a/scripts/review-fix-loop.sh b/scripts/review-fix-loop.sh index 2deb062..2ee3819 100755 --- a/scripts/review-fix-loop.sh +++ b/scripts/review-fix-loop.sh @@ -20,7 +20,7 @@ sentinel=NO_CRITICAL_FINDINGS cd "$(git rev-parse --show-toplevel)" -if ! git diff --quiet HEAD; then +if [[ -n "$(git status --porcelain)" ]]; then echo "working tree has uncommitted changes — commit or stash them first" >&2 exit 1 fi @@ -79,7 +79,8 @@ $(cat "$review")" \ exit 0 fi - if ! git diff --quiet HEAD; then + # --porcelain also lists untracked files (e.g. a new regression test) + if [[ -n "$(git status --porcelain)" ]]; then echo "round $round left uncommitted changes (build red?) — stopping, see $fix_log" >&2 exit 1 fi From 31a918715f535786ecfd4857f0b4dd216b2f7e7b Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 15:41:02 +0200 Subject: [PATCH 11/25] Guard review-fix-loop against committed yarn.lock and a dirty tree at the sentinel - stop when a round commits kotlin-js-store/yarn.lock, which the JS test build narrows, and revert the lock after every fix round - check for uncommitted changes before honouring the sentinel, so a fixer that edited files cannot end the loop with a dirty tree Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- scripts/review-fix-loop.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/review-fix-loop.sh b/scripts/review-fix-loop.sh index 2ee3819..140eff3 100755 --- a/scripts/review-fix-loop.sh +++ b/scripts/review-fix-loop.sh @@ -17,6 +17,8 @@ max_rounds=${2:-5} permission_mode=${PERMISSION_MODE:-acceptEdits} log_dir=${LOG_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/review-fix-loop.XXXXXX")} sentinel=NO_CRITICAL_FINDINGS +# running the JS tests narrows this lock, which breaks the full build if committed +yarn_lock=kotlin-js-store/yarn.lock cd "$(git rev-parse --show-toplevel)" @@ -61,7 +63,8 @@ not intentional divergences documented in CLAUDE.md. Check each one against the For each bug you fix, follow the TDD rule in CLAUDE.md: add a permanent, named regression test first, watch it fail, then fix. Drop a finding whose test does not fail. Run jvmTest for every module you touched, the JS tests if you changed a commonMain Regex, -and apiCheck if you changed public API. If everything is green, create ONE commit (never stage .claude/) +and apiCheck if you changed public API. If everything is green, create ONE commit +(never stage .claude/ or $yarn_lock — the JS test build narrows the lock; leave it, it is reverted for you) matching the subject style of \`git log -5 --format=%s\`, and end its message with: Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> If you cannot get the build green, do not commit. @@ -74,17 +77,25 @@ $(cat "$review")" \ --allowedTools "${fix_tools[@]}" \ > "$fix_log" - if [[ "$(tail -n 1 "$fix_log" | tr -d '[:space:]')" == "$sentinel" ]]; then - echo "done: no critical/major findings left after $round round(s)" - exit 0 + if ! git diff --quiet "$head_before" HEAD -- "$yarn_lock"; then + echo "round $round committed $yarn_lock (narrowed by the JS tests?) — stopping, see $fix_log" >&2 + exit 1 fi + git checkout -- "$yarn_lock" - # --porcelain also lists untracked files (e.g. a new regression test) + # checked before the sentinel, so a fixer that edited files and then printed + # it cannot end the loop with a dirty tree; --porcelain also lists untracked + # files (e.g. a new regression test) if [[ -n "$(git status --porcelain)" ]]; then echo "round $round left uncommitted changes (build red?) — stopping, see $fix_log" >&2 exit 1 fi + if [[ "$(tail -n 1 "$fix_log" | tr -d '[:space:]')" == "$sentinel" ]]; then + echo "done: no critical/major findings left after $round round(s)" + exit 0 + fi + if [[ "$(git rev-parse HEAD)" == "$head_before" ]]; then echo "round $round: the fixer committed nothing (all findings dropped?) — stopping, see $fix_log" exit 0 From 3a1d5e37158a521f2c13258df1cbe118f914ff1a Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 15:41:02 +0200 Subject: [PATCH 12/25] Resolve duplicate head metadata keys per source format; keep front matter titles readable (#82) - simplifyHtml keeps the first of duplicate <meta> names (HTML); wrapInHtmlDocument keeps the later front matter entry (Psych/PyYAML), the lowercase spelling beating a case variant - ensureFrontmatterTitle drops blank/null entries spelled `title`, respells the usable variant wrapInHtmlDocument picks, and accepts a frontmatter preceded by blank text, emitting it first - add String.stripHtmlWhitespace() to markanywhere-html-spec and use it in place of private trims - skip JSON array word joining for meta values within the length limit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 3 + README.md | 3 +- markanywhere-html-spec/README.md | 1 + .../api/markanywhere-html-spec.api | 1 + .../src/commonMain/kotlin/HtmlWhitespace.kt | 8 + .../commonTest/kotlin/HtmlWhitespaceTest.kt | 9 + .../commonMain/kotlin/ApplicationStateMeta.kt | 12 +- .../kotlin/EnsureFrontmatterTitle.kt | 243 ++++++++++-------- .../src/commonMain/kotlin/HeadMetadata.kt | 30 ++- .../src/commonMain/kotlin/SimplifyHtml.kt | 8 +- .../commonMain/kotlin/WrapInHtmlDocument.kt | 11 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 128 ++++++++- .../kotlin/WrapInHtmlDocumentTest.kt | 63 ++++- 13 files changed, 371 insertions(+), 149 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2295cf9..9fedfa8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -562,6 +562,9 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc; the shared rules are `isScalarEntryType` / `isMetadataValue`), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. It also holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. + Duplicate keys resolve the way readers of the **source format** resolve them, so the two directions deliberately differ: `simplifyHtml` keeps the first `<meta>` of a name (HTML), `wrapInHtmlDocument` the later entry (Psych/PyYAML), the lowercase spelling beating a case variant (`HeadMetadata.addFromHtml` / `addFromFrontMatter`). + Do not "align" them for the round-trip's sake — each side emits one entry per name, so neither ever sees the other's duplicates; aligning them once made `wrapInHtmlDocument` show a title no front matter reader shows. + For the same reason `ensureFrontmatterTitle` edits a frontmatter holding a usable title so a case-sensitive, later-wins reader reads that title too: it drops blank/`null` entries spelled `title` and respells the picked variant. ## Test conventions diff --git a/README.md b/README.md index 6e1acd7..402de9c 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,8 @@ Will print: ### Wrapping parsed Markdown in a complete HTML document `wrapInHtmlDocument()` (in `markanywhere-html`) wraps the event stream in an `html`/`head`/`body` structure, populating the `head` from a leading front matter block: `title` becomes `<title>`, `lang` becomes the `<html lang>` attribute, and every other flat key becomes a `<meta name content>` — the inverse of `simplifyHtml`'s `<head>`-to-front-matter extraction. -Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title, and of keys differing only in letter case the first non-blank one wins, as for an exact duplicate — the same rule `simplifyHtml` applies to `<meta>` names. +Keys are read ASCII case-insensitively, as HTML reads `<meta>` names: a `Title` key is the title. +Of duplicate keys the later non-blank one wins, as front matter readers (Jekyll, PyYAML) resolve them, except that the lowercase spelling beats a variant in another letter case — `simplifyHtml` instead keeps the first of duplicate `<meta>` names, as HTML does. The two round-trip up to what they normalise or discard as carrying nothing for a reader: blank, nested and null values, technical noise names (`viewport`, `robots`, …), and application state (a JSON object, an opaque over-long blob) are dropped, case-variant keys are merged, `title` and `lang` come back spelled in lowercase, the title's whitespace is collapsed as `document.title` reads it, and a typed scalar comes back as a string. ```kotlin diff --git a/markanywhere-html-spec/README.md b/markanywhere-html-spec/README.md index 7c4d66a..c879372 100644 --- a/markanywhere-html-spec/README.md +++ b/markanywhere-html-spec/README.md @@ -11,6 +11,7 @@ so code that reads a semantic event stream carrying HTML attributes can use it w |---------------------------------------------------|------------------------------------------------------------------------------------------------| | `HTML_WHITESPACE_CHARS`, `Char.isHtmlWhitespace()` | HTML "ASCII whitespace": TAB, LF, FF, CR, SPACE — not NBSP, which HTML treats as content | | `String.isHtmlBlank()` | Empty or HTML whitespace only | +| `String.stripHtmlWhitespace()` | Trimmed of HTML whitespace at both ends, NBSP kept | | `String.stripAndCollapseHtmlWhitespace()` | Trimmed of HTML whitespace, inner runs collapsed to one space, as `document.title` reads it | | `Char.asciiLowercase()`, `String.asciiLowercase()` | ASCII lowercase: only `A`–`Z` fold, as HTML compares names "ASCII case-insensitively" | | `HTML_VOID_ELEMENTS` | Elements with no content and no closing tag, including the obsolete `keygen` and `param` | diff --git a/markanywhere-html-spec/api/markanywhere-html-spec.api b/markanywhere-html-spec/api/markanywhere-html-spec.api index 9cb4fbe..71fb6d8 100644 --- a/markanywhere-html-spec/api/markanywhere-html-spec.api +++ b/markanywhere-html-spec/api/markanywhere-html-spec.api @@ -12,6 +12,7 @@ public final class com/xemantic/markanywhere/html/spec/HtmlWhitespaceKt { public static final fun isHtmlBlank (Ljava/lang/String;)Z public static final fun isHtmlWhitespace (C)Z public static final fun stripAndCollapseHtmlWhitespace (Ljava/lang/String;)Ljava/lang/String; + public static final fun stripHtmlWhitespace (Ljava/lang/String;)Ljava/lang/String; } public final class com/xemantic/markanywhere/html/spec/MarkClassListKt { diff --git a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt index 33a1b97..3f469f7 100644 --- a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt +++ b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt @@ -42,6 +42,14 @@ public fun Char.isHtmlWhitespace(): Boolean = this in HTML_WHITESPACE_CHARS */ public fun String.isHtmlBlank(): Boolean = all { it.isHtmlWhitespace() } +/** + * This string with leading and trailing [HTML whitespace][isHtmlWhitespace] + * removed — the WHATWG Infra "strip leading and trailing ASCII whitespace", + * which HTML applies to attribute values such as `lang`. NBSP is content and + * stays. + */ +public fun String.stripHtmlWhitespace(): String = trim { it.isHtmlWhitespace() } + /** * This string with leading and trailing [HTML whitespace][isHtmlWhitespace] * removed and every inner run of it replaced by a single space — the WHATWG diff --git a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt index 43f2e65..d218ea9 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt @@ -97,4 +97,13 @@ class HtmlWhitespaceTest { assert(result == "") } + @Test + fun `should strip leading and trailing HTML whitespace keeping inner runs and NBSP`() { + // when + val result = "\n\t\u00A0a b\u00A0 \u000C\r".stripHtmlWhitespace() + + // then + assert(result == "\u00A0a b\u00A0") + } + } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 027e0ce..a105db0 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -16,7 +16,7 @@ package com.xemantic.markanywhere.html -import com.xemantic.markanywhere.html.spec.isHtmlWhitespace +import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -48,11 +48,13 @@ import kotlinx.serialization.json.jsonPrimitive // array is read by its elements, not the quotes and commas serialising // them, so a long list of words is kept while a long list of hashes is not. internal fun isApplicationStateMeta(content: String): Boolean { - val value = content.trimHtmlWhitespace().let { - if (it.firstOrNull() == '%') it.percentDecodedOrNull()?.trimHtmlWhitespace() ?: it else it + val value = content.stripHtmlWhitespace().let { + if (it.firstOrNull() == '%') it.percentDecodedOrNull()?.stripHtmlWhitespace() ?: it else it } val json = value.parseJsonCandidateOrNull() if (json?.isState() == true) return true + // a flat array's words are never longer than the value serialising them + if (value.length <= MAX_META_VALUE_LENGTH) return false val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else value return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() } @@ -61,8 +63,6 @@ internal fun isApplicationStateMeta(content: String): Boolean { // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 -private fun String.trimHtmlWhitespace(): String = trim { it.isHtmlWhitespace() } - // Only a value opening like a JSON object, array or string is parsed. private fun String.parseJsonCandidateOrNull(): JsonElement? { val first = firstOrNull() @@ -80,7 +80,7 @@ private fun JsonElement.isState(): Boolean = when (this) { private fun JsonElement.isEncodedState(): Boolean = this is JsonPrimitive && isString && - content.trimHtmlWhitespace().parseJsonCandidateOrNull()?.isState() == true + content.stripHtmlWhitespace().parseJsonCandidateOrNull()?.isState() == true // Text is made of words: at least half the chars are letters (hex and // number lists are mostly digits) — a combining mark counting as one, since diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 4956e94..7bdd59e 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -26,16 +26,24 @@ import kotlinx.coroutines.flow.Flow * Ensures the stream starts with a `frontmatter` mark defining a `title`, * deriving a missing title from the first `h1`. * - * The frontmatter and title entry judged are the ones [wrapInHtmlDocument] - * reads: an untagged `frontmatter` mark that is the very first event — one - * anywhere else, or a tagged one, is ordinary content — and in it the first - * top-level `entry` with `key="title"` (in any ASCII letter case) holding - * non-blank scalar text — entries holding a nested structure, an empty - * collection, `null` or blank text are skipped, as there. A stream whose - * leading `frontmatter` holds such an entry passes through untouched, except - * that a key spelled in another letter case (`Title`) is respelled `title`, - * the one spelling every front matter reader recognises — unless an entry - * spelled `title` is also present, which the respelling would duplicate. + * The frontmatter judged is the one [wrapInHtmlDocument] reads: an untagged + * `frontmatter` mark opening the stream — blank text before it is + * insignificant, and the frontmatter is emitted ahead of it, so that + * [wrapInHtmlDocument] reads it too; one anywhere else, or a tagged one, is + * ordinary content. Its title entries are its top-level `entry` marks with + * `key="title"` in any ASCII letter case, and one holding non-blank scalar + * text is usable — entries holding a nested structure, an empty collection, + * `null` or blank text are not, as there. + * + * With a usable title entry the frontmatter passes through, edited only so + * that a front matter reader — matching keys case-sensitively and keeping + * the later of duplicate keys — reads the title [wrapInHtmlDocument] reads: + * every blank or `null` entry spelled `title` is dropped, and when the + * usable entry [wrapInHtmlDocument] picks is spelled in another letter case + * (`Title`) its key is respelled `title` — unless an entry spelled `title` + * holding a nested structure or an empty collection is present, which the + * respelling would duplicate. + * * Otherwise the frontmatter is held back and the title is derived from the * very first `h1` following it (only blank text may intervene): the `h1` * subtree's flattened text — its text events plus the `alt` of every `img` @@ -44,16 +52,16 @@ import kotlinx.coroutines.flow.Flow * `document.title` reads a `<title>`, becomes the `title`, and only then the * frontmatter and the buffered `h1` are emitted, in source order. The * derived title replaces, in place, the first blank or `null` (a bare - * `title:` line) entry spelled `title`, else the first blank scalar title - * entry in another spelling, else the first `null` one — spelling its key - * `title` on the same terms as above; without any it is injected as the - * first `entry` of the frontmatter — unless an entry spelled `title` holding - * a nested structure or an empty collection is present, which is then left - * to stand alone: replacing it would lose content, and a second entry would - * duplicate its key. When no frontmatter exists at all, one carrying just - * the derived `title` is synthesized as the **first** event — ahead of any - * blank text that preceded the `h1`, since [wrapInHtmlDocument] reads only - * a frontmatter that opens the stream. + * `title:` line) entry spelled `title` — dropping every other one, as above + * — else the first blank scalar title entry in another spelling, else the + * first `null` one, spelling its key `title` on the same terms as above; + * without any it is injected as the first `entry` of the frontmatter — + * unless an entry spelled `title` holding a nested structure or an empty + * collection is present, which is then left to stand alone: replacing it + * would lose content, and a second entry would duplicate its key. When no + * frontmatter exists at all, one carrying just the derived `title` is + * synthesized as the **first** event — ahead of any blank text that + * preceded the `h1`. * * When no title can be derived — the first non-blank event after the * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — @@ -72,28 +80,23 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s var frontmatterEvents: MutableList<SemanticEvent>? = null var frontmatterDepth = 0 - // top-level title entries (in any letter case) within `frontmatterEvents` - // the first non-blank scalar one — the one wrapInHtmlDocument reads - var usableTitle: TitleSlot? = null - // the first blank or `null` one spelled exactly `title`, the first blank - // scalar one and the first `null` one, the slots the derived title - // replaces, in this order - var exactTitle: TitleSlot? = null - var blankTitle: TitleSlot? = null - var nullTitle: TitleSlot? = null - // an entry spelled exactly `title`, which a respelled key or an injected - // entry would duplicate - var titleKeyTaken = false - // the slot replaced by the derived entry on commit - var replacedTitle: TitleSlot? = null - - // the top-level title entry currently open (its `end` not yet known), - // its `type` and text, and whether it holds nested marks - var openTitle: TitleSlot? = null - var openTitleType: String? = null + // the top-level title entries (in any letter case) within + // `frontmatterEvents`, in source order + val titleSlots = mutableListOf<TitleSlot>() + // the top-level title entry currently open: its mark and start, its + // text, and whether it holds nested marks + var openTitle: SemanticEvent.Mark? = null + var openTitleStart = -1 var openTitleHasChildren = false val openTitleText = StringBuilder() + // the blank or `null` title entries spelled `title`, the slot of them (or + // of a variant) the derived title replaces, and whether an unreadable + // entry spelled `title` stands, which a respelled key would duplicate + var emptyTitles: List<TitleSlot> = emptyList() + var replacedTitle: TitleSlot? = null + var titleKeyTaken = false + // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit val blanks = mutableListOf<SemanticEvent>() @@ -109,8 +112,18 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s blanks.clear() } - suspend fun flushHeld() { - frontmatterEvents?.let { emit(it) } + // emits the held frontmatter, the span of each slot in `edits` replaced + // by what its edit emits, then the blanks held with it + suspend fun flushHeld(edits: Map<TitleSlot, suspend () -> Unit> = emptyMap()) { + frontmatterEvents?.let { held -> + var next = 0 + for ((slot, edit) in edits.entries.sortedBy { it.key.start }) { + emit(held.subList(next, slot.start)) + edit() + next = slot.end + 1 + } + emit(held.subList(next, held.size)) + } frontmatterEvents = null flushBlanks() } @@ -119,43 +132,75 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s "entry"("key" to key) { +title } } - // the key a title entry is written under - fun titleKey(slot: TitleSlot?): String = - if (slot == null || !titleKeyTaken) "title" else slot.key + // the edits dropping every empty title entry spelled `title` but `kept` + fun dropEmptyTitles(kept: TitleSlot? = null): Map<TitleSlot, suspend () -> Unit> = + emptyTitles.filter { it != kept }.associateWith { {} } suspend fun commitHeading() { val title = headingText.toString().stripAndCollapseHtmlWhitespace() - val held = frontmatterEvents val replaced = replacedTitle - if (title.isBlank()) { - flushHeld() - } else if (held == null) { - "frontmatter" { + when { + !isMetadataValue(title) -> flushHeld() + frontmatterEvents == null -> { + "frontmatter" { + emitTitleEntry(title) + } + flushBlanks() + } + // the derived entry in place of the unusable title entry + replaced != null -> { + val key = if (titleKeyTaken) replaced.key else "title" + flushHeld(dropEmptyTitles(kept = replaced) + (replaced to { emitTitleEntry(title, key) })) + } + // a single title entry prepended to the frontmatter's content + else -> { + val held = frontmatterEvents!! + frontmatterEvents = null + emit(held.first()) emitTitleEntry(title) + emit(held.subList(1, held.size)) + flushBlanks() } - flushBlanks() - } else if (replaced != null) { - // replay the original frontmatter verbatim, with the derived - // entry in place of the unusable title entry - frontmatterEvents = null - emit(held.subList(0, replaced.start)) - emitTitleEntry(title, titleKey(replaced)) - emit(held.subList(replaced.end + 1, held.size)) - flushBlanks() - } else { - // replay the original frontmatter verbatim, with a single title - // entry prepended to its content - frontmatterEvents = null - emit(held.first()) - emitTitleEntry(title) - emit(held.subList(1, held.size)) - flushBlanks() } emit(headingEvents) headingEvents.clear() state = PassThrough } + // decides, once the frontmatter closes, whether it holds a usable title + suspend fun judgeFrontmatter() { + titleKeyTaken = titleSlots.any { it.kind == UNREADABLE && it.key == "title" } + emptyTitles = titleSlots.filter { it.key == "title" && (it.kind == BLANK || it.kind == NULL) } + // the entry wrapInHtmlDocument reads (HeadMetadata.addFromFrontMatter) + val usable = titleSlots.lastOrNull { it.kind == USABLE && it.key == "title" } + ?: titleSlots.lastOrNull { it.kind == USABLE } + if (usable != null) { + val edits = dropEmptyTitles() + val held = frontmatterEvents!! + flushHeld( + if (usable.key == "title" || titleKeyTaken) edits + else edits + (usable to { + val mark = held[usable.start] as Mark + emit(mark.copy(attributes = mark.attributes + ("key" to "title"))) + emit(held.subList(usable.start + 1, usable.end + 1)) + }) + ) + state = PassThrough + return + } + replacedTitle = emptyTitles.firstOrNull() + ?: titleSlots.firstOrNull { it.kind == BLANK } + ?: titleSlots.firstOrNull { it.kind == NULL } + // with no slot to replace, an entry spelled `title` left is + // unreadable: injecting would duplicate its key + if (replacedTitle == null && titleKeyTaken) { + flushHeld() + state = PassThrough + } else { + state = AwaitingHeading + } + } + fun startHeading(event: SemanticEvent) { headingEvents += event headingDepth = 1 @@ -165,9 +210,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s collect { event -> when (state) { AtStart -> when { - // only a frontmatter opening the stream is the page's - // metadata; anywhere else it is content, as for wrapInHtmlDocument - event is Mark && !event.isTagged && event.name == "frontmatter" && blanks.isEmpty() -> { + // only a frontmatter opening the stream (but for blank text) + // is the page's metadata; anywhere else it is content + event is Mark && !event.isTagged && event.name == "frontmatter" -> { frontmatterEvents = mutableListOf(event) frontmatterDepth = 1 state = InFrontmatter @@ -188,11 +233,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s frontmatterDepth++ if (frontmatterDepth == 2) { // a top-level entry (a direct child) - val key = event["key"] - if (event.name == "entry" && key?.asciiLowercase() == "title") { - if (key == "title") titleKeyTaken = true - openTitle = TitleSlot(events.lastIndex, -1, key) - openTitleType = event["type"] + if (event.name == "entry" && event["key"]?.asciiLowercase() == "title") { + openTitle = event + openTitleStart = events.lastIndex openTitleHasChildren = false openTitleText.clear() } @@ -204,42 +247,16 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s is Unmark -> when (--frontmatterDepth) { 1 -> openTitle?.let { open -> openTitle = null - val slot = open.copy(end = events.lastIndex) - val type = openTitleType - when { - // unreadable: kept, never replaced - openTitleHasChildren || type != "null" && !isScalarEntryType(type) -> {} - type == "null" -> { - if (nullTitle == null) nullTitle = slot - if (exactTitle == null && slot.key == "title") exactTitle = slot - } - isMetadataValue(openTitleText.toString()) -> - if (usableTitle == null) usableTitle = slot - else -> { - if (blankTitle == null) blankTitle = slot - if (exactTitle == null && slot.key == "title") exactTitle = slot - } - } - } - 0 -> { - val usable = usableTitle - if (usable != null && usable.key != titleKey(usable)) { - val mark = events[usable.start] as Mark - events[usable.start] = mark.copy( - attributes = mark.attributes + ("key" to "title") - ) - } - replacedTitle = exactTitle ?: blankTitle ?: nullTitle - // with no slot to replace, an entry spelled - // `title` left is unreadable: injecting would - // duplicate its key - if (usable != null || replacedTitle == null && titleKeyTaken) { - flushHeld() - state = PassThrough - } else { - state = AwaitingHeading + val type = open["type"] + val kind: TitleKind = when { + openTitleHasChildren || type != "null" && !isScalarEntryType(type) -> UNREADABLE + type == "null" -> NULL + isMetadataValue(openTitleText.toString()) -> USABLE + else -> BLANK } + titleSlots += TitleSlot(openTitleStart, events.lastIndex, open["key"]!!, kind) } + 0 -> judgeFrontmatter() } } } @@ -287,6 +304,12 @@ private enum class State { AtStart, InFrontmatter, AwaitingHeading, InHeading, PassThrough } +// How a title entry reads: a non-blank scalar, blank text, `null`, or a +// nested structure or an empty collection. +private enum class TitleKind { + USABLE, BLANK, NULL, UNREADABLE +} + // A top-level title entry: the span of its events within the held -// frontmatter, and its key as spelled. -private data class TitleSlot(val start: Int, val end: Int, val key: String) +// frontmatter, its key as spelled, and how it reads. +private data class TitleSlot(val start: Int, val end: Int, val key: String, val kind: TitleKind) diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 44d0359..a967fcd 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -41,11 +41,11 @@ internal data class MetadataEntry(val key: String, val value: String) // The `<head>` metadata a front matter holds, in insertion order and keyed // the way HTML reads `<meta>` names: ASCII case-insensitively (HTML §4.2.5). -// The one duplicate policy [simplifyHtml] and [wrapInHtmlDocument] share, so -// the two stay inverses whichever way a document travels: the first -// occurrence of a name, in any letter case, with a non-blank value wins — -// spelling and value. A blank value carries nothing for a reader, so it is -// never added, and cannot shadow a later non-blank variant. +// A blank value carries nothing for a reader, so it is never added, and +// cannot shadow a variant holding one. Duplicates resolve the way readers of +// the format they were read from resolve them — [simplifyHtml] reads HTML, +// [wrapInHtmlDocument] YAML. The two policies need not agree for the two to +// stay inverses: each emits a single entry per name. internal class HeadMetadata { private val entries = LinkedHashMap<String, MetadataEntry>() @@ -59,13 +59,29 @@ internal class HeadMetadata { operator fun get(name: String): MetadataEntry? = entries[name.asciiLowercase()] - // Adds the entry unless its value is blank or its name already present. - fun add(key: String, value: String) { + // Adds a `<meta>` read from HTML, where the first of duplicate elements + // is the one a query finds: the first occurrence of a name, in any letter + // case, wins — spelling and value. + fun addFromHtml(key: String, value: String) { if (!isMetadataValue(value)) return val name = key.asciiLowercase() if (name !in entries) entries[name] = MetadataEntry(key, value) } + // Adds a front matter entry the way its readers resolve a duplicate key + // (Psych — Jekyll's — and PyYAML keep the later one), except that the + // lowercase spelling, the one a case-sensitive reader such as Jekyll + // looks up for `title`, beats a variant wherever it occurs. The name keeps + // the position of its first occurrence. + fun addFromFrontMatter(key: String, value: String) { + if (!isMetadataValue(value)) return + val name = key.asciiLowercase() + val existing = entries[name] + if (existing == null || key == name || existing.key != name) { + entries[name] = MetadataEntry(key, value) + } + } + // Sets the entry whether or not its name is present, keeping the position // of an existing one. operator fun set(key: String, value: String) { diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 1024ecb..602e98c 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -19,7 +19,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations import com.xemantic.markanywhere.html.spec.asciiLowercase -import com.xemantic.markanywhere.html.spec.isHtmlWhitespace +import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform @@ -260,7 +260,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // --- metadata extraction (explicit per-tag) ------------------------- match("html") { event -> - event["lang"]?.let { metadata.add("lang", it.trim { c -> c.isHtmlWhitespace() }) } + event["lang"]?.let { metadata.addFromHtml("lang", it.stripHtmlWhitespace()) } children() } @@ -309,12 +309,12 @@ public fun Flow<SemanticEvent>.simplifyHtml( // as a <title> element's text reads (document.title) "title" -> content.stripAndCollapseHtmlWhitespace() // as <html lang> is read above - "lang" -> content.trim { it.isHtmlWhitespace() } + "lang" -> content.stripHtmlWhitespace() else -> content } // the keys wrapInHtmlDocument turns back into <title> and // <html lang> are spelled as it reads them - metadata.add(if (normalizedName in HEAD_KEYS) normalizedName else name, value) + metadata.addFromHtml(if (normalizedName in HEAD_KEYS) normalizedName else name, value) } } } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 932e334..938d18e 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -37,8 +37,11 @@ import kotlinx.coroutines.flow.Flow * [simplifyHtml] produces. A nested mapping or sequence, a null value, a * blank value and verbatim text are skipped (never an error). Keys are read * the way HTML reads `<meta>` names, ASCII case-insensitively — a `Title` - * entry is the title — and of duplicate keys, in any letter case, the first - * one wins (spelling and value), as in [simplifyHtml]. + * entry is the title. Of duplicate keys the later one wins, as front matter + * readers (Jekyll, PyYAML) resolve them, except that a key spelled in + * lowercase — the spelling a case-sensitive reader looks up — beats a + * variant in another letter case wherever it occurs; the winner keeps the + * position of the first duplicate. * * Passing the result back through [simplifyHtml] restores the front matter * except for what either side normalises or discards: skipped entries @@ -114,7 +117,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman openDocument() } else { if (depth == 1) { - entryKey?.let { metadata.add(it, entryText.toString()) } + entryKey?.let { metadata.addFromFrontMatter(it, entryText.toString()) } entryKey = null } depth-- @@ -136,7 +139,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // still used, including an entry left open — its text is complete by // then; an empty stream yields the bare skeleton. if (collectingFrontmatter && depth == 1) { - entryKey?.let { metadata.add(it, entryText.toString()) } + entryKey?.let { metadata.addFromFrontMatter(it, entryText.toString()) } } if (!opened) openDocument() unmark("body") diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 5c28fb2..3f3d83f 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -616,10 +616,10 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should pass through a usable title variant following a blank title entry`() = runTest { + fun `should respell a usable title variant following a blank title entry dropping the blank one`() = runTest { // given — wrapInHtmlDocument skips a blank entry, as simplifyHtml - // never extracts one, and reads the later variant; the `title` key - // is taken, so the variant keeps its spelling + // never extracts one, and reads the later variant, while a + // case-sensitive reader (Jekyll, Hugo) reads only the blank `title` val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { +" " } @@ -631,11 +631,10 @@ class EnsureFrontmatterTitleTest { // when val output = input.ensureFrontmatterTitle() - // then + // then — the blank entry carried nothing for either reader output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +" " } - "entry"("key" to "Title") { +"Later" } + "entry"("key" to "title") { +"Later" } } "h1" { +"Hello" } } @@ -692,7 +691,7 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should pass through a title variant following a null title entry`() = runTest { + fun `should respell a title variant following a null title entry dropping the null one`() = runTest { // given — wrapInHtmlDocument skips the null entry and reads the // variant after it as the title val input = semanticEvents { @@ -709,8 +708,7 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title", "type" to "null") { } - "entry"("key" to "Title") { +"Real" } + "entry"("key" to "title") { +"Real" } } "h1" { +"Heading" } } @@ -874,9 +872,10 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should treat a frontmatter after leading blank text as content`() = runTest { - // given — wrapInHtmlDocument reads only a frontmatter opening the - // stream, so this one is body content, not the page's metadata + fun `should move a frontmatter preceded by blank text ahead of it`() = runTest { + // given — blank text before the first element is insignificant, and + // renderMarkdown drops it, but wrapInHtmlDocument reads only a + // frontmatter opening the stream val input = semanticEvents { +"\n" "frontmatter" { @@ -890,8 +889,113 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } + "entry"("key" to "author") { +"Alice" } + } +"\n" + "h1" { +"Heading" } + } + } + + @Test + fun `should put the title of a frontmatter preceded by blank text into the head`() = runTest { + // given + val input = semanticEvents { + +"\n" + "frontmatter" { + "entry"("key" to "title") { +"Page" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Page" } + } + "body" { + +"\n" + "h1" { +"Heading" } + } + } + } + } + + @Test + fun `should drop a blank title entry following a usable one`() = runTest { + // given — a front matter reader keeps the later duplicate, so the + // blank entry would hide the title wrapInHtmlDocument reads + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real" } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real" } + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should respell the last usable title variant as wrapInHtmlDocument reads it`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { +"First" } + "entry"("key" to "TITLE") { +"Last" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { +"First" } + "entry"("key" to "title") { +"Last" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should drop every other empty title entry when deriving the title`() = runTest { + // given — a front matter reader keeps the later `title:`, which + // would hide the derived title in the first slot + val input = semanticEvents { "frontmatter" { + "entry"("key" to "title") { } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title", "type" to "null") { } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } "entry"("key" to "author") { +"Alice" } } "h1" { +"Heading" } diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index abe9dc0..0571c25 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -166,8 +166,8 @@ class WrapInHtmlDocumentTest { } @Test - fun `should let the first of duplicate keys win`() = runTest { - // given + fun `should let the last of duplicate keys win as front matter readers do`() = runTest { + // given — Psych (Jekyll) and PyYAML keep the later duplicate val input = semanticEvents { "frontmatter" { "entry"("key" to "author") { +"Alice" } @@ -182,6 +182,34 @@ class WrapInHtmlDocumentTest { output sameAs semanticEvents { "html" { "head" { + "meta"("name" to "author", "content" to "Bob") { } + } + "body" { } + } + } + } + + @Test + fun `should read the title front matter readers read of duplicate title keys`() = runTest { + // given + val document = flowOf( + """ + --- + title: Draft + author: Alice + title: Final + --- + """.trimIndent() + ) + + // when + val output = document.parse().wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Final" } "meta"("name" to "author", "content" to "Alice") { } } "body" { } @@ -189,6 +217,31 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should prefer the lowercase spelling of a key over a later variant`() = runTest { + // given — a case-sensitive reader (Jekyll) looks up `title`, whatever + // follows it in another letter case + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real" } + "entry"("key" to "Title") { +"Variant" } + } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Real" } + } + "body" { } + } + } + } + @Test fun `should pass a mid-stream frontmatter block through into body`() = runTest { // given — only a *leading* frontmatter feeds the head; anywhere else it @@ -315,8 +368,8 @@ class WrapInHtmlDocumentTest { @Test fun `should read keys ASCII case-insensitively like meta names`() = runTest { - // given — a later key differing only in letter case is a duplicate, - // so the first one wins, spelling and value, as in simplifyHtml + // given — a key differing only in letter case is a duplicate, and + // its lowercase spelling wins, at the position of the first one val input = semanticEvents { "frontmatter" { "entry"("key" to "Title") { +"My Page" } @@ -335,7 +388,7 @@ class WrapInHtmlDocumentTest { "html"("lang" to "de") { "head" { "title" { +"My Page" } - "meta"("name" to "Author", "content" to "Alice") { } + "meta"("name" to "author", "content" to "Bob") { } "meta"("name" to "description", "content" to "A doc") { } } "body" { } From 2afddbcc44f647f7b2896a3df77ea9dd5ff3526b Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 17:06:45 +0200 Subject: [PATCH 13/25] Remove review-fix-loop script Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- scripts/review-fix-loop.sh | 107 ------------------------------------- 1 file changed, 107 deletions(-) delete mode 100755 scripts/review-fix-loop.sh diff --git a/scripts/review-fix-loop.sh b/scripts/review-fix-loop.sh deleted file mode 100755 index 140eff3..0000000 --- a/scripts/review-fix-loop.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# -# Automates the "/code-review" → "fix + commit" cycle on a feature branch. -# -# Each step is a separate headless `claude -p` process, so every review starts -# from a fresh context and never sees the reasoning behind the previous fixes. -# The loop stops when the fixer finds no critical/major correctness finding in a -# review, when a round leaves uncommitted changes (build red), or at MAX_ROUNDS. -# -# Usage: scripts/review-fix-loop.sh [base-branch] [max-rounds] -# env: PERMISSION_MODE (default acceptEdits), LOG_DIR (default: a temp dir) - -set -euo pipefail - -base=${1:-main} -max_rounds=${2:-5} -permission_mode=${PERMISSION_MODE:-acceptEdits} -log_dir=${LOG_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/review-fix-loop.XXXXXX")} -sentinel=NO_CRITICAL_FINDINGS -# running the JS tests narrows this lock, which breaks the full build if committed -yarn_lock=kotlin-js-store/yarn.lock - -cd "$(git rev-parse --show-toplevel)" - -if [[ -n "$(git status --porcelain)" ]]; then - echo "working tree has uncommitted changes — commit or stash them first" >&2 - exit 1 -fi - -read_tools=( - "Bash(git diff:*)" "Bash(git log:*)" "Bash(git show:*)" "Bash(git status:*)" - "Bash(git rev-parse:*)" "Bash(git merge-base:*)" - "Bash(grep:*)" "Bash(find:*)" "Bash(ls:*)" "Bash(cat:*)" "Bash(head:*)" "Bash(tail:*)" - "Bash(sed -n:*)" "Bash(wc:*)" -) -fix_tools=("${read_tools[@]}" "Bash(./gradlew:*)" "Bash(git add:*)" "Bash(git commit:*)") - -echo "logs: $log_dir" - -for round in $(seq 1 "$max_rounds"); do - review="$log_dir/review-$round.md" - fix_log="$log_dir/fix-$round.log" - - echo "== round $round: review" - claude -p "/code-review high review the current branch against $base" \ - --permission-mode "$permission_mode" \ - --allowedTools "${read_tools[@]}" \ - > "$review" - - if [[ ! -s "$review" ]]; then - echo "round $round: the review produced no output — stopping" >&2 - exit 1 - fi - - echo "== round $round: fix" - head_before=$(git rev-parse HEAD) - claude -p "Below is a code review of the current branch against $base. - -Fix only the findings that are real correctness bugs of critical or major severity caused by this branch: -not cleanups (reuse, simplification, efficiency, altitude, conventions), not contrived edge cases, -not intentional divergences documented in CLAUDE.md. Check each one against the code before acting on it. - -For each bug you fix, follow the TDD rule in CLAUDE.md: add a permanent, named regression test first, -watch it fail, then fix. Drop a finding whose test does not fail. -Run jvmTest for every module you touched, the JS tests if you changed a commonMain Regex, -and apiCheck if you changed public API. If everything is green, create ONE commit -(never stage .claude/ or $yarn_lock — the JS test build narrows the lock; leave it, it is reverted for you) -matching the subject style of \`git log -5 --format=%s\`, and end its message with: -Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> -If you cannot get the build green, do not commit. - -If no finding qualifies, change nothing and print $sentinel as the very last line of your answer. - ---- review --- -$(cat "$review")" \ - --permission-mode "$permission_mode" \ - --allowedTools "${fix_tools[@]}" \ - > "$fix_log" - - if ! git diff --quiet "$head_before" HEAD -- "$yarn_lock"; then - echo "round $round committed $yarn_lock (narrowed by the JS tests?) — stopping, see $fix_log" >&2 - exit 1 - fi - git checkout -- "$yarn_lock" - - # checked before the sentinel, so a fixer that edited files and then printed - # it cannot end the loop with a dirty tree; --porcelain also lists untracked - # files (e.g. a new regression test) - if [[ -n "$(git status --porcelain)" ]]; then - echo "round $round left uncommitted changes (build red?) — stopping, see $fix_log" >&2 - exit 1 - fi - - if [[ "$(tail -n 1 "$fix_log" | tr -d '[:space:]')" == "$sentinel" ]]; then - echo "done: no critical/major findings left after $round round(s)" - exit 0 - fi - - if [[ "$(git rev-parse HEAD)" == "$head_before" ]]; then - echo "round $round: the fixer committed nothing (all findings dropped?) — stopping, see $fix_log" - exit 0 - fi - - echo "round $round: $(git log -1 --format='%h %s')" -done - -echo "stopped: reached $max_rounds rounds" From 86d330587a1ac8dbb7a321ffdf1e5f0ac5f1d3f1 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 17:06:45 +0200 Subject: [PATCH 14/25] Treat blank text before front matter alike; keep a usable title after a shadowing one (#82) - wrapInHtmlDocument reads a frontmatter preceded by blank text, moving that text to the start of body, as ensureFrontmatterTitle does - ensureFrontmatterTitle moves a usable `title` entry after a later collection-valued `title` entry, which a later-wins reader would pick - simplifyHtml dedupes meta names on the normalized name (first wins) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 4 +- .../kotlin/EnsureFrontmatterTitle.kt | 43 ++++++++++++------- .../src/commonMain/kotlin/SimplifyHtml.kt | 8 ++-- .../commonMain/kotlin/WrapInHtmlDocument.kt | 32 ++++++++------ .../kotlin/EnsureFrontmatterTitleTest.kt | 30 +++++++++++++ .../kotlin/WrapInHtmlDocumentTest.kt | 28 ++++++++++++ 6 files changed, 113 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9fedfa8..4fa0067 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -561,10 +561,10 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc; the shared rules are `isScalarEntryType` / `isMetadataValue`), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. - It also holds any blank text seen *before* the first `h1` so a synthesized `frontmatter` is the **first** event, the only position `wrapInHtmlDocument` reads it from. + Both treat blank text ahead of the frontmatter as insignificant; `ensureFrontmatterTitle` additionally emits the frontmatter (held or synthesized) **ahead** of that text, since rendered Markdown front matter must open the document. Duplicate keys resolve the way readers of the **source format** resolve them, so the two directions deliberately differ: `simplifyHtml` keeps the first `<meta>` of a name (HTML), `wrapInHtmlDocument` the later entry (Psych/PyYAML), the lowercase spelling beating a case variant (`HeadMetadata.addFromHtml` / `addFromFrontMatter`). Do not "align" them for the round-trip's sake — each side emits one entry per name, so neither ever sees the other's duplicates; aligning them once made `wrapInHtmlDocument` show a title no front matter reader shows. - For the same reason `ensureFrontmatterTitle` edits a frontmatter holding a usable title so a case-sensitive, later-wins reader reads that title too: it drops blank/`null` entries spelled `title` and respells the picked variant. + For the same reason `ensureFrontmatterTitle` edits a frontmatter holding a usable title so a case-sensitive, later-wins reader reads that title too (see its KDoc). ## Test conventions diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 7bdd59e..0069e0e 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -27,10 +27,10 @@ import kotlinx.coroutines.flow.Flow * deriving a missing title from the first `h1`. * * The frontmatter judged is the one [wrapInHtmlDocument] reads: an untagged - * `frontmatter` mark opening the stream — blank text before it is - * insignificant, and the frontmatter is emitted ahead of it, so that - * [wrapInHtmlDocument] reads it too; one anywhere else, or a tagged one, is - * ordinary content. Its title entries are its top-level `entry` marks with + * `frontmatter` mark opening the stream, blank text before it being + * insignificant — the frontmatter is emitted ahead of that text, so it opens + * the stream as Markdown front matter must; one anywhere else, or a tagged + * one, is ordinary content. Its title entries are its top-level `entry` marks with * `key="title"` in any ASCII letter case, and one holding non-blank scalar * text is usable — entries holding a nested structure, an empty collection, * `null` or blank text are not, as there. @@ -38,11 +38,13 @@ import kotlinx.coroutines.flow.Flow * With a usable title entry the frontmatter passes through, edited only so * that a front matter reader — matching keys case-sensitively and keeping * the later of duplicate keys — reads the title [wrapInHtmlDocument] reads: - * every blank or `null` entry spelled `title` is dropped, and when the - * usable entry [wrapInHtmlDocument] picks is spelled in another letter case - * (`Title`) its key is respelled `title` — unless an entry spelled `title` - * holding a nested structure or an empty collection is present, which the - * respelling would duplicate. + * every blank or `null` entry spelled `title` is dropped; when the usable + * entry [wrapInHtmlDocument] picks is spelled `title` and followed by one + * holding a nested structure or an empty collection, it is moved right after + * that one; and when it is spelled in another letter case (`Title`) its key + * is respelled `title` — unless an entry spelled `title` holding a nested + * structure or an empty collection is present, which the respelling would + * duplicate. * * Otherwise the frontmatter is held back and the title is derived from the * very first `h1` following it (only blank text may intervene): the `h1` @@ -177,13 +179,24 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s if (usable != null) { val edits = dropEmptyTitles() val held = frontmatterEvents!! + // an unreadable entry spelled `title` after the usable one, which + // a later-wins reader would read instead + val shadowing = titleSlots.lastOrNull { it.kind == UNREADABLE && it.key == "title" } + ?.takeIf { usable.key == "title" && it.start > usable.start } flushHeld( - if (usable.key == "title" || titleKeyTaken) edits - else edits + (usable to { - val mark = held[usable.start] as Mark - emit(mark.copy(attributes = mark.attributes + ("key" to "title"))) - emit(held.subList(usable.start + 1, usable.end + 1)) - }) + when { + // the usable entry moved right after the shadowing one + shadowing != null -> edits + (usable to {}) + (shadowing to { + emit(held.subList(shadowing.start, shadowing.end + 1)) + emit(held.subList(usable.start, usable.end + 1)) + }) + usable.key == "title" || titleKeyTaken -> edits + else -> edits + (usable to { + val mark = held[usable.start] as Mark + emit(mark.copy(attributes = mark.attributes + ("key" to "title"))) + emit(held.subList(usable.start + 1, usable.end + 1)) + }) + } ) state = PassThrough return diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 602e98c..a329a4f 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -139,7 +139,9 @@ import kotlinx.coroutines.flow.Flow * application state that single-page apps ship in `<meta>` (serialised JSON, * framework config blobs — see [isApplicationStateMeta]). Meta names are * ASCII case-insensitive, so of several names differing only in letter case - * the first one (spelling and value) wins, as in [wrapInHtmlDocument]; a + * the first one (spelling and value) wins, as HTML resolves duplicate + * `<meta>` elements — unlike [wrapInHtmlDocument], which resolves duplicate + * front matter keys as YAML readers do; a * `lang` meta is spelled `lang` and yields to `<html lang>`, the document's * actual language. Those discarded values and merged * names are what does not survive a [wrapInHtmlDocument] round-trip. When @@ -299,9 +301,9 @@ public fun Flow<SemanticEvent>.simplifyHtml( val content = event["content"] if (name != null && !content.isNullOrBlank()) { // cheapest checks first: the JSON parse runs only for a name - // that would otherwise be kept + // that would otherwise be kept (the first of duplicates wins) val normalizedName = name.asciiLowercase() - if (name !in metadata + if (normalizedName !in metadata && !isNoiseMetaName(normalizedName) && !isApplicationStateMeta(content) ) { diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 938d18e..0bd2d87 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -51,12 +51,14 @@ import kotlinx.coroutines.flow.Flow * back spelled in lowercase, the title with its whitespace stripped and * collapsed (as `document.title` reads it) and `lang` trimmed, and a typed * scalar comes back as a string. - * A `frontmatter` mark appearing anywhere past the first event is ordinary - * content and flows into `body` verbatim. + * Blank text ahead of the frontmatter is insignificant — it is moved to the + * start of `body`, as [ensureFrontmatterTitle] moves it after the + * frontmatter. A `frontmatter` mark appearing past any other event is + * ordinary content and flows into `body` verbatim. * - * Only the frontmatter subtree is read ahead (bounded); without one the - * document opening is emitted on the first event and body content streams - * through untouched. All synthetic marks are untagged, consistent with the + * Only leading blank text and the frontmatter subtree are read ahead + * (bounded); without a frontmatter the document opening is emitted on the + * first non-blank event and body content streams through untouched. All synthetic marks are untagged, consistent with the * parser's `frontmatter` mark and [simplifyHtml] output. An empty input * stream still yields the full document skeleton. */ @@ -70,6 +72,8 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman var entryKey: String? = null val entryText = StringBuilder() val metadata = HeadMetadata() + // blank text ahead of the frontmatter, replayed at the start of `body` + val blanks = mutableListOf<SemanticEvent>() // `head` and its subtree are lexically scoped, so the paired `"name" { }` // builder fits; `html` and `body` close only at end-of-stream, so their @@ -95,6 +99,8 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman } } mark("body") + emit(blanks) + blanks.clear() } collect { event -> @@ -123,13 +129,15 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman depth-- } } - !opened -> if ( - event is Mark && !event.isTagged && event.name == "frontmatter" - ) { - collectingFrontmatter = true - } else { - openDocument() - emit(event) + !opened -> when { + event is Mark && !event.isTagged && event.name == "frontmatter" -> { + collectingFrontmatter = true + } + event is Text && event.text.isBlank() -> blanks += event + else -> { + openDocument() + emit(event) + } } else -> emit(event) } diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 3f3d83f..194825d 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -952,6 +952,36 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should move a usable title entry after a later unreadable one`() = runTest { + // given — a front matter reader keeps the later duplicate, so the + // collection would hide the title wrapInHtmlDocument reads; moving the + // usable entry after it keeps both entries + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real" } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title", "type" to "seq") { } + "entry"("key" to "tags") { +"x" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title", "type" to "seq") { } + "entry"("key" to "title") { +"Real" } + "entry"("key" to "tags") { +"x" } + } + "h1" { +"Heading" } + } + } + @Test fun `should respell the last usable title variant as wrapInHtmlDocument reads it`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 0571c25..711a024 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -270,6 +270,34 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should read a frontmatter preceded by blank text into the head`() = runTest { + // given — blank text is insignificant, as for ensureFrontmatterTitle + val input = semanticEvents { + +"\n" + "frontmatter" { + "entry"("key" to "title") { +"Page" } + } + "p" { +"Body." } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Page" } + } + "body" { + +"\n" + "p" { +"Body." } + } + } + } + } + @Test fun `should wrap parsed Markdown in a complete HTML document`() = runTest { // given From 4772c9e9c8b02d6ace4256747df0f0e01060f23a Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 17:34:08 +0200 Subject: [PATCH 15/25] Trim edge NBSP from titles; place a derived title after a shadowing one (#82) - normalizeTitle trims non-breaking space at a title's edges (inside it stays content), shared by simplifyHtml and ensureFrontmatterTitle - ensureFrontmatterTitle puts a derived title right after a later collection-valued `title` entry, which a later-wins reader would pick, and picks the usable entry via frontMatterKeySupersedes, the rule HeadMetadata.addFromFrontMatter uses - use getOrPutIfMissing for first-wins HTML metadata - declare kotlinx-serialization-json directly for ApplicationStateMeta Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- markanywhere-html/build.gradle.kts | 1 + .../kotlin/EnsureFrontmatterTitle.kt | 144 ++++++++++-------- .../src/commonMain/kotlin/HeadMetadata.kt | 29 +++- .../src/commonMain/kotlin/SimplifyHtml.kt | 12 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 45 +++++- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 10 +- 6 files changed, 153 insertions(+), 88 deletions(-) diff --git a/markanywhere-html/build.gradle.kts b/markanywhere-html/build.gradle.kts index e59d11d..51664b5 100644 --- a/markanywhere-html/build.gradle.kts +++ b/markanywhere-html/build.gradle.kts @@ -42,6 +42,7 @@ kotlin { implementation(project(":markanywhere-dump")) implementation(project(":markanywhere-html-spec")) api(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) implementation(libs.xemantic.kotlin.core) } } diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 0069e0e..f35c21e 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -19,7 +19,6 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase -import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import kotlinx.coroutines.flow.Flow /** @@ -51,13 +50,16 @@ import kotlinx.coroutines.flow.Flow * subtree's flattened text — its text events plus the `alt` of every `img` * mark, in document order, the way an accessible name is computed from * content — with its HTML whitespace stripped and collapsed, as - * `document.title` reads a `<title>`, becomes the `title`, and only then the + * `document.title` reads a `<title>`, and any non-breaking space at its edges + * trimmed, becomes the `title`, and only then the * frontmatter and the buffered `h1` are emitted, in source order. The * derived title replaces, in place, the first blank or `null` (a bare * `title:` line) entry spelled `title` — dropping every other one, as above * — else the first blank scalar title entry in another spelling, else the - * first `null` one, spelling its key `title` on the same terms as above; - * without any it is injected as the first `entry` of the frontmatter — + * first `null` one, spelling its key `title` on the same terms as above — + * or, spelled `title`, it goes right after a later entry spelled `title` + * holding a nested structure or an empty collection, as a usable one is + * moved; without any it is injected as the first `entry` of the frontmatter — * unless an entry spelled `title` holding a nested structure or an empty * collection is present, which is then left to stand alone: replacing it * would lose content, and a second entry would duplicate its key. When no @@ -92,12 +94,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s var openTitleHasChildren = false val openTitleText = StringBuilder() - // the blank or `null` title entries spelled `title`, the slot of them (or - // of a variant) the derived title replaces, and whether an unreadable - // entry spelled `title` stands, which a respelled key would duplicate - var emptyTitles: List<TitleSlot> = emptyList() - var replacedTitle: TitleSlot? = null - var titleKeyTaken = false + // how a derived title goes into the held frontmatter, decided once it + // closes; null while none is held + var derivation: Derivation? = null // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit @@ -114,14 +113,20 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s blanks.clear() } - // emits the held frontmatter, the span of each slot in `edits` replaced - // by what its edit emits, then the blanks held with it - suspend fun flushHeld(edits: Map<TitleSlot, suspend () -> Unit> = emptyMap()) { + // emits the held frontmatter — `prepended` right after its mark, the span + // of each slot in `rewrite` replaced by its events (none drops the entry) + // — then the blanks held with it + suspend fun flushHeld( + rewrite: Map<TitleSlot, List<SemanticEvent>> = emptyMap(), + prepended: List<SemanticEvent> = emptyList() + ) { frontmatterEvents?.let { held -> - var next = 0 - for ((slot, edit) in edits.entries.sortedBy { it.key.start }) { + emit(held.first()) + emit(prepended) + var next = 1 + for ((slot, events) in rewrite.entries.sortedBy { it.key.start }) { emit(held.subList(next, slot.start)) - edit() + emit(events) next = slot.end + 1 } emit(held.subList(next, held.size)) @@ -130,39 +135,41 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushBlanks() } - suspend fun emitTitleEntry(title: String, key: String = "title") { - "entry"("key" to key) { +title } + // the events of a slot's entry within the held frontmatter + fun span(slot: TitleSlot): List<SemanticEvent> = frontmatterEvents!!.subList(slot.start, slot.end + 1) + + // the rewrite putting `entry`, keyed `key`, in place of `slot` — or, + // spelled `title` and followed by an unreadable entry spelled `title` + // which a later-wins reader would read instead, right after that one + fun placed(slot: TitleSlot, entry: List<SemanticEvent>, key: String): Map<TitleSlot, List<SemanticEvent>> { + val shadowing = titleSlots + .lastOrNull { it.kind == UNREADABLE && it.key == "title" } + ?.takeIf { key == "title" && it.start > slot.start } + return if (shadowing == null) { + mapOf(slot to entry) + } else { + mapOf(slot to emptyList(), shadowing to span(shadowing) + entry) + } } - // the edits dropping every empty title entry spelled `title` but `kept` - fun dropEmptyTitles(kept: TitleSlot? = null): Map<TitleSlot, suspend () -> Unit> = - emptyTitles.filter { it != kept }.associateWith { {} } + fun dropping(slots: List<TitleSlot>): Map<TitleSlot, List<SemanticEvent>> = + slots.associateWith { emptyList() } suspend fun commitHeading() { - val title = headingText.toString().stripAndCollapseHtmlWhitespace() - val replaced = replacedTitle + val title = headingText.toString().normalizeTitle() + val plan = derivation when { !isMetadataValue(title) -> flushHeld() frontmatterEvents == null -> { "frontmatter" { - emitTitleEntry(title) + emit(titleEntry(title, "title")) } flushBlanks() } - // the derived entry in place of the unusable title entry - replaced != null -> { - val key = if (titleKeyTaken) replaced.key else "title" - flushHeld(dropEmptyTitles(kept = replaced) + (replaced to { emitTitleEntry(title, key) })) - } // a single title entry prepended to the frontmatter's content - else -> { - val held = frontmatterEvents!! - frontmatterEvents = null - emit(held.first()) - emitTitleEntry(title) - emit(held.subList(1, held.size)) - flushBlanks() - } + plan!!.slot == null -> flushHeld(prepended = titleEntry(title, plan.key)) + // the derived entry in place of the unusable title entry + else -> flushHeld(dropping(plan.dropped) + placed(plan.slot, titleEntry(title, plan.key), plan.key)) } emit(headingEvents) headingEvents.clear() @@ -171,45 +178,34 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // decides, once the frontmatter closes, whether it holds a usable title suspend fun judgeFrontmatter() { - titleKeyTaken = titleSlots.any { it.kind == UNREADABLE && it.key == "title" } - emptyTitles = titleSlots.filter { it.key == "title" && (it.kind == BLANK || it.kind == NULL) } - // the entry wrapInHtmlDocument reads (HeadMetadata.addFromFrontMatter) - val usable = titleSlots.lastOrNull { it.kind == USABLE && it.key == "title" } - ?: titleSlots.lastOrNull { it.kind == USABLE } + // an unreadable entry spelled `title`, which a respelled key would duplicate + val titleKeyTaken = titleSlots.any { it.kind == UNREADABLE && it.key == "title" } + val emptyTitles = titleSlots.filter { it.key == "title" && (it.kind == BLANK || it.kind == NULL) } + // the entry wrapInHtmlDocument reads + val usable = titleSlots + .filter { it.kind == USABLE } + .reduceOrNull { read, next -> if (frontMatterKeySupersedes(read.key, next.key)) next else read } if (usable != null) { - val edits = dropEmptyTitles() - val held = frontmatterEvents!! - // an unreadable entry spelled `title` after the usable one, which - // a later-wins reader would read instead - val shadowing = titleSlots.lastOrNull { it.kind == UNREADABLE && it.key == "title" } - ?.takeIf { usable.key == "title" && it.start > usable.start } - flushHeld( - when { - // the usable entry moved right after the shadowing one - shadowing != null -> edits + (usable to {}) + (shadowing to { - emit(held.subList(shadowing.start, shadowing.end + 1)) - emit(held.subList(usable.start, usable.end + 1)) - }) - usable.key == "title" || titleKeyTaken -> edits - else -> edits + (usable to { - val mark = held[usable.start] as Mark - emit(mark.copy(attributes = mark.attributes + ("key" to "title"))) - emit(held.subList(usable.start + 1, usable.end + 1)) - }) - } - ) + val key = if (titleKeyTaken) usable.key else "title" + val entry = if (key == usable.key) span(usable) else respelled(span(usable), key) + flushHeld(dropping(emptyTitles) + placed(usable, entry, key)) state = PassThrough return } - replacedTitle = emptyTitles.firstOrNull() + val slot = emptyTitles.firstOrNull() ?: titleSlots.firstOrNull { it.kind == BLANK } ?: titleSlots.firstOrNull { it.kind == NULL } // with no slot to replace, an entry spelled `title` left is // unreadable: injecting would duplicate its key - if (replacedTitle == null && titleKeyTaken) { + if (slot == null && titleKeyTaken) { flushHeld() state = PassThrough } else { + derivation = Derivation( + slot = slot, + key = if (slot != null && titleKeyTaken) slot.key else "title", + dropped = emptyTitles.filter { it != slot } + ) state = AwaitingHeading } } @@ -326,3 +322,21 @@ private enum class TitleKind { // A top-level title entry: the span of its events within the held // frontmatter, its key as spelled, and how it reads. private data class TitleSlot(val start: Int, val end: Int, val key: String, val kind: TitleKind) + +// How a derived title goes into the held frontmatter: in place of `slot` +// (null: prepended to the frontmatter's entries), keyed `key`, the `dropped` +// empty title entries removed. +private class Derivation(val slot: TitleSlot?, val key: String, val dropped: List<TitleSlot>) + +// A top-level front matter entry keyed `key` holding the scalar `value`. +private fun titleEntry(value: String, key: String): List<SemanticEvent> = listOf( + SemanticEvent.Mark("entry", attributes = mapOf("key" to key)), + SemanticEvent.Text(value), + SemanticEvent.Unmark("entry") +) + +// An entry's events with its key spelled `key`. +private fun respelled(entry: List<SemanticEvent>, key: String): List<SemanticEvent> { + val mark = entry.first() as Mark + return listOf(mark.copy(attributes = mark.attributes + ("key" to key))) + entry.drop(1) +} diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index a967fcd..f69f251 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.html.spec.asciiLowercase +import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace // The keys wrapInHtmlDocument turns into `<title>` and `<html lang>` rather // than a `<meta>`, spelled as it reads them. @@ -36,6 +37,22 @@ internal fun isScalarEntryType(type: String?): Boolean = // Whether a value carries anything for a reader — a blank one does not. internal fun isMetadataValue(value: String): Boolean = value.isNotBlank() +// A title as `document.title` reads a `<title>` — HTML whitespace stripped +// and collapsed — with any other whitespace at its edges (the NBSP padding an +// icon often leaves) trimmed too: inside, NBSP is content and stays, at an +// edge it only forces the title into quotes. +internal fun String.normalizeTitle(): String = stripAndCollapseHtmlWhitespace().trim() + +// Whether a front matter entry spelled `candidate` supersedes an earlier one +// of the same name spelled `existing`, as front matter readers resolve a +// duplicate key: the later one wins (Psych — Jekyll's — and PyYAML), except +// that the lowercase spelling, the one a case-sensitive reader such as Jekyll +// looks up for `title`, beats a variant wherever it occurs. +internal fun frontMatterKeySupersedes(existing: String, candidate: String): Boolean { + val name = candidate.asciiLowercase() + return candidate == name || existing != name +} + // A front matter entry: the name as first spelled, and its value. internal data class MetadataEntry(val key: String, val value: String) @@ -64,20 +81,18 @@ internal class HeadMetadata { // case, wins — spelling and value. fun addFromHtml(key: String, value: String) { if (!isMetadataValue(value)) return - val name = key.asciiLowercase() - if (name !in entries) entries[name] = MetadataEntry(key, value) + @OptIn(ExperimentalStdlibApi::class) + entries.getOrPutIfMissing(key.asciiLowercase()) { MetadataEntry(key, value) } } // Adds a front matter entry the way its readers resolve a duplicate key - // (Psych — Jekyll's — and PyYAML keep the later one), except that the - // lowercase spelling, the one a case-sensitive reader such as Jekyll - // looks up for `title`, beats a variant wherever it occurs. The name keeps - // the position of its first occurrence. + // ([frontMatterKeySupersedes]). The name keeps the position of its first + // occurrence. fun addFromFrontMatter(key: String, value: String) { if (!isMetadataValue(value)) return val name = key.asciiLowercase() val existing = entries[name] - if (existing == null || key == name || existing.key != name) { + if (existing == null || frontMatterKeySupersedes(existing.key, key)) { entries[name] = MetadataEntry(key, value) } } diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index a329a4f..ecdc6d7 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -20,7 +20,6 @@ import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations import com.xemantic.markanywhere.html.spec.asciiLowercase import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace -import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform import kotlinx.coroutines.flow.Flow @@ -147,7 +146,8 @@ import kotlinx.coroutines.flow.Flow * names are what does not survive a [wrapInHtmlDocument] round-trip. When * `<head>` holds several `<title>`s, the first non-blank one wins, over a * `<meta name="title">` (in any letter case) too, with its whitespace - * stripped and collapsed as `document.title` does. If `<head>` is absent or + * stripped and collapsed as `document.title` does and any non-breaking space + * at its edges trimmed. If `<head>` is absent or * yields no metadata, no frontmatter mark is emitted. * * Matcher registration is grouped: per-tag explicit matchers come first @@ -289,8 +289,8 @@ public fun Flow<SemanticEvent>.simplifyHtml( children(mode = "titleText") afterClose { if (!titleFromElement && titleText.isNotBlank()) { - // as `document.title` reads it - metadata["title"] = titleText.toString().stripAndCollapseHtmlWhitespace() + // as `document.title` reads it, edges trimmed (normalizeTitle) + metadata["title"] = titleText.toString().normalizeTitle() titleFromElement = true } } @@ -308,8 +308,8 @@ public fun Flow<SemanticEvent>.simplifyHtml( && !isApplicationStateMeta(content) ) { val value = when (normalizedName) { - // as a <title> element's text reads (document.title) - "title" -> content.stripAndCollapseHtmlWhitespace() + // as a <title> element's text reads + "title" -> content.normalizeTitle() // as <html lang> is read above "lang" -> content.stripHtmlWhitespace() else -> content diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 194825d..11e352c 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -217,11 +217,12 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should keep a non-breaking space inside the derived title`() = runTest { - // given — NBSP is content, not HTML whitespace, as for a <title> - // read by simplifyHtml; an NBSP-only heading still yields no title + fun `should keep a non-breaking space inside the derived title trimming the edges`() = runTest { + // given — NBSP inside is content, not HTML whitespace, as for a + // <title> read by simplifyHtml; at the edges (typically after an + // icon) it is padding, which would only force the title into quotes val input = semanticEvents { - "h1" { +"\u00A0Foo\u00A0Bar\n" } + "h1" { +"\u00A0Foo\u00A0Bar\u00A0\n" } } // when @@ -230,9 +231,9 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"\u00A0Foo\u00A0Bar" } + "entry"("key" to "title") { +"Foo\u00A0Bar" } } - "h1" { +"\u00A0Foo\u00A0Bar\n" } + "h1" { +"\u00A0Foo\u00A0Bar\u00A0\n" } } } @@ -1032,6 +1033,38 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should put a derived title after a later nested title entry`() = runTest { + // given — `title:` then `title: {en: Hello}`; a front matter reader + // keeps the later duplicate, so the derived title in the first slot + // would be hidden by the mapping wrapInHtmlDocument skips + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "title") { +"Heading" } + } + "h1" { +"Heading" } + } + } + @Test fun `should treat a tagged frontmatter as content`() = runTest { // given — a literal `<frontmatter>` tag, not front matter diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 442f893..d1f184d 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1496,12 +1496,14 @@ class SimplifyHtmlTest { } @Test - fun `should keep non-breaking spaces in a title`() = runTest { - // given — NBSP is content, not HTML whitespace, like in a meta value + fun `should keep non-breaking spaces inside a title trimming the edges`() = runTest { + // given — NBSP inside is content, not HTML whitespace, like in a meta + // value; at the edges it is padding, which would only force the + // title into quotes val input = semanticEvents(tagged = true) { "html" { "head" { - "title" { +" \u00A0Page\u00A0\n" } + "title" { +" \u00A0Page\u00A0One\u00A0\n" } "title" { +"Later" } } "body" { "p" { +"x" } } @@ -1514,7 +1516,7 @@ class SimplifyHtmlTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"\u00A0Page\u00A0" } + "entry"("key" to "title") { +"Page\u00A0One" } } "p" { +"x" } } From 2ef66f0af9491e4c5a5b834263b8d2e2b3f0aee2 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 21:51:06 +0200 Subject: [PATCH 16/25] Keep a single title entry; judge blanks by HTML whitespace (#82) - ensureFrontmatterTitle leaves exactly one title entry, spelled `title`, whenever a title comes out, dropping every other title variant: js-yaml (gray-matter) rejects duplicate keys outright, so reordering duplicates for later-wins readers was not enough - ensureFrontmatterTitle and wrapInHtmlDocument treat only HTML whitespace ahead of the frontmatter as insignificant (an NBSP is content) - isMetadataValue rejects zero-width / format chars as well as whitespace; simplifyHtml judges <title> text with it after normalization - isApplicationStateMeta: a flat JSON array of scalars is metadata whatever its bare words parse as; the length backstop judges a value as written (not percent-decoded) and drops an over-long blob unparsed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 4 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 48 +++-- .../kotlin/EnsureFrontmatterTitle.kt | 164 ++++++------------ .../src/commonMain/kotlin/HeadMetadata.kt | 7 +- .../src/commonMain/kotlin/SimplifyHtml.kt | 32 ++-- .../commonMain/kotlin/WrapInHtmlDocument.kt | 14 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 140 ++++++++++----- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 86 +++++++-- .../kotlin/WrapInHtmlDocumentTest.kt | 28 +++ 9 files changed, 304 insertions(+), 219 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4fa0067..1008865 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -561,10 +561,10 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' - `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc; the shared rules are `isScalarEntryType` / `isMetadataValue`), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. - Both treat blank text ahead of the frontmatter as insignificant; `ensureFrontmatterTitle` additionally emits the frontmatter (held or synthesized) **ahead** of that text, since rendered Markdown front matter must open the document. + Both treat text of **HTML** whitespace ahead of the frontmatter as insignificant (`isHtmlBlank`, not `isBlank` — an NBSP is content and makes the frontmatter ordinary content); `ensureFrontmatterTitle` additionally emits the frontmatter (held or synthesized) **ahead** of that text, since rendered Markdown front matter must open the document. Duplicate keys resolve the way readers of the **source format** resolve them, so the two directions deliberately differ: `simplifyHtml` keeps the first `<meta>` of a name (HTML), `wrapInHtmlDocument` the later entry (Psych/PyYAML), the lowercase spelling beating a case variant (`HeadMetadata.addFromHtml` / `addFromFrontMatter`). Do not "align" them for the round-trip's sake — each side emits one entry per name, so neither ever sees the other's duplicates; aligning them once made `wrapInHtmlDocument` show a title no front matter reader shows. - For the same reason `ensureFrontmatterTitle` edits a frontmatter holding a usable title so a case-sensitive, later-wins reader reads that title too (see its KDoc). + For the same reason `ensureFrontmatterTitle` leaves **exactly one** title entry (any letter case), spelled `title`, whenever a title comes out — dropping every other title variant, even a nested one — rather than reordering duplicates for later-wins readers: js-yaml (gray-matter: Eleventy, Astro, Gatsby) rejects a duplicate key outright and loses the whole front matter. ## Test conventions diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index a105db0..07141f7 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -20,7 +20,6 @@ import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonPrimitive @@ -30,32 +29,39 @@ import kotlinx.serialization.json.jsonPrimitive // `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A // name denylist cannot keep up with names private to each site's framework, // so this judges the *value*, which is what tells metadata apart from state. -// A percent-encoded value is judged by what it decodes to, by every rule -// below, so the verdict never depends on the encoding: +// A percent-encoded value's structure is judged by what it decodes to, so +// encoding state does not hide it: // - a value that parses as a JSON object. Parsing, not a look at the first // and last char, is what keeps human text that merely starts with a // bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); -// - a JSON array holding anything but strings and numbers (an object, a -// nested array, a flag, a `null`). A flat list of words or numbers — -// an empty one included — is metadata a person writes (`keywords`, -// `article:tag`, `citation_volume`); +// - a JSON array holding an object or a nested array. A flat list of +// scalars — an empty one included — is metadata a person writes +// (`keywords`, `article:tag`, `citation_volume`), whether a bare word in it +// reads as a string, a number, a flag or `null`; // - a JSON string whose content is itself state by these rules — state // serialised twice, a common single-page-app double encoding — whether // it stands alone or as an element of an array; // - a value longer than [MAX_META_VALUE_LENGTH] that does not read as text — // the backstop for opaque blobs of any other shape (base64, hash lists, -// truncated JSON), which a long abstract in prose is not. A flat JSON -// array is read by its elements, not the quotes and commas serialising -// them, so a long list of words is kept while a long list of hashes is not. +// truncated JSON), which a long abstract in prose is not. It judges the +// value as it will be written, so a percent-encoded one by its escapes, +// never by what they decode to. A flat JSON array is read by its elements, +// not the quotes and commas serialising them, so a long list of words is +// kept while a long list of hashes is not. internal fun isApplicationStateMeta(content: String): Boolean { - val value = content.stripHtmlWhitespace().let { - if (it.firstOrNull() == '%') it.percentDecodedOrNull()?.stripHtmlWhitespace() ?: it else it - } + val written = content.stripHtmlWhitespace() + val decoded = if (written.firstOrNull() == '%') written.percentDecodedOrNull()?.stripHtmlWhitespace() else null + val value = decoded ?: written + val long = written.length > MAX_META_VALUE_LENGTH + // An over-long value is kept only if it reads as text, whatever it + // parses as, so a blob — megabytes of JSON, say — is dropped unparsed. + // Only a raw array must be parsed first, to be read by its words. + val readByWords = decoded == null && value.firstOrNull() == '[' + if (long && !readByWords && !written.readsAsText()) return true val json = value.parseJsonCandidateOrNull() if (json?.isState() == true) return true - // a flat array's words are never longer than the value serialising them - if (value.length <= MAX_META_VALUE_LENGTH) return false - val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else value + if (!long || !readByWords) return false + val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else written return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() } @@ -74,7 +80,7 @@ private fun String.parseJsonCandidateOrNull(): JsonElement? { // logarithmic in the value's length. private fun JsonElement.isState(): Boolean = when (this) { is JsonObject -> true - is JsonArray -> any { !it.isWordOrNumber() || it.isEncodedState() } + is JsonArray -> any { it !is JsonPrimitive || it.isEncodedState() } is JsonPrimitive -> isEncodedState() } @@ -132,14 +138,6 @@ private fun String.parseJsonOrNull(): JsonElement? = try { null } -// kotlinx's tree reader is no strict validator: it takes any unquoted token -// as a literal (`[PDF]` parses as an array, `{"a":abc}` as an object) and -// never checks a number, so the verdict rests on the element's shape, never -// on "it parsed". Everything that is not a string, a flag or `null` counts — -// a number, or a word that no JSON writer would have produced. -private fun JsonElement.isWordOrNumber(): Boolean = - this is JsonPrimitive && this !is JsonNull && (isString || content != "true" && content != "false") - // Decodes `%XX` escapes as UTF-8, or null when the value is not // percent-encoded text (a malformed escape, a raw non-ASCII char). private fun String.percentDecodedOrNull(): String? { diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index f35c21e..fc5332a 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -19,6 +19,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase +import com.xemantic.markanywhere.html.spec.isHtmlBlank import kotlinx.coroutines.flow.Flow /** @@ -26,46 +27,34 @@ import kotlinx.coroutines.flow.Flow * deriving a missing title from the first `h1`. * * The frontmatter judged is the one [wrapInHtmlDocument] reads: an untagged - * `frontmatter` mark opening the stream, blank text before it being - * insignificant — the frontmatter is emitted ahead of that text, so it opens - * the stream as Markdown front matter must; one anywhere else, or a tagged - * one, is ordinary content. Its title entries are its top-level `entry` marks with - * `key="title"` in any ASCII letter case, and one holding non-blank scalar - * text is usable — entries holding a nested structure, an empty collection, - * `null` or blank text are not, as there. + * `frontmatter` mark opening the stream, text of HTML whitespace before it + * being insignificant — the frontmatter is emitted ahead of that text, so it + * opens the stream as Markdown front matter must; one anywhere else, or a + * tagged one, is ordinary content. Its title entries are its top-level + * `entry` marks with `key="title"` in any ASCII letter case, and one holding + * scalar text with visible content is usable — entries holding a nested + * structure, an empty collection, `null` or blank text are not, as there. * - * With a usable title entry the frontmatter passes through, edited only so - * that a front matter reader — matching keys case-sensitively and keeping - * the later of duplicate keys — reads the title [wrapInHtmlDocument] reads: - * every blank or `null` entry spelled `title` is dropped; when the usable - * entry [wrapInHtmlDocument] picks is spelled `title` and followed by one - * holding a nested structure or an empty collection, it is moved right after - * that one; and when it is spelled in another letter case (`Title`) its key - * is respelled `title` — unless an entry spelled `title` holding a nested - * structure or an empty collection is present, which the respelling would - * duplicate. + * Whenever a title comes out, the frontmatter holds exactly one title entry, + * spelled `title`: every front matter reader then reads the same title — + * whether it matches keys case-sensitively (Jekyll) or not, keeps the later + * of duplicate keys (Psych, PyYAML) or rejects them (js-yaml) — and it is the + * one [wrapInHtmlDocument] reads. With a usable title entry that is the one + * [wrapInHtmlDocument] picks, respelled `title` in place; every other title + * entry is dropped. * * Otherwise the frontmatter is held back and the title is derived from the - * very first `h1` following it (only blank text may intervene): the `h1` - * subtree's flattened text — its text events plus the `alt` of every `img` - * mark, in document order, the way an accessible name is computed from + * very first `h1` following it (only HTML whitespace text may intervene): the + * `h1` subtree's flattened text — its text events plus the `alt` of every + * `img` mark, in document order, the way an accessible name is computed from * content — with its HTML whitespace stripped and collapsed, as - * `document.title` reads a `<title>`, and any non-breaking space at its edges - * trimmed, becomes the `title`, and only then the - * frontmatter and the buffered `h1` are emitted, in source order. The - * derived title replaces, in place, the first blank or `null` (a bare - * `title:` line) entry spelled `title` — dropping every other one, as above - * — else the first blank scalar title entry in another spelling, else the - * first `null` one, spelling its key `title` on the same terms as above — - * or, spelled `title`, it goes right after a later entry spelled `title` - * holding a nested structure or an empty collection, as a usable one is - * moved; without any it is injected as the first `entry` of the frontmatter — - * unless an entry spelled `title` holding a nested structure or an empty - * collection is present, which is then left to stand alone: replacing it - * would lose content, and a second entry would duplicate its key. When no - * frontmatter exists at all, one carrying just the derived `title` is - * synthesized as the **first** event — ahead of any blank text that - * preceded the `h1`. + * `document.title` reads a `<title>`, and any other whitespace at its edges + * trimmed, becomes the `title`, and only then the frontmatter and the + * buffered `h1` are emitted, in source order. The derived entry takes the + * place of the first title entry, every other one dropped, or is injected as + * the first `entry` of a frontmatter without any. When no frontmatter exists + * at all, one carrying just the derived `title` is synthesized as the + * **first** event — ahead of any whitespace text that preceded the `h1`. * * When no title can be derived — the first non-blank event after the * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — @@ -94,10 +83,6 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s var openTitleHasChildren = false val openTitleText = StringBuilder() - // how a derived title goes into the held frontmatter, decided once it - // closes; null while none is held - var derivation: Derivation? = null - // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit val blanks = mutableListOf<SemanticEvent>() @@ -135,41 +120,25 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushBlanks() } - // the events of a slot's entry within the held frontmatter - fun span(slot: TitleSlot): List<SemanticEvent> = frontmatterEvents!!.subList(slot.start, slot.end + 1) - - // the rewrite putting `entry`, keyed `key`, in place of `slot` — or, - // spelled `title` and followed by an unreadable entry spelled `title` - // which a later-wins reader would read instead, right after that one - fun placed(slot: TitleSlot, entry: List<SemanticEvent>, key: String): Map<TitleSlot, List<SemanticEvent>> { - val shadowing = titleSlots - .lastOrNull { it.kind == UNREADABLE && it.key == "title" } - ?.takeIf { key == "title" && it.start > slot.start } - return if (shadowing == null) { - mapOf(slot to entry) - } else { - mapOf(slot to emptyList(), shadowing to span(shadowing) + entry) - } + // emits the held frontmatter with `entry` in place of `slot`, every + // other title entry dropped + suspend fun flushWithSingleTitle(slot: TitleSlot, entry: List<SemanticEvent>) { + flushHeld(titleSlots.associateWith { if (it == slot) entry else emptyList() }) } - fun dropping(slots: List<TitleSlot>): Map<TitleSlot, List<SemanticEvent>> = - slots.associateWith { emptyList() } - suspend fun commitHeading() { val title = headingText.toString().normalizeTitle() - val plan = derivation + val slot = titleSlots.firstOrNull() when { !isMetadataValue(title) -> flushHeld() frontmatterEvents == null -> { "frontmatter" { - emit(titleEntry(title, "title")) + emit(titleEntry(title)) } flushBlanks() } - // a single title entry prepended to the frontmatter's content - plan!!.slot == null -> flushHeld(prepended = titleEntry(title, plan.key)) - // the derived entry in place of the unusable title entry - else -> flushHeld(dropping(plan.dropped) + placed(plan.slot, titleEntry(title, plan.key), plan.key)) + slot == null -> flushHeld(prepended = titleEntry(title)) + else -> flushWithSingleTitle(slot, titleEntry(title)) } emit(headingEvents) headingEvents.clear() @@ -178,34 +147,15 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // decides, once the frontmatter closes, whether it holds a usable title suspend fun judgeFrontmatter() { - // an unreadable entry spelled `title`, which a respelled key would duplicate - val titleKeyTaken = titleSlots.any { it.kind == UNREADABLE && it.key == "title" } - val emptyTitles = titleSlots.filter { it.key == "title" && (it.kind == BLANK || it.kind == NULL) } // the entry wrapInHtmlDocument reads val usable = titleSlots - .filter { it.kind == USABLE } + .filter { it.usable } .reduceOrNull { read, next -> if (frontMatterKeySupersedes(read.key, next.key)) next else read } if (usable != null) { - val key = if (titleKeyTaken) usable.key else "title" - val entry = if (key == usable.key) span(usable) else respelled(span(usable), key) - flushHeld(dropping(emptyTitles) + placed(usable, entry, key)) - state = PassThrough - return - } - val slot = emptyTitles.firstOrNull() - ?: titleSlots.firstOrNull { it.kind == BLANK } - ?: titleSlots.firstOrNull { it.kind == NULL } - // with no slot to replace, an entry spelled `title` left is - // unreadable: injecting would duplicate its key - if (slot == null && titleKeyTaken) { - flushHeld() + val events = frontmatterEvents!!.subList(usable.start, usable.end + 1) + flushWithSingleTitle(usable, respelled(events)) state = PassThrough } else { - derivation = Derivation( - slot = slot, - key = if (slot != null && titleKeyTaken) slot.key else "title", - dropped = emptyTitles.filter { it != slot } - ) state = AwaitingHeading } } @@ -227,7 +177,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) - event is Text && event.text.isBlank() -> blanks += event + event is Text && event.text.isHtmlBlank() -> blanks += event else -> { flushBlanks() emit(event) @@ -257,20 +207,16 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s 1 -> openTitle?.let { open -> openTitle = null val type = open["type"] - val kind: TitleKind = when { - openTitleHasChildren || type != "null" && !isScalarEntryType(type) -> UNREADABLE - type == "null" -> NULL - isMetadataValue(openTitleText.toString()) -> USABLE - else -> BLANK - } - titleSlots += TitleSlot(openTitleStart, events.lastIndex, open["key"]!!, kind) + val usable = !openTitleHasChildren && isScalarEntryType(type) && + isMetadataValue(openTitleText.toString()) + titleSlots += TitleSlot(openTitleStart, events.lastIndex, open["key"]!!, usable) } 0 -> judgeFrontmatter() } } } AwaitingHeading -> when (event) { - is Text if event.text.isBlank() -> blanks += event + is Text if event.text.isHtmlBlank() -> blanks += event is Mark if event.name == "h1" -> startHeading(event) else -> { flushHeld() @@ -313,30 +259,20 @@ private enum class State { AtStart, InFrontmatter, AwaitingHeading, InHeading, PassThrough } -// How a title entry reads: a non-blank scalar, blank text, `null`, or a -// nested structure or an empty collection. -private enum class TitleKind { - USABLE, BLANK, NULL, UNREADABLE -} - // A top-level title entry: the span of its events within the held -// frontmatter, its key as spelled, and how it reads. -private data class TitleSlot(val start: Int, val end: Int, val key: String, val kind: TitleKind) - -// How a derived title goes into the held frontmatter: in place of `slot` -// (null: prepended to the frontmatter's entries), keyed `key`, the `dropped` -// empty title entries removed. -private class Derivation(val slot: TitleSlot?, val key: String, val dropped: List<TitleSlot>) +// frontmatter, its key as spelled, and whether it holds a usable title. +private data class TitleSlot(val start: Int, val end: Int, val key: String, val usable: Boolean) -// A top-level front matter entry keyed `key` holding the scalar `value`. -private fun titleEntry(value: String, key: String): List<SemanticEvent> = listOf( - SemanticEvent.Mark("entry", attributes = mapOf("key" to key)), +// A top-level front matter `title` entry holding the scalar `value`. +private fun titleEntry(value: String): List<SemanticEvent> = listOf( + SemanticEvent.Mark("entry", attributes = mapOf("key" to "title")), SemanticEvent.Text(value), SemanticEvent.Unmark("entry") ) -// An entry's events with its key spelled `key`. -private fun respelled(entry: List<SemanticEvent>, key: String): List<SemanticEvent> { +// An entry's events with its key spelled `title`. +private fun respelled(entry: List<SemanticEvent>): List<SemanticEvent> { val mark = entry.first() as Mark - return listOf(mark.copy(attributes = mark.attributes + ("key" to key))) + entry.drop(1) + return if (mark["key"] == "title") entry + else listOf(mark.copy(attributes = mark.attributes + ("key" to "title"))) + entry.drop(1) } diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index f69f251..96f3801 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -34,8 +34,11 @@ private val SCALAR_ENTRY_TYPES = setOf("bool", "int", "float", "timestamp") internal fun isScalarEntryType(type: String?): Boolean = type == null || type in SCALAR_ENTRY_TYPES -// Whether a value carries anything for a reader — a blank one does not. -internal fun isMetadataValue(value: String): Boolean = value.isNotBlank() +// Whether a value carries anything for a reader: a char that shows — not +// whitespace (NBSP included) and not an invisible format char such as a +// zero-width space or a byte order mark. +internal fun isMetadataValue(value: String): Boolean = + value.any { !it.isWhitespace() && it.category != CharCategory.FORMAT } // A title as `document.title` reads a `<title>` — HTML whitespace stripped // and collapsed — with any other whitespace at its edges (the NBSP padding an diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index ecdc6d7..850dee8 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -131,12 +131,16 @@ import kotlinx.coroutines.flow.Flow * and emitted as a single synthetic `frontmatter` mark holding one `entry` * (`key` attribute, value as text) per item — the parser's structured front * matter vocabulary — just before `<body>` content streams through. Technical meta - * names that carry no content signal (rendering hints, crawler / verification - * directives, platform tile metadata — see [isNoiseMetaName]) are dropped so - * they don't inflate the frontmatter, and so are a blank value (NBSP-only - * included, `<html lang>` too) and - * application state that single-page apps ship in `<meta>` (serialised JSON, - * framework config blobs — see [isApplicationStateMeta]). Meta names are + * names that carry no content signal (rendering hints such as `viewport` and + * `theme-color`, crawler / verification directives such as `robots` and + * `*-verification`, platform tile metadata such as `msapplication-*` and + * `apple-*`) are dropped so they don't inflate the frontmatter, and so are a + * value with nothing visible in it (whitespace, NBSP included, or invisible + * format chars such as a zero-width space — `<html lang>` too) and + * application state that single-page apps ship in `<meta>`: a JSON object, a + * JSON array holding an object or a nested array, JSON state serialised into + * a JSON string — raw or percent-encoded — and a value over 4096 chars that, + * as written, does not read as text. Meta names are * ASCII case-insensitive, so of several names differing only in letter case * the first one (spelling and value) wins, as HTML resolves duplicate * `<meta>` elements — unlike [wrapInHtmlDocument], which resolves duplicate @@ -179,10 +183,11 @@ public fun Flow<SemanticEvent>.simplifyHtml( svgMode: SvgMode = SvgMode.RESOLVE, ): Flow<SemanticEvent> = transform { - // A value that tells a reader nothing is never added — blank judged with - // Unicode whitespace (NBSP included), not HTML's: rendered, an NBSP-only - // value is as empty as a blank one. `ensureFrontmatterTitle` judges a - // title the same way, so a title kept here is never replaced there. + // A value that tells a reader nothing is never added — judged by + // `isMetadataValue`, not HTML whitespace: rendered, an NBSP-only or + // zero-width value is as empty as a blank one. `ensureFrontmatterTitle` + // judges a title the same way, so a title kept here is never replaced + // there. val metadata = HeadMetadata() val titleText = StringBuilder() // The first non-blank `<title>` wins, over a `<meta name="title">` too — @@ -288,9 +293,10 @@ public fun Flow<SemanticEvent>.simplifyHtml( titleText.clear() children(mode = "titleText") afterClose { - if (!titleFromElement && titleText.isNotBlank()) { - // as `document.title` reads it, edges trimmed (normalizeTitle) - metadata["title"] = titleText.toString().normalizeTitle() + // as `document.title` reads it, edges trimmed (normalizeTitle) + val title = titleText.toString().normalizeTitle() + if (!titleFromElement && isMetadataValue(title)) { + metadata["title"] = title titleFromElement = true } } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 0bd2d87..395b482 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -19,6 +19,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase +import com.xemantic.markanywhere.html.spec.isHtmlBlank import kotlinx.coroutines.flow.Flow /** @@ -51,15 +52,16 @@ import kotlinx.coroutines.flow.Flow * back spelled in lowercase, the title with its whitespace stripped and * collapsed (as `document.title` reads it) and `lang` trimmed, and a typed * scalar comes back as a string. - * Blank text ahead of the frontmatter is insignificant — it is moved to the - * start of `body`, as [ensureFrontmatterTitle] moves it after the + * Text of HTML whitespace ahead of the frontmatter is insignificant — it is + * moved to the start of `body` (a non-breaking space is content, as + * everywhere in HTML, and opens the body instead), as [ensureFrontmatterTitle] moves it after the * frontmatter. A `frontmatter` mark appearing past any other event is * ordinary content and flows into `body` verbatim. * - * Only leading blank text and the frontmatter subtree are read ahead + * Only leading whitespace text and the frontmatter subtree are read ahead * (bounded); without a frontmatter the document opening is emitted on the - * first non-blank event and body content streams through untouched. All synthetic marks are untagged, consistent with the - * parser's `frontmatter` mark and [simplifyHtml] output. An empty input + * first other event and body content streams through untouched. All + * synthetic marks are untagged, consistent with the parser's `frontmatter` mark and [simplifyHtml] output. An empty input * stream still yields the full document skeleton. */ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = semanticEvents { @@ -133,7 +135,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman event is Mark && !event.isTagged && event.name == "frontmatter" -> { collectingFrontmatter = true } - event is Text && event.text.isBlank() -> blanks += event + event is Text && event.text.isHtmlBlank() -> blanks += event else -> { openDocument() emit(event) diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 11e352c..5d183ca 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -642,7 +642,7 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should pass through a usable first title entry followed by a blank variant`() = runTest { + fun `should drop a blank title variant following a usable title entry`() = runTest { // given val input = semanticEvents { "frontmatter" { @@ -655,26 +655,26 @@ class EnsureFrontmatterTitleTest { // when val output = input.ensureFrontmatterTitle() - // then + // then — a single title entry, which no reader can take for another output sameAs semanticEvents { "frontmatter" { "entry"("key" to "title") { +"Foo" } - "entry"("key" to "TITLE") { } } "h1" { +"Hello" } } } @Test - fun `should keep a title entry holding a nested structure`() = runTest { - // given — not a usable title, but replacing it would lose content + fun `should replace a title entry holding a nested structure with the derived title`() = runTest { + // given — no reader shows a mapping as the page title val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { "entry"("key" to "en") { +"Hello" } } + "entry"("key" to "author") { +"Alice" } } - "h1" { +"Hello" } + "h1" { +"Heading" } } // when @@ -683,11 +683,10 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { - "entry"("key" to "en") { +"Hello" } - } + "entry"("key" to "title") { +"Heading" } + "entry"("key" to "author") { +"Alice" } } - "h1" { +"Hello" } + "h1" { +"Heading" } } } @@ -773,7 +772,7 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should replace a null title entry following a nested title entry`() = runTest { + fun `should replace a nested title entry and a null variant following it with one title`() = runTest { // given val input = semanticEvents { "frontmatter" { @@ -791,19 +790,15 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { - "entry"("key" to "en") { +"Hello" } - } - "entry"("key" to "Title") { +"Heading" } + "entry"("key" to "title") { +"Heading" } } "h1" { +"Heading" } } } @Test - fun `should keep a title entry holding an empty collection`() = runTest { - // given — `title: []`, unreadable as a title but not replaceable - // without losing it + fun `should replace a title entry holding an empty collection with the derived title`() = runTest { + // given — `title: []` val input = semanticEvents { "frontmatter" { "entry"("key" to "title", "type" to "seq") { } @@ -817,19 +812,19 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title", "type" to "seq") { } + "entry"("key" to "title") { +"Heading" } } "h1" { +"Heading" } } } @Test - fun `should replace the slot spelled title over a blank variant preceding it`() = runTest { - // given — `Title: ""` then `title:`; a case-sensitive reader - // (Jekyll, Hugo) reads only the latter + fun `should put the derived title in the first of several empty title entries`() = runTest { + // given — `Title: ""` then `title:` val input = semanticEvents { "frontmatter" { "entry"("key" to "Title") { } + "entry"("key" to "author") { +"Alice" } "entry"("key" to "title", "type" to "null") { } } "h1" { +"Heading" } @@ -841,17 +836,17 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "Title") { } "entry"("key" to "title") { +"Heading" } + "entry"("key" to "author") { +"Alice" } } "h1" { +"Heading" } } } @Test - fun `should inject a title next to an unreadable title variant`() = runTest { - // given — `Title: []`, which wrapInHtmlDocument skips; a `title` entry - // collides with nothing + fun `should replace an unreadable title variant with the derived title`() = runTest { + // given — `Title: []`, which wrapInHtmlDocument skips; a `title` + // entry beside it would duplicate the key for a reader folding case val input = semanticEvents { "frontmatter" { "entry"("key" to "Title", "type" to "seq") { } @@ -866,7 +861,6 @@ class EnsureFrontmatterTitleTest { output sameAs semanticEvents { "frontmatter" { "entry"("key" to "title") { +"Heading" } - "entry"("key" to "Title", "type" to "seq") { } } "h1" { +"Heading" } } @@ -954,10 +948,9 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should move a usable title entry after a later unreadable one`() = runTest { - // given — a front matter reader keeps the later duplicate, so the - // collection would hide the title wrapInHtmlDocument reads; moving the - // usable entry after it keeps both entries + fun `should drop an unreadable title entry following a usable one`() = runTest { + // given — `title: Real` then `title: []`: a later-wins reader would + // read the collection, and js-yaml rejects the duplicate key outright val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { +"Real" } @@ -965,26 +958,25 @@ class EnsureFrontmatterTitleTest { "entry"("key" to "title", "type" to "seq") { } "entry"("key" to "tags") { +"x" } } - "h1" { +"Heading" } + "p" { +"Body." } } // when val output = input.ensureFrontmatterTitle() - // then + // then — decided without an h1, as a usable title needs none output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "author") { +"Alice" } - "entry"("key" to "title", "type" to "seq") { } "entry"("key" to "title") { +"Real" } + "entry"("key" to "author") { +"Alice" } "entry"("key" to "tags") { +"x" } } - "h1" { +"Heading" } + "p" { +"Body." } } } @Test - fun `should respell the last usable title variant as wrapInHtmlDocument reads it`() = runTest { + fun `should keep only the title variant wrapInHtmlDocument reads`() = runTest { // given val input = semanticEvents { "frontmatter" { @@ -1000,7 +992,6 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "Title") { +"First" } "entry"("key" to "title") { +"Last" } } "h1" { +"Heading" } @@ -1034,10 +1025,9 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should put a derived title after a later nested title entry`() = runTest { - // given — `title:` then `title: {en: Hello}`; a front matter reader - // keeps the later duplicate, so the derived title in the first slot - // would be hidden by the mapping wrapInHtmlDocument skips + fun `should leave a single title when deriving it past a later nested title entry`() = runTest { + // given — `title:` then `title: {en: Hello}`; a later-wins reader + // would read the mapping over a derived title in the first slot val input = semanticEvents { "frontmatter" { "entry"("key" to "title", "type" to "null") { } @@ -1055,16 +1045,76 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { + "entry"("key" to "title") { +"Heading" } "entry"("key" to "author") { +"Alice" } - "entry"("key" to "title") { - "entry"("key" to "en") { +"Hello" } - } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should not derive a title from an invisible h1`() = runTest { + // given — a zero-width space shows nothing + val input = semanticEvents { + "h1" { +"\u200B" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "h1" { +"\u200B" } + } + } + + @Test + fun `should treat a zero-width title entry as blank`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\u200B" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { "entry"("key" to "title") { +"Heading" } } "h1" { +"Heading" } } } + @Test + fun `should keep a non-breaking space before the frontmatter as content`() = runTest { + // given — NBSP is content in HTML, so the frontmatter does not open + // the stream + val input = semanticEvents { + +"\u00A0" + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + +"\u00A0" + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "h1" { +"Heading" } + } + } + @Test fun `should treat a tagged frontmatter as content`() = runTest { // given — a literal `<frontmatter>` tag, not front matter diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index d1f184d..daca500 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -851,9 +851,6 @@ class SimplifyHtmlTest { "meta"("name" to "empty-object", "content" to "{}") { } "meta"("name" to "feed/config/environment", "content" to "%7B%22x%22%3A1%7D") { } "meta"("name" to "jam/config/environment", "content" to "%7b%7d") { } - "meta"("name" to "encoded-flags", "content" to "%5Btrue%5D") { } - "meta"("name" to "flags", "content" to "[true,false]") { } - "meta"("name" to "slots", "content" to "[null]") { } "meta"("name" to "spaced", "content" to "%5B%20%7B%22a%22%3A1%7D%20%5D") { } "meta"("name" to "description", "content" to "Save {50%} today") { } } @@ -865,8 +862,7 @@ class SimplifyHtmlTest { val output = input.simplifyHtml() // then — only a value that *is* a JSON object, or an array that is - // not a flat list of words or numbers, raw or percent-encoded, marks - // state + // not a flat list of scalars, raw or percent-encoded, marks state output sameAs semanticEvents { "frontmatter" { "entry"("key" to "description") { +"Save {50%} today" } @@ -1121,16 +1117,46 @@ class SimplifyHtmlTest { } @Test - fun `should judge the length of a percent-encoded value by what it decodes to`() = runTest { - // given — prose whose encoding is past the cap while its text is not, - // and an encoded blob past it either way - val prose = "本研究では大規模言語モデルの挙動を分析した。".repeat(30).percentEncoded() + fun `should judge the length of a percent-encoded value as it is written`() = runTest { + // given — a value is written to the front matter still encoded, so + // prose whose escapes run past the cap is as opaque there as a blob, + // while a short encoded value is kept + val longProse = "本研究では大規模言語モデルの挙動を分析した。".repeat(30).percentEncoded() + val shortProse = "本研究では大規模言語モデルの挙動を分析した。".percentEncoded() val blob = ("A" + "x".repeat(5000)).percentEncoded() val input = semanticEvents(tagged = true) { "html" { "head" { - "meta"("name" to "description", "content" to prose) { } + "meta"("name" to "description", "content" to longProse) { } "meta"("name" to "blob", "content" to blob) { } + "meta"("name" to "abstract", "content" to shortProse) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "abstract") { +shortProse } + } + "p" { +"text" } + } + } + + @Test + fun `should drop a long JSON object even when its content reads as text`() = runTest { + // given — an over-long value that reads as text is still parsed, so + // state wrapping prose is not kept for its prose + val prose = "A study of how language models behave in long dialogues. ".repeat(100) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "state", "content" to "{\"text\": \"$prose\"}") { } + "meta"("name" to "description", "content" to prose) { } } "body" { "p" { +"text" } } } @@ -1523,9 +1549,10 @@ class SimplifyHtmlTest { } @Test - fun `should keep a flat JSON list of words or numbers in frontmatter`() = runTest { + fun `should keep a flat JSON list of scalars in frontmatter`() = runTest { // given — a list a person writes, not serialised state — an empty - // one, or one percent-encoded, included + // one, one percent-encoded, and one whose bare words JSON would read + // as flags or `null` included val input = semanticEvents(tagged = true) { "html" { "head" { @@ -1537,6 +1564,10 @@ class SimplifyHtmlTest { "meta"("name" to "none", "content" to "[]") { } "meta"("name" to "none-spaced", "content" to "[ ]") { } "meta"("name" to "encoded", "content" to "%5B%22a%22%5D") { } + "meta"("name" to "nothing", "content" to "[null]") { } + "meta"("name" to "answer", "content" to "[true]") { } + "meta"("name" to "options", "content" to "[true, false]") { } + "meta"("name" to "encoded-flag", "content" to "%5Btrue%5D") { } } "body" { "p" { +"text" } } } @@ -1556,6 +1587,10 @@ class SimplifyHtmlTest { "entry"("key" to "none") { +"[]" } "entry"("key" to "none-spaced") { +"[ ]" } "entry"("key" to "encoded") { +"%5B%22a%22%5D" } + "entry"("key" to "nothing") { +"[null]" } + "entry"("key" to "answer") { +"[true]" } + "entry"("key" to "options") { +"[true, false]" } + "entry"("key" to "encoded-flag") { +"%5Btrue%5D" } } "p" { +"text" } } @@ -1618,6 +1653,33 @@ class SimplifyHtmlTest { } } + @Test + fun `should skip a title of invisible format chars only`() = runTest { + // given — a zero-width space or a byte order mark is not whitespace, + // yet shows nothing + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"\u200B" } + "meta"("name" to "title", "content" to "Real Page") { } + "meta"("name" to "description", "content" to "\uFEFF\u200B") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real Page" } + } + "p" { +"x" } + } + } + @Test fun `should strip and collapse a title meta like a title element`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 711a024..dcd49fe 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -298,6 +298,34 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should treat a frontmatter preceded by a non-breaking space as content`() = runTest { + // given — NBSP is content in HTML, so the frontmatter does not open + // the stream + val input = semanticEvents { + +"\u00A0" + "frontmatter" { + "entry"("key" to "title") { +"Page" } + } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { } + "body" { + +"\u00A0" + "frontmatter" { + "entry"("key" to "title") { +"Page" } + } + } + } + } + } + @Test fun `should wrap parsed Markdown in a complete HTML document`() = runTest { // given From 66255c28427299b2455a235a39e5ff95802e58ec Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 22:15:17 +0200 Subject: [PATCH 17/25] Share one front matter entry reader; keep nested titles (#82) - wrapInHtmlDocument and ensureFrontmatterTitle read the front matter through one FrontMatterEntryReader instead of private copies - ensureFrontmatterTitle keeps a nested title entry (localized titles) rather than deriving one from the h1, dropping only empty variants - normalizeTitle trims invisible format chars (zero-width space, BOM) at a title's edges; simplifyHtml judges meta blanks by isMetadataValue - isApplicationStateMeta measures a flat JSON array by its elements joined, and drops a long array opening with a nested element unparsed unless it reads as text - document why asciiLowercase differs from lowercase (Kelvin sign) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 7 +- .../src/commonMain/kotlin/AsciiCase.kt | 6 +- .../src/commonTest/kotlin/AsciiCaseTest.kt | 13 +++ .../commonMain/kotlin/ApplicationStateMeta.kt | 25 ++++- .../kotlin/EnsureFrontmatterTitle.kt | 82 ++++++-------- .../kotlin/FrontMatterEntryReader.kt | 102 ++++++++++++++++++ .../src/commonMain/kotlin/HeadMetadata.kt | 18 ++-- .../src/commonMain/kotlin/SimplifyHtml.kt | 13 ++- .../commonMain/kotlin/WrapInHtmlDocument.kt | 48 ++------- .../kotlin/EnsureFrontmatterTitleTest.kt | 63 +++++++---- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 79 ++++++++++++++ 11 files changed, 325 insertions(+), 131 deletions(-) create mode 100644 markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt diff --git a/CLAUDE.md b/CLAUDE.md index 1008865..3b3b1a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -558,13 +558,14 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Key typing is not value typing: go-yaml v2 decodes a **top-level** front matter key into a string but a nested one as `interface{}`, so `y:` must stay plain at the top level (it was once rewritten to `"y":`) and be quoted below it (`isTypedPlainKey`). The writer's plain-key rule (`isIdentifierKey`) and the detector share their character rules in `YamlKeyLine.kt`, and `YamlWriterTest` "should write every key as a line the front matter detector accepts" pins the two — widen one without the other and every document whose *first* entry has such a key silently loses its front matter on re-parse. **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. -- `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter by **depth** (a direct child `entry` is depth 1 / 2 respectively, counted from the `frontmatter` mark): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. - Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader. - The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc; the shared rules are `isScalarEntryType` / `isMetadataValue`), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads. +- `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter through **one** reader, `FrontMatterEntryReader` (top-level `entry` marks by depth, `FrontMatterEntry.isHeadMetadata` for what counts): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. + Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader, and do not give either operator a private copy of the entry reader again. + The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads — hand-rolled copies of the reader in each operator were how the two drifted apart. Both treat text of **HTML** whitespace ahead of the frontmatter as insignificant (`isHtmlBlank`, not `isBlank` — an NBSP is content and makes the frontmatter ordinary content); `ensureFrontmatterTitle` additionally emits the frontmatter (held or synthesized) **ahead** of that text, since rendered Markdown front matter must open the document. Duplicate keys resolve the way readers of the **source format** resolve them, so the two directions deliberately differ: `simplifyHtml` keeps the first `<meta>` of a name (HTML), `wrapInHtmlDocument` the later entry (Psych/PyYAML), the lowercase spelling beating a case variant (`HeadMetadata.addFromHtml` / `addFromFrontMatter`). Do not "align" them for the round-trip's sake — each side emits one entry per name, so neither ever sees the other's duplicates; aligning them once made `wrapInHtmlDocument` show a title no front matter reader shows. For the same reason `ensureFrontmatterTitle` leaves **exactly one** title entry (any letter case), spelled `title`, whenever a title comes out — dropping every other title variant, even a nested one — rather than reordering duplicates for later-wins readers: js-yaml (gray-matter: Eleventy, Astro, Gatsby) rejects a duplicate key outright and loses the whole front matter. + A **nested** title (a mapping of localized titles) with no usable title beside it is the exception: it is no title for `wrapInHtmlDocument`, but deriving one from the `h1` would delete it, so no title comes out and only the *empty* title entries are dropped. ## Test conventions diff --git a/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt index a6cc6d2..c5757e1 100644 --- a/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt +++ b/markanywhere-html-spec/src/commonMain/kotlin/AsciiCase.kt @@ -30,8 +30,10 @@ public fun Char.asciiLowercase(): Char = /** * This string with every ASCII upper alpha replaced by its lowercase - * counterpart (see [Char.asciiLowercase]) — unlike [String.lowercase], `tıtle` - * (a dotless ı) stays distinct from `title`. + * counterpart (see [Char.asciiLowercase]) — unlike [String.lowercase], which + * folds some non-ASCII letters into ASCII ones: `"\u212Aey".lowercase()` + * (a Kelvin sign `K`, U+212A) is `"key"`, while this leaves it unchanged, so + * it never matches a `key` name. */ public fun String.asciiLowercase(): String = if (none { it in 'A'..'Z' }) this else CharArray(length) { this[it].asciiLowercase() }.concatToString() diff --git a/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt index 1634f59..548d6fb 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt @@ -43,4 +43,17 @@ class AsciiCaseTest { assert(umlaut == "Ärger") assert(kelvin == 'K') } + + @Test + fun `should not fold a Kelvin sign into an ASCII k as lowercase does`() { + // given + val kelvinKey = "\u212Aey" + + // when + val lowered = kelvinKey.asciiLowercase() + + // then + assert(kelvinKey.lowercase() == "key") + assert(lowered == kelvinKey) + } } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 07141f7..2790a1a 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -45,9 +45,11 @@ import kotlinx.serialization.json.jsonPrimitive // the backstop for opaque blobs of any other shape (base64, hash lists, // truncated JSON), which a long abstract in prose is not. It judges the // value as it will be written, so a percent-encoded one by its escapes, -// never by what they decode to. A flat JSON array is read by its elements, -// not the quotes and commas serialising them, so a long list of words is -// kept while a long list of hashes is not. +// never by what they decode to. A flat JSON array is measured by its +// elements joined, not the quotes and commas serialising them — its length +// as well as whether it reads as text — so it is judged as the same list +// written plainly would be: a long list of words is kept, a long list of +// hashes is not, and one whose elements fit under the cap is short. internal fun isApplicationStateMeta(content: String): Boolean { val written = content.stripHtmlWhitespace() val decoded = if (written.firstOrNull() == '%') written.percentDecodedOrNull()?.stripHtmlWhitespace() else null @@ -55,12 +57,16 @@ internal fun isApplicationStateMeta(content: String): Boolean { val long = written.length > MAX_META_VALUE_LENGTH // An over-long value is kept only if it reads as text, whatever it // parses as, so a blob — megabytes of JSON, say — is dropped unparsed. - // Only a raw array must be parsed first, to be read by its words. + // Only a raw array must be parsed first, to be read by its words — unless + // its first element is an object or an array: then it is state if it + // parses and a blob if it does not, dropped either way. val readByWords = decoded == null && value.firstOrNull() == '[' - if (long && !readByWords && !written.readsAsText()) return true + if (long && (!readByWords || value.opensNestedElement()) && !written.readsAsText()) return true val json = value.parseJsonCandidateOrNull() if (json?.isState() == true) return true if (!long || !readByWords) return false + // a flat array is measured by its elements joined, in length as in text, + // as the same elements written as a plain list would be val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else written return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() } @@ -69,6 +75,15 @@ internal fun isApplicationStateMeta(content: String): Boolean { // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 +// Whether this array's first element opens an object or an array. +private fun String.opensNestedElement(): Boolean { + var i = 1 + while (i < length && this[i] in JSON_WHITESPACE) i++ + return i < length && (this[i] == '{' || this[i] == '[') +} + +private const val JSON_WHITESPACE = " \t\n\r" + // Only a value opening like a JSON object, array or string is parsed. private fun String.parseJsonCandidateOrNull(): JsonElement? { val first = firstOrNull() diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index fc5332a..1ab4b37 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -35,6 +35,13 @@ import kotlinx.coroutines.flow.Flow * scalar text with visible content is usable — entries holding a nested * structure, an empty collection, `null` or blank text are not, as there. * + * A title entry holding a nested structure (localized titles, say) is no + * title for [wrapInHtmlDocument], but it is content: with no usable title + * entry beside it, no title is derived, and the frontmatter passes through + * with only its empty title entries — `null`, blank, an empty collection — + * dropped, as they carry nothing and a duplicate key makes js-yaml reject the + * whole front matter. + * * Whenever a title comes out, the frontmatter holds exactly one title entry, * spelled `title`: every front matter reader then reads the same title — * whether it matches keys case-sensitively (Jekyll) or not, keeps the later @@ -71,17 +78,13 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // the held frontmatter subtree (mark + body events + unmark); null when // no frontmatter was present and a default one is to be synthesized var frontmatterEvents: MutableList<SemanticEvent>? = null - var frontmatterDepth = 0 // the top-level title entries (in any letter case) within - // `frontmatterEvents`, in source order - val titleSlots = mutableListOf<TitleSlot>() - // the top-level title entry currently open: its mark and start, its - // text, and whether it holds nested marks - var openTitle: SemanticEvent.Mark? = null - var openTitleStart = -1 - var openTitleHasChildren = false - val openTitleText = StringBuilder() + // `frontmatterEvents`, in source order, their spans indexing it + val titleSlots = mutableListOf<FrontMatterEntry>() + val reader = FrontMatterEntryReader { entry -> + if (entry.key.asciiLowercase() == "title") titleSlots += entry + } // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit @@ -102,7 +105,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // of each slot in `rewrite` replaced by its events (none drops the entry) // — then the blanks held with it suspend fun flushHeld( - rewrite: Map<TitleSlot, List<SemanticEvent>> = emptyMap(), + rewrite: Map<FrontMatterEntry, List<SemanticEvent>> = emptyMap(), prepended: List<SemanticEvent> = emptyList() ) { frontmatterEvents?.let { held -> @@ -122,7 +125,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // emits the held frontmatter with `entry` in place of `slot`, every // other title entry dropped - suspend fun flushWithSingleTitle(slot: TitleSlot, entry: List<SemanticEvent>) { + suspend fun flushWithSingleTitle(slot: FrontMatterEntry, entry: List<SemanticEvent>) { flushHeld(titleSlots.associateWith { if (it == slot) entry else emptyList() }) } @@ -149,14 +152,20 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s suspend fun judgeFrontmatter() { // the entry wrapInHtmlDocument reads val usable = titleSlots - .filter { it.usable } + .filter { it.isHeadMetadata } .reduceOrNull { read, next -> if (frontMatterKeySupersedes(read.key, next.key)) next else read } - if (usable != null) { - val events = frontmatterEvents!!.subList(usable.start, usable.end + 1) - flushWithSingleTitle(usable, respelled(events)) - state = PassThrough - } else { - state = AwaitingHeading + state = when { + usable != null -> { + val events = frontmatterEvents!!.subList(usable.start, usable.end + 1) + flushWithSingleTitle(usable, respelled(events)) + PassThrough + } + // a nested title is content, not ours to replace + titleSlots.any { it.hasChildren } -> { + flushHeld(titleSlots.filterNot { it.hasChildren }.associateWith { emptyList() }) + PassThrough + } + else -> AwaitingHeading } } @@ -173,7 +182,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // is the page's metadata; anywhere else it is content event is Mark && !event.isTagged && event.name == "frontmatter" -> { frontmatterEvents = mutableListOf(event) - frontmatterDepth = 1 + reader.read(event) state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) @@ -185,35 +194,8 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } } InFrontmatter -> { - val events = frontmatterEvents!! - events += event - when (event) { - is Mark -> { - frontmatterDepth++ - if (frontmatterDepth == 2) { - // a top-level entry (a direct child) - if (event.name == "entry" && event["key"]?.asciiLowercase() == "title") { - openTitle = event - openTitleStart = events.lastIndex - openTitleHasChildren = false - openTitleText.clear() - } - } else if (openTitle != null) { - openTitleHasChildren = true - } - } - is Text -> if (openTitle != null && frontmatterDepth == 2) openTitleText.append(event.text) - is Unmark -> when (--frontmatterDepth) { - 1 -> openTitle?.let { open -> - openTitle = null - val type = open["type"] - val usable = !openTitleHasChildren && isScalarEntryType(type) && - isMetadataValue(openTitleText.toString()) - titleSlots += TitleSlot(openTitleStart, events.lastIndex, open["key"]!!, usable) - } - 0 -> judgeFrontmatter() - } - } + frontmatterEvents!! += event + if (reader.read(event)) judgeFrontmatter() } AwaitingHeading -> when (event) { is Text if event.text.isHtmlBlank() -> blanks += event @@ -259,10 +241,6 @@ private enum class State { AtStart, InFrontmatter, AwaitingHeading, InHeading, PassThrough } -// A top-level title entry: the span of its events within the held -// frontmatter, its key as spelled, and whether it holds a usable title. -private data class TitleSlot(val start: Int, val end: Int, val key: String, val usable: Boolean) - // A top-level front matter `title` entry holding the scalar `value`. private fun titleEntry(value: String): List<SemanticEvent> = listOf( SemanticEvent.Mark("entry", attributes = mapOf("key" to "title")), diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt new file mode 100644 index 0000000..2ac519a --- /dev/null +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Kazimierz Pogoda / Xemantic + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xemantic.markanywhere.html + +import com.xemantic.markanywhere.SemanticEvent + +// A top-level front matter `entry`: its key as spelled, `type`, text, whether +// it holds nested marks, and the span of its events among those read. +internal class FrontMatterEntry( + val key: String, + val type: String?, + val text: String, + val hasChildren: Boolean, + val start: Int, + val end: Int +) { + + // Whether it holds head metadata: scalar text with visible content — + // not a nested structure, an empty collection, `null` or blank text. + val isHeadMetadata: Boolean + get() = !hasChildren && isScalarEntryType(type) && isMetadataValue(text) + +} + +// Reads the top-level entries of a `frontmatter` subtree, reporting each to +// `onEntry` as it closes. The one reader of front matter as head metadata — +// wrapInHtmlDocument turns the entries into `<head>`, ensureFrontmatterTitle +// judges the title entries among them, and the two must agree on both. +internal class FrontMatterEntryReader( + private val onEntry: (FrontMatterEntry) -> Unit +) { + + // 1 inside the frontmatter, 2 inside a top-level entry + private var depth = 0 + private var index = -1 + private var open: SemanticEvent.Mark? = null + private var openStart = 0 + private var openHasChildren = false + private val openText = StringBuilder() + + // Reads the next event of the subtree, the frontmatter mark first; true + // once the frontmatter's own unmark is read. + fun read(event: SemanticEvent): Boolean { + index++ + when (event) { + is Mark -> { + depth++ + if (depth == 2) { + open = if (event.name == "entry" && event["key"] != null) event else null + openStart = index + openHasChildren = false + openText.clear() + } else if (depth > 2) { + openHasChildren = true + } + } + is Text -> if (depth == 2) openText.append(event.text) + is Unmark -> { + if (--depth == 1) closeEntry() + return depth == 0 + } + } + return false + } + + // Reports an entry left open by a frontmatter that never closed (a broken + // upstream contract) — its text is complete by then. + fun finish() { + closeEntry() + } + + private fun closeEntry() { + open?.let { + open = null + onEntry( + FrontMatterEntry( + key = it["key"]!!, + type = it["type"], + text = openText.toString(), + hasChildren = openHasChildren, + start = openStart, + end = index + ) + ) + } + } + +} diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 96f3801..e212cfb 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -34,17 +34,19 @@ private val SCALAR_ENTRY_TYPES = setOf("bool", "int", "float", "timestamp") internal fun isScalarEntryType(type: String?): Boolean = type == null || type in SCALAR_ENTRY_TYPES -// Whether a value carries anything for a reader: a char that shows — not -// whitespace (NBSP included) and not an invisible format char such as a +// Whether a value carries anything for a reader: a char that shows. +internal fun isMetadataValue(value: String): Boolean = value.any { !it.isInvisible() } + +// Whitespace (NBSP included) or an invisible format char such as a // zero-width space or a byte order mark. -internal fun isMetadataValue(value: String): Boolean = - value.any { !it.isWhitespace() && it.category != CharCategory.FORMAT } +private fun Char.isInvisible(): Boolean = isWhitespace() || category == CharCategory.FORMAT // A title as `document.title` reads a `<title>` — HTML whitespace stripped -// and collapsed — with any other whitespace at its edges (the NBSP padding an -// icon often leaves) trimmed too: inside, NBSP is content and stays, at an -// edge it only forces the title into quotes. -internal fun String.normalizeTitle(): String = stripAndCollapseHtmlWhitespace().trim() +// and collapsed — with any other invisible char at its edges (the NBSP +// padding an icon often leaves, a byte order mark) trimmed too: inside, NBSP +// is content and stays, at an edge it only forces the title into quotes. +internal fun String.normalizeTitle(): String = + stripAndCollapseHtmlWhitespace().trim { it.isInvisible() } // Whether a front matter entry spelled `candidate` supersedes an earlier one // of the same name spelled `existing`, as front matter readers resolve a diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 850dee8..59ba06d 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -139,8 +139,10 @@ import kotlinx.coroutines.flow.Flow * format chars such as a zero-width space — `<html lang>` too) and * application state that single-page apps ship in `<meta>`: a JSON object, a * JSON array holding an object or a nested array, JSON state serialised into - * a JSON string — raw or percent-encoded — and a value over 4096 chars that, - * as written, does not read as text. Meta names are + * a JSON string — raw or percent-encoded — and a value over 4096 chars that + * does not read as text, judged as written (a percent-encoded one by its + * escapes) except for a flat JSON array, whose length and text are those of + * its elements joined — not the quotes and commas serialising them. Meta names are * ASCII case-insensitive, so of several names differing only in letter case * the first one (spelling and value) wins, as HTML resolves duplicate * `<meta>` elements — unlike [wrapInHtmlDocument], which resolves duplicate @@ -305,9 +307,10 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("meta") { event -> val name = event["name"] val content = event["content"] - if (name != null && !content.isNullOrBlank()) { - // cheapest checks first: the JSON parse runs only for a name - // that would otherwise be kept (the first of duplicates wins) + // blank by the rule addFromHtml applies; cheapest checks first, so + // the JSON parse runs only for a name that would otherwise be kept + // (an already present one loses, the first of duplicates winning) + if (name != null && content != null && isMetadataValue(content)) { val normalizedName = name.asciiLowercase() if (normalizedName !in metadata && !isNoiseMetaName(normalizedName) diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 395b482..c44a2bf 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -67,13 +67,9 @@ import kotlinx.coroutines.flow.Flow public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = semanticEvents { var opened = false - var collectingFrontmatter = false - // nesting depth inside the frontmatter: 1 while inside a top-level entry - var depth = 0 - // the top-level entry being read, null when it is not a scalar to keep - var entryKey: String? = null - val entryText = StringBuilder() val metadata = HeadMetadata() + // reads the frontmatter while it is being collected + var frontmatter: FrontMatterEntryReader? = null // blank text ahead of the frontmatter, replayed at the start of `body` val blanks = mutableListOf<SemanticEvent>() @@ -82,7 +78,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman // unmark cannot come from a builder block — explicit mark/unmark instead. suspend fun openDocument() { opened = true - collectingFrontmatter = false + frontmatter = null mark( "html", attributes = metadata["lang"] @@ -106,34 +102,14 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman } collect { event -> + val reader = frontmatter when { - collectingFrontmatter -> when (event) { - is Mark -> { - depth++ - if (depth == 1) { - val type = event["type"] - entryKey = if ( - event.name == "entry" && isScalarEntryType(type) - ) event["key"] else null - entryText.clear() - } else { - entryKey = null // a nested mark: not a scalar - } - } - is Text -> if (depth == 1) entryText.append(event.text) - is Unmark -> if (depth == 0) { - openDocument() - } else { - if (depth == 1) { - entryKey?.let { metadata.addFromFrontMatter(it, entryText.toString()) } - entryKey = null - } - depth-- - } - } + reader != null -> if (reader.read(event)) openDocument() !opened -> when { event is Mark && !event.isTagged && event.name == "frontmatter" -> { - collectingFrontmatter = true + frontmatter = FrontMatterEntryReader { entry -> + if (entry.isHeadMetadata) metadata.addFromFrontMatter(entry.key, entry.text) + }.also { it.read(event) } } event is Text && event.text.isHtmlBlank() -> blanks += event else -> { @@ -146,11 +122,9 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman } // An unclosed frontmatter at end of stream (broken upstream contract) is - // still used, including an entry left open — its text is complete by - // then; an empty stream yields the bare skeleton. - if (collectingFrontmatter && depth == 1) { - entryKey?.let { metadata.addFromFrontMatter(it, entryText.toString()) } - } + // still used, including an entry left open; an empty stream yields the + // bare skeleton. + frontmatter?.finish() if (!opened) openDocument() unmark("body") unmark("html") diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 5d183ca..ab5e350 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -665,12 +665,14 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should replace a title entry holding a nested structure with the derived title`() = runTest { - // given — no reader shows a mapping as the page title + fun `should keep a title entry holding a nested structure instead of deriving a title`() = runTest { + // given — no reader shows a mapping as the page title, but replacing + // it would delete the author's content (localized titles, say) val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { "entry"("key" to "en") { +"Hello" } + "entry"("key" to "de") { +"Hallo" } } "entry"("key" to "author") { +"Alice" } } @@ -683,7 +685,10 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Heading" } + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + "entry"("key" to "de") { +"Hallo" } + } "entry"("key" to "author") { +"Alice" } } "h1" { +"Heading" } @@ -742,9 +747,8 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should replace a blank title variant following a nested title entry`() = runTest { - // given — wrapInHtmlDocument skips the nested entry, so the blank - // variant after it is the title it reads + fun `should keep a nested title entry dropping a blank variant following it`() = runTest { + // given — the blank variant carries nothing, the nested entry does val input = semanticEvents { "frontmatter" { "entry"("key" to "title") { @@ -756,23 +760,21 @@ class EnsureFrontmatterTitleTest { } // when - val output = input.ensureFrontmatterTitle().wrapInHtmlDocument() + val output = input.ensureFrontmatterTitle() // then output sameAs semanticEvents { - "html" { - "head" { - "title" { +"Heading" } - } - "body" { - "h1" { +"Heading" } + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } } } + "h1" { +"Heading" } } } @Test - fun `should replace a nested title entry and a null variant following it with one title`() = runTest { + fun `should keep a nested title entry dropping a null variant following it`() = runTest { // given val input = semanticEvents { "frontmatter" { @@ -790,7 +792,9 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Heading" } + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } } "h1" { +"Heading" } } @@ -1025,9 +1029,9 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should leave a single title when deriving it past a later nested title entry`() = runTest { - // given — `title:` then `title: {en: Hello}`; a later-wins reader - // would read the mapping over a derived title in the first slot + fun `should keep a later nested title entry dropping an earlier null one`() = runTest { + // given — `title:` then `title: {en: Hello}`; js-yaml would reject + // the duplicate key, and only the null one carries nothing val input = semanticEvents { "frontmatter" { "entry"("key" to "title", "type" to "null") { } @@ -1045,13 +1049,34 @@ class EnsureFrontmatterTitleTest { // then output sameAs semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"Heading" } "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + } } "h1" { +"Heading" } } } + @Test + fun `should trim invisible format chars at the edges of the derived title`() = runTest { + // given + val input = semanticEvents { + "h1" { +"\u200BHeading\uFEFF" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } + } + "h1" { +"\u200BHeading\uFEFF" } + } + } + @Test fun `should not derive a title from an invisible h1`() = runTest { // given — a zero-width space shows nothing diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index daca500..601cfe3 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1116,6 +1116,59 @@ class SimplifyHtmlTest { } } + @Test + fun `should measure a flat JSON array by its elements as the same list written plainly`() = runTest { + // given — over the cap as written, under it once the quotes and + // commas are gone: short, as the same hashes written plainly are + val jsonHashes = (1..380).joinToString(",", "[", "]") { "\"a1b2c3d4\"" } + val plainHashes = (1..380).joinToString(" ") { "a1b2c3d4" } + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "json-hashes", "content" to jsonHashes) { } + "meta"("name" to "plain-hashes", "content" to plainHashes) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "json-hashes") { +jsonHashes } + "entry"("key" to "plain-hashes") { +plainHashes } + } + "p" { +"text" } + } + } + + @Test + fun `should drop a long array opening with a nested element unless it reads as text`() = runTest { + // given — state if it parses, a blob if it does not; neither is text + val state = (1..600).joinToString(",", "[", "]") { "[$it,$it]" } + val truncated = state.dropLast(1) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "state", "content" to state) { } + "meta"("name" to "truncated", "content" to truncated) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "p" { +"text" } + } + } + @Test fun `should judge the length of a percent-encoded value as it is written`() = runTest { // given — a value is written to the front matter still encoded, so @@ -1521,6 +1574,32 @@ class SimplifyHtmlTest { } } + @Test + fun `should trim invisible format chars at the edges of a title`() = runTest { + // given — a byte order mark and a zero-width space show nothing, at + // an edge they would only force the title into quotes + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"\uFEFFMy\u200BPage\u200B" } + "meta"("name" to "Title", "content" to "\u200BIgnored") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"My\u200BPage" } + } + "p" { +"x" } + } + } + @Test fun `should keep non-breaking spaces inside a title trimming the edges`() = runTest { // given — NBSP inside is content, not HTML whitespace, like in a meta From 2764ee049bf05285aed1392678eae4749cc3c4ca Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 22:44:16 +0200 Subject: [PATCH 18/25] Undo nested encodings of meta state; drop empty titles without an h1 (#82) - isApplicationStateMeta percent-decodes repeatedly (bounded) and inside JSON strings, and spots a nested element anywhere in an over-long array - ensureFrontmatterTitle drops empty title entries when no title is derived, and derives a title past NBSP text before the h1 (which still keeps a following frontmatter from opening the stream) - normalizeTitle collapses line-breaking whitespace; HeadMetadata judges front matter entries by FrontMatterEntry.isHeadMetadata - condense the front matter title entry in CLAUDE.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 12 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 79 ++++++++---- .../kotlin/EnsureFrontmatterTitle.kt | 59 +++++---- .../src/commonMain/kotlin/HeadMetadata.kt | 23 +++- .../commonMain/kotlin/WrapInHtmlDocument.kt | 4 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 115 ++++++++++++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 34 +++++- 7 files changed, 265 insertions(+), 61 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3b3b1a6..2aeee65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -558,14 +558,10 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Key typing is not value typing: go-yaml v2 decodes a **top-level** front matter key into a string but a nested one as `interface{}`, so `y:` must stay plain at the top level (it was once rewritten to `"y":`) and be quoted below it (`isTypedPlainKey`). The writer's plain-key rule (`isIdentifierKey`) and the detector share their character rules in `YamlKeyLine.kt`, and `YamlWriterTest` "should write every key as a line the front matter detector accepts" pins the two — widen one without the other and every document whose *first* entry has such a key silently loses its front matter on re-parse. **DIVERGENCE (no round-trip)**: a `frontmatter` whose root is a **sequence** (top-level `item`s) renders as `---\n- a\n…`, which line 2 rejects — Markdown reads `---` + `- a` as a thematic break + list, and no front matter consumer (Jekyll, Hugo, `wrapInHtmlDocument`) accepts a non-mapping — so the shape is pinned as such in `FrontMatterRoundTripTest` rather than widened into detection. -- `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter through **one** reader, `FrontMatterEntryReader` (top-level `entry` marks by depth, `FrontMatterEntry.isHeadMetadata` for what counts): only top-level scalar entries become `<meta>`/`<title>`, only a top-level `entry key=title` counts as an existing title. - Neither contains any YAML parsing — that was the ~300-line `Frontmatter.kt` codec this representation replaced; do not reintroduce a text-level reader, and do not give either operator a private copy of the entry reader again. - The two must agree on **which** title entry is read and what a *usable* one is (see the `ensureFrontmatterTitle` KDoc), or the derived `h1` title silently displaces a real one: judging the first title entry *of any type or value* once let a leading `title:` / `title: ""` hide a later `Title: Real`, which `wrapInHtmlDocument` reads — hand-rolled copies of the reader in each operator were how the two drifted apart. - Both treat text of **HTML** whitespace ahead of the frontmatter as insignificant (`isHtmlBlank`, not `isBlank` — an NBSP is content and makes the frontmatter ordinary content); `ensureFrontmatterTitle` additionally emits the frontmatter (held or synthesized) **ahead** of that text, since rendered Markdown front matter must open the document. - Duplicate keys resolve the way readers of the **source format** resolve them, so the two directions deliberately differ: `simplifyHtml` keeps the first `<meta>` of a name (HTML), `wrapInHtmlDocument` the later entry (Psych/PyYAML), the lowercase spelling beating a case variant (`HeadMetadata.addFromHtml` / `addFromFrontMatter`). - Do not "align" them for the round-trip's sake — each side emits one entry per name, so neither ever sees the other's duplicates; aligning them once made `wrapInHtmlDocument` show a title no front matter reader shows. - For the same reason `ensureFrontmatterTitle` leaves **exactly one** title entry (any letter case), spelled `title`, whenever a title comes out — dropping every other title variant, even a nested one — rather than reordering duplicates for later-wins readers: js-yaml (gray-matter: Eleventy, Astro, Gatsby) rejects a duplicate key outright and loses the whole front matter. - A **nested** title (a mapping of localized titles) with no usable title beside it is the exception: it is no title for `wrapInHtmlDocument`, but deriving one from the `h1` would delete it, so no title comes out and only the *empty* title entries are dropped. +- `wrapInHtmlDocument` and `ensureFrontmatterTitle` read the front matter through **one** reader, `FrontMatterEntryReader`, and must agree on which title entry is read and what a usable one is — hand-rolled copies of the reader in each operator drifted apart once and let a leading `title:` hide a later `Title: Real`, so the derived `h1` title displaced a real one. + Do not reintroduce a text-level YAML reader (the ~300-line `Frontmatter.kt` this replaced) or a private copy of the entry reader. + Duplicate keys deliberately resolve differently per direction (`simplifyHtml` first `<meta>` wins as in HTML, `wrapInHtmlDocument` later entry wins as in Psych/PyYAML): each side emits one entry per name, so neither sees the other's duplicates — "aligning" them once made `wrapInHtmlDocument` show a title no front matter reader shows. + `ensureFrontmatterTitle` never leaves duplicate title entries rather than reordering them for later-wins readers, because js-yaml (gray-matter: Eleventy, Astro, Gatsby) rejects a duplicate key and loses the whole front matter. ## Test conventions diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 2790a1a..696bed4 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -29,8 +29,9 @@ import kotlinx.serialization.json.jsonPrimitive // `<app>/config/environment`, … — 95% of a page's Markdown, issue #82). A // name denylist cannot keep up with names private to each site's framework, // so this judges the *value*, which is what tells metadata apart from state. -// A percent-encoded value's structure is judged by what it decodes to, so -// encoding state does not hide it: +// A percent-encoded value's structure is judged by what it decodes to — undoing +// the encoding as many times over as it was applied, and inside a JSON string +// too — so encoding state does not hide it: // - a value that parses as a JSON object. Parsing, not a look at the first // and last char, is what keeps human text that merely starts with a // bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); @@ -52,18 +53,18 @@ import kotlinx.serialization.json.jsonPrimitive // hashes is not, and one whose elements fit under the cap is short. internal fun isApplicationStateMeta(content: String): Boolean { val written = content.stripHtmlWhitespace() - val decoded = if (written.firstOrNull() == '%') written.percentDecodedOrNull()?.stripHtmlWhitespace() else null + val decoded = written.percentDecodedOrNull() val value = decoded ?: written val long = written.length > MAX_META_VALUE_LENGTH // An over-long value is kept only if it reads as text, whatever it // parses as, so a blob — megabytes of JSON, say — is dropped unparsed. // Only a raw array must be parsed first, to be read by its words — unless - // its first element is an object or an array: then it is state if it + // an element of it is an object or an array: then it is state if it // parses and a blob if it does not, dropped either way. val readByWords = decoded == null && value.firstOrNull() == '[' - if (long && (!readByWords || value.opensNestedElement()) && !written.readsAsText()) return true + if (long && (!readByWords || value.hasNestedElement()) && !written.readsAsText()) return true val json = value.parseJsonCandidateOrNull() - if (json?.isState() == true) return true + if (json?.isState(MAX_DECODING_DEPTH) == true) return true if (!long || !readByWords) return false // a flat array is measured by its elements joined, in length as in text, // as the same elements written as a plain list would be @@ -75,14 +76,31 @@ internal fun isApplicationStateMeta(content: String): Boolean { // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 -// Whether this array's first element opens an object or an array. -private fun String.opensNestedElement(): Boolean { - var i = 1 - while (i < length && this[i] in JSON_WHITESPACE) i++ - return i < length && (this[i] == '{' || this[i] == '[') +// Whether this (raw, possibly malformed) array holds an object or an array +// — a bracket past the opening one outside a JSON string — found by a scan, +// without parsing. +private fun String.hasNestedElement(): Boolean { + var inString = false + var escaped = false + for (i in 1 until length) { + val c = this[i] + when { + escaped -> escaped = false + inString -> when (c) { + '\\' -> escaped = true + '"' -> inString = false + } + c == '"' -> inString = true + c == '{' || c == '[' -> return true + } + } + return false } -private const val JSON_WHITESPACE = " \t\n\r" +// How many layers of encoding — percent-encoding, a JSON string — are undone +// in all to find state. Real double encodings take two or three; the bound +// keeps a crafted value from costing a pass over itself per layer. +private const val MAX_DECODING_DEPTH = 8 // Only a value opening like a JSON object, array or string is parsed. private fun String.parseJsonCandidateOrNull(): JsonElement? { @@ -90,18 +108,21 @@ private fun String.parseJsonCandidateOrNull(): JsonElement? { return if (first == '{' || first == '[' || first == '"') parseJsonOrNull() else null } -// Recursion unwraps one JSON string per level, and each level's escaping at -// least doubles the backslashes before a quote, so the depth stays -// logarithmic in the value's length. -private fun JsonElement.isState(): Boolean = when (this) { +// Recursion unwraps one JSON string per level, `depth` bounding the levels +// ([MAX_DECODING_DEPTH]). +private fun JsonElement.isState(depth: Int): Boolean = when (this) { is JsonObject -> true - is JsonArray -> any { it !is JsonPrimitive || it.isEncodedState() } - is JsonPrimitive -> isEncodedState() + is JsonArray -> any { it !is JsonPrimitive || it.isEncodedState(depth) } + is JsonPrimitive -> isEncodedState(depth) } -private fun JsonElement.isEncodedState(): Boolean = - this is JsonPrimitive && isString && - content.stripHtmlWhitespace().parseJsonCandidateOrNull()?.isState() == true +// A JSON string whose content — percent-decoded, when it is encoded — is +// state. +private fun JsonElement.isEncodedState(depth: Int): Boolean { + if (this !is JsonPrimitive || !isString || depth == 0) return false + val content = content.stripHtmlWhitespace() + return (content.percentDecodedOrNull() ?: content).parseJsonCandidateOrNull()?.isState(depth - 1) == true +} // Text is made of words: at least half the chars are letters (hex and // number lists are mostly digits) — a combining mark counting as one, since @@ -153,9 +174,23 @@ private fun String.parseJsonOrNull(): JsonElement? = try { null } +// This value with its percent-encoding undone — as many times over as it was +// applied, up to [MAX_DECODING_DEPTH] — and HTML whitespace stripped, or null +// when it is not percent-encoded: it does not open with an escape, or its +// first decoding fails. +private fun String.percentDecodedOrNull(): String? { + var value = this + var depth = 0 + while (depth < MAX_DECODING_DEPTH && value.firstOrNull() == '%') { + value = value.percentDecodedOnceOrNull()?.stripHtmlWhitespace() ?: break + depth++ + } + return if (depth == 0) null else value +} + // Decodes `%XX` escapes as UTF-8, or null when the value is not // percent-encoded text (a malformed escape, a raw non-ASCII char). -private fun String.percentDecodedOrNull(): String? { +private fun String.percentDecodedOnceOrNull(): String? { val bytes = ByteArray(length) var size = 0 var i = 0 diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 1ab4b37..8f54139 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -51,12 +51,13 @@ import kotlinx.coroutines.flow.Flow * entry is dropped. * * Otherwise the frontmatter is held back and the title is derived from the - * very first `h1` following it (only HTML whitespace text may intervene): the + * very first `h1` following it (only blank text may intervene — an NBSP + * included, which keeps a frontmatter *following* it from opening the + * stream, but not a synthesized one from going first): the * `h1` subtree's flattened text — its text events plus the `alt` of every * `img` mark, in document order, the way an accessible name is computed from - * content — with its HTML whitespace stripped and collapsed, as - * `document.title` reads a `<title>`, and any other whitespace at its edges - * trimmed, becomes the `title`, and only then the frontmatter and the + * content — normalized as [normalizeTitle] reads a `<title>` — becomes the + * `title`, and only then the frontmatter and the * buffered `h1` are emitted, in source order. The derived entry takes the * place of the first title entry, every other one dropped, or is injected as * the first `entry` of a frontmatter without any. When no frontmatter exists @@ -65,8 +66,9 @@ import kotlinx.coroutines.flow.Flow * * When no title can be derived — the first non-blank event after the * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — - * everything held is flushed unchanged: a stream without a leading `h1` - * passes through untouched and no empty frontmatter is fabricated. + * everything held is flushed unchanged but for the empty title entries of + * the frontmatter, dropped as above: a stream without a leading `h1` passes + * through untouched and no empty frontmatter is fabricated. * * Buffering is bounded to the frontmatter subtree plus one `h1` subtree — * everything after the decision point is forwarded as it arrives. @@ -89,6 +91,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit val blanks = mutableListOf<SemanticEvent>() + // whether all of `blanks` is HTML whitespace, which alone lets a + // frontmatter following it open the stream + var blanksAreHtmlWhitespace = true // the h1 subtree, buffered between its mark and balanced unmark so the // title can be derived from the flattened text @@ -101,9 +106,14 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s blanks.clear() } + fun holdBlank(event: SemanticEvent.Text) { + blanks += event + if (!event.text.isHtmlBlank()) blanksAreHtmlWhitespace = false + } + // emits the held frontmatter — `prepended` right after its mark, the span - // of each slot in `rewrite` replaced by its events (none drops the entry) - // — then the blanks held with it + // of each slot in `rewrite` (in source order, as `titleSlots` is) replaced + // by its events (none drops the entry) — then the blanks held with it suspend fun flushHeld( rewrite: Map<FrontMatterEntry, List<SemanticEvent>> = emptyMap(), prepended: List<SemanticEvent> = emptyList() @@ -112,7 +122,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s emit(held.first()) emit(prepended) var next = 1 - for ((slot, events) in rewrite.entries.sortedBy { it.key.start }) { + for ((slot, events) in rewrite) { emit(held.subList(next, slot.start)) emit(events) next = slot.end + 1 @@ -129,11 +139,17 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushHeld(titleSlots.associateWith { if (it == slot) entry else emptyList() }) } + // emits the held frontmatter without its empty title entries, keeping + // the nested ones + suspend fun flushWithoutEmptyTitles() { + flushHeld(titleSlots.filterNot { it.hasChildren }.associateWith { emptyList() }) + } + suspend fun commitHeading() { val title = headingText.toString().normalizeTitle() val slot = titleSlots.firstOrNull() when { - !isMetadataValue(title) -> flushHeld() + !isMetadataValue(title) -> flushWithoutEmptyTitles() frontmatterEvents == null -> { "frontmatter" { emit(titleEntry(title)) @@ -162,7 +178,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } // a nested title is content, not ours to replace titleSlots.any { it.hasChildren } -> { - flushHeld(titleSlots.filterNot { it.hasChildren }.associateWith { emptyList() }) + flushWithoutEmptyTitles() PassThrough } else -> AwaitingHeading @@ -178,15 +194,16 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s collect { event -> when (state) { AtStart -> when { - // only a frontmatter opening the stream (but for blank text) - // is the page's metadata; anywhere else it is content - event is Mark && !event.isTagged && event.name == "frontmatter" -> { + // only a frontmatter opening the stream (but for HTML + // whitespace text) is the page's metadata; anywhere else it + // is content + event is Mark && !event.isTagged && event.name == "frontmatter" && blanksAreHtmlWhitespace -> { frontmatterEvents = mutableListOf(event) reader.read(event) state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) - event is Text && event.text.isHtmlBlank() -> blanks += event + event is Text && event.text.isBlank() -> holdBlank(event) else -> { flushBlanks() emit(event) @@ -198,10 +215,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s if (reader.read(event)) judgeFrontmatter() } AwaitingHeading -> when (event) { - is Text if event.text.isHtmlBlank() -> blanks += event + is Text if event.text.isBlank() -> holdBlank(event) is Mark if event.name == "h1" -> startHeading(event) else -> { - flushHeld() + flushWithoutEmptyTitles() emit(event) state = PassThrough } @@ -230,10 +247,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // an unclosed h1 (broken upstream contract) still commits so no buffered // events are lost; otherwise flush whatever is still held - if (state == InHeading) { - commitHeading() - } else { - flushHeld() + when (state) { + InHeading -> commitHeading() + AwaitingHeading -> flushWithoutEmptyTitles() + else -> flushHeld() } } diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index e212cfb..f39b62e 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -39,14 +39,22 @@ internal fun isMetadataValue(value: String): Boolean = value.any { !it.isInvisib // Whitespace (NBSP included) or an invisible format char such as a // zero-width space or a byte order mark. -private fun Char.isInvisible(): Boolean = isWhitespace() || category == CharCategory.FORMAT +private fun Char.isInvisible(): Boolean = isWhitespace() || category == FORMAT // A title as `document.title` reads a `<title>` — HTML whitespace stripped // and collapsed — with any other invisible char at its edges (the NBSP // padding an icon often leaves, a byte order mark) trimmed too: inside, NBSP // is content and stays, at an edge it only forces the title into quotes. +// Whitespace that breaks a line (a vertical tab, a line or paragraph +// separator, a next line char) collapses like HTML whitespace, as a title is +// one line. internal fun String.normalizeTitle(): String = - stripAndCollapseHtmlWhitespace().trim { it.isInvisible() } + map { if (it in LINE_BREAKING_WHITESPACE) ' ' else it } + .joinToString("") + .stripAndCollapseHtmlWhitespace() + .trim { it.isInvisible() } + +private const val LINE_BREAKING_WHITESPACE = "\u000B\u0085\u2028\u2029" // Whether a front matter entry spelled `candidate` supersedes an earlier one // of the same name spelled `existing`, as front matter readers resolve a @@ -91,14 +99,17 @@ internal class HeadMetadata { } // Adds a front matter entry the way its readers resolve a duplicate key - // ([frontMatterKeySupersedes]). The name keeps the position of its first + // ([frontMatterKeySupersedes]), unless it is no head metadata + // ([FrontMatterEntry.isHeadMetadata], the rule ensureFrontmatterTitle + // judges title entries by). The name keeps the position of its first // occurrence. - fun addFromFrontMatter(key: String, value: String) { - if (!isMetadataValue(value)) return + fun addFromFrontMatter(entry: FrontMatterEntry) { + if (!entry.isHeadMetadata) return + val key = entry.key val name = key.asciiLowercase() val existing = entries[name] if (existing == null || frontMatterKeySupersedes(existing.key, key)) { - entries[name] = MetadataEntry(key, value) + entries[name] = MetadataEntry(key, entry.text) } } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index c44a2bf..da808dc 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -107,9 +107,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman reader != null -> if (reader.read(event)) openDocument() !opened -> when { event is Mark && !event.isTagged && event.name == "frontmatter" -> { - frontmatter = FrontMatterEntryReader { entry -> - if (entry.isHeadMetadata) metadata.addFromFrontMatter(entry.key, entry.text) - }.also { it.read(event) } + frontmatter = FrontMatterEntryReader(metadata::addFromFrontMatter).also { it.read(event) } } event is Text && event.text.isHtmlBlank() -> blanks += event else -> { diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index ab5e350..f11581d 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -1140,6 +1140,121 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should derive a title past a non-breaking space before the h1`() = runTest { + // given — an ` ` spacer only stops a frontmatter following it + // from opening the stream; a synthesized one goes first anyway + val input = semanticEvents { + +"\u00A0" + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + } + +"\u00A0" + "h1" { +"Hello" } + } + } + + @Test + fun `should derive a title past a non-breaking space between the frontmatter and the h1`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + +"\u00A0" + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + "entry"("key" to "author") { +"Alice" } + } + +"\u00A0" + "h1" { +"Hello" } + } + } + + @Test + fun `should drop empty title entries when the h1 yields no title`() = runTest { + // given — duplicate keys make js-yaml reject the whole front matter + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "author") { +"Alice" } + "entry"("key" to "title") { } + } + "h1" { } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "h1" { } + } + } + + @Test + fun `should drop empty title entries when no h1 follows`() = runTest { + // given + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "Title") { } + "entry"("key" to "author") { +"Alice" } + } + "p" { +"Body." } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "author") { +"Alice" } + } + "p" { +"Body." } + } + } + + @Test + fun `should collapse line-breaking whitespace in the derived title`() = runTest { + // given — a line or paragraph separator, a vertical tab or a next + // line char pasted into a heading would break the title's line + val input = semanticEvents { + "h1" { +"Foo\u2028Bar\u000BBaz\u2029\u0085Qux" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Foo Bar Baz Qux" } + } + "h1" { +"Foo\u2028Bar\u000BBaz\u2029\u0085Qux" } + } + } + @Test fun `should treat a tagged frontmatter as content`() = runTest { // given — a literal `<frontmatter>` tag, not front matter diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 601cfe3..3d88dc8 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -971,6 +971,33 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop state percent-encoded inside a JSON string or more than once`() = runTest { + // given — encoding state once more, in either layer, must not hide it + val state = "{\"modulePrefix\":\"app\"}" + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "app/config", "content" to "\"${state.percentEncoded()}\"") { } + "meta"("name" to "app/config-2", "content" to state.percentEncoded().percentEncoded()) { } + "meta"("name" to "subject", "content" to "\"50%25 off\"") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "subject") { +"\"50%25 off\"" } + } + "p" { +"text" } + } + } + @Test fun `should keep long text written in supplementary-plane letters or emoji`() = runTest { // given — each of these letters and emoji is a UTF-16 surrogate pair @@ -1146,15 +1173,20 @@ class SimplifyHtmlTest { } @Test - fun `should drop a long array opening with a nested element unless it reads as text`() = runTest { + fun `should drop a long array holding a nested element unless it reads as text`() = runTest { // given — state if it parses, a blob if it does not; neither is text val state = (1..600).joinToString(",", "[", "]") { "[$it,$it]" } val truncated = state.dropLast(1) + // the nested element need not come first + val late = (1..600).joinToString(",", "[\"x\",", "]") { "{\"id\":$it}" } + val lateTruncated = late.dropLast(1) val input = semanticEvents(tagged = true) { "html" { "head" { "meta"("name" to "state", "content" to state) { } "meta"("name" to "truncated", "content" to truncated) { } + "meta"("name" to "late", "content" to late) { } + "meta"("name" to "late-truncated", "content" to lateTruncated) { } } "body" { "p" { +"text" } } } From e3738c05c07901380214aef7120656168297802b Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Tue, 29 Sep 2026 23:00:52 +0200 Subject: [PATCH 19/25] Keep nested titles beside a variant; share one decoding budget (#82) - ensureFrontmatterTitle keeps a usable title variant beside a nested title entry as spelled, and leaves a frontmatter whose root is a sequence unchanged (FrontMatterEntryReader.isSequence) - isApplicationStateMeta shares one layer budget between percent and JSON-string decoding, trims invisible chars (BOM, NBSP) at the edges, and judges an over-long value as written before decoding it - wrapInHtmlDocument strips HTML whitespace around the lang entry - extract trimInvisible; condense the simplifyHtml KDoc Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 1 + .../commonMain/kotlin/ApplicationStateMeta.kt | 78 ++++++++++--------- .../kotlin/EnsureFrontmatterTitle.kt | 32 +++++--- .../kotlin/FrontMatterEntryReader.kt | 5 ++ .../src/commonMain/kotlin/HeadMetadata.kt | 13 +++- .../src/commonMain/kotlin/SimplifyHtml.kt | 9 +-- .../commonMain/kotlin/WrapInHtmlDocument.kt | 7 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 57 ++++++++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 55 +++++++++++++ .../kotlin/WrapInHtmlDocumentTest.kt | 24 ++++++ 10 files changed, 222 insertions(+), 59 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2aeee65..741f3f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -562,6 +562,7 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Do not reintroduce a text-level YAML reader (the ~300-line `Frontmatter.kt` this replaced) or a private copy of the entry reader. Duplicate keys deliberately resolve differently per direction (`simplifyHtml` first `<meta>` wins as in HTML, `wrapInHtmlDocument` later entry wins as in Psych/PyYAML): each side emits one entry per name, so neither sees the other's duplicates — "aligning" them once made `wrapInHtmlDocument` show a title no front matter reader shows. `ensureFrontmatterTitle` never leaves duplicate title entries rather than reordering them for later-wins readers, because js-yaml (gray-matter: Eleventy, Astro, Gatsby) rejects a duplicate key and loses the whole front matter. + The one exception is a nested title entry (localized titles): it is content, so a usable variant beside it is kept as found rather than collapsing the two into one. ## Test conventions diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 696bed4..91eb4cc 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -16,7 +16,6 @@ package com.xemantic.markanywhere.html -import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -31,7 +30,8 @@ import kotlinx.serialization.json.jsonPrimitive // so this judges the *value*, which is what tells metadata apart from state. // A percent-encoded value's structure is judged by what it decodes to — undoing // the encoding as many times over as it was applied, and inside a JSON string -// too — so encoding state does not hide it: +// too — so encoding state does not hide it, nor do invisible chars (a byte +// order mark, NBSP) at its edges or those of a JSON string within: // - a value that parses as a JSON object. Parsing, not a look at the first // and last char, is what keeps human text that merely starts with a // bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); @@ -52,23 +52,25 @@ import kotlinx.serialization.json.jsonPrimitive // written plainly would be: a long list of words is kept, a long list of // hashes is not, and one whose elements fit under the cap is short. internal fun isApplicationStateMeta(content: String): Boolean { - val written = content.stripHtmlWhitespace() - val decoded = written.percentDecodedOrNull() - val value = decoded ?: written + val written = content.trimInvisible() val long = written.length > MAX_META_VALUE_LENGTH + val writtenReadsAsText by lazy { written.readsAsText() } // An over-long value is kept only if it reads as text, whatever it - // parses as, so a blob — megabytes of JSON, say — is dropped unparsed. - // Only a raw array must be parsed first, to be read by its words — unless - // an element of it is an object or an array: then it is state if it - // parses and a blob if it does not, dropped either way. - val readByWords = decoded == null && value.firstOrNull() == '[' - if (long && (!readByWords || value.hasNestedElement()) && !written.readsAsText()) return true - val json = value.parseJsonCandidateOrNull() - if (json?.isState(MAX_DECODING_DEPTH) == true) return true + // parses or decodes as, so a blob — megabytes of JSON or of escapes, + // say — is dropped unparsed and undecoded. Only a raw array must be + // parsed first, to be read by its words — unless an element of it is an + // object or an array: then it is state if it parses and a blob if it + // does not, dropped either way. + val readByWords = written.firstOrNull() == '[' + if (long && (!readByWords || written.hasNestedElement()) && !writtenReadsAsText) return true + val decoded = written.percentDecodedOrNull(MAX_DECODING_DEPTH) + val json = (decoded?.value ?: written).parseJsonCandidateOrNull() + if (json?.isState(decoded?.layersLeft ?: MAX_DECODING_DEPTH) == true) return true if (!long || !readByWords) return false + if (json !is JsonArray) return !writtenReadsAsText // a flat array is measured by its elements joined, in length as in text, // as the same elements written as a plain list would be - val text = if (json is JsonArray) json.joinToString(" ") { it.jsonPrimitive.content } else written + val text = json.joinToString(" ") { it.jsonPrimitive.content } return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() } @@ -97,9 +99,10 @@ private fun String.hasNestedElement(): Boolean { return false } -// How many layers of encoding — percent-encoding, a JSON string — are undone -// in all to find state. Real double encodings take two or three; the bound -// keeps a crafted value from costing a pass over itself per layer. +// How many layers of encoding — percent-encoding and JSON strings, one budget +// for both — are undone, one within another, to find state. Real double +// encodings take two or three; the bound keeps a crafted value from costing +// a pass over itself per layer. private const val MAX_DECODING_DEPTH = 8 // Only a value opening like a JSON object, array or string is parsed. @@ -108,20 +111,22 @@ private fun String.parseJsonCandidateOrNull(): JsonElement? { return if (first == '{' || first == '[' || first == '"') parseJsonOrNull() else null } -// Recursion unwraps one JSON string per level, `depth` bounding the levels -// ([MAX_DECODING_DEPTH]). -private fun JsonElement.isState(depth: Int): Boolean = when (this) { +// Recursion unwraps one JSON string per level, `layers` bounding the +// layers of encoding still to be undone ([MAX_DECODING_DEPTH]). +private fun JsonElement.isState(layers: Int): Boolean = when (this) { is JsonObject -> true - is JsonArray -> any { it !is JsonPrimitive || it.isEncodedState(depth) } - is JsonPrimitive -> isEncodedState(depth) + is JsonArray -> any { it !is JsonPrimitive || it.isEncodedState(layers) } + is JsonPrimitive -> isEncodedState(layers) } // A JSON string whose content — percent-decoded, when it is encoded — is // state. -private fun JsonElement.isEncodedState(depth: Int): Boolean { - if (this !is JsonPrimitive || !isString || depth == 0) return false - val content = content.stripHtmlWhitespace() - return (content.percentDecodedOrNull() ?: content).parseJsonCandidateOrNull()?.isState(depth - 1) == true +private fun JsonElement.isEncodedState(layers: Int): Boolean { + if (this !is JsonPrimitive || !isString || layers == 0) return false + val content = content.trimInvisible() + val decoded = content.percentDecodedOrNull(layers - 1) + return (decoded?.value ?: content).parseJsonCandidateOrNull() + ?.isState(decoded?.layersLeft ?: (layers - 1)) == true } // Text is made of words: at least half the chars are letters (hex and @@ -174,18 +179,21 @@ private fun String.parseJsonOrNull(): JsonElement? = try { null } +// A value with some layers of its encoding undone, and how many more may be. +private class Decoded(val value: String, val layersLeft: Int) + // This value with its percent-encoding undone — as many times over as it was -// applied, up to [MAX_DECODING_DEPTH] — and HTML whitespace stripped, or null -// when it is not percent-encoded: it does not open with an escape, or its -// first decoding fails. -private fun String.percentDecodedOrNull(): String? { +// applied, up to `layers` times — and invisible chars trimmed from its +// edges, or null when it is not percent-encoded: it does not open with an +// escape, its first decoding fails, or no layer may be undone. +private fun String.percentDecodedOrNull(layers: Int): Decoded? { var value = this - var depth = 0 - while (depth < MAX_DECODING_DEPTH && value.firstOrNull() == '%') { - value = value.percentDecodedOnceOrNull()?.stripHtmlWhitespace() ?: break - depth++ + var left = layers + while (left > 0 && value.firstOrNull() == '%') { + value = value.percentDecodedOnceOrNull()?.trimInvisible() ?: break + left-- } - return if (depth == 0) null else value + return if (left == layers) null else Decoded(value, left) } // Decodes `%XX` escapes as UTF-8, or null when the value is not diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 8f54139..754499d 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -36,13 +36,16 @@ import kotlinx.coroutines.flow.Flow * structure, an empty collection, `null` or blank text are not, as there. * * A title entry holding a nested structure (localized titles, say) is no - * title for [wrapInHtmlDocument], but it is content: with no usable title - * entry beside it, no title is derived, and the frontmatter passes through - * with only its empty title entries — `null`, blank, an empty collection — + * title for [wrapInHtmlDocument], but it is content: no title is derived, + * no other title entry respelled, and the frontmatter passes through with + * only its empty title entries — `null`, blank, an empty collection — * dropped, as they carry nothing and a duplicate key makes js-yaml reject the - * whole front matter. + * whole front matter. A usable title variant beside it is kept as spelled, + * so readers may disagree on the title as they did on the input. A + * frontmatter whose root is a sequence (top-level `item` marks) has no place + * for a title entry and passes through unchanged. * - * Whenever a title comes out, the frontmatter holds exactly one title entry, + * Otherwise, whenever a title comes out, the frontmatter holds exactly one title entry, * spelled `title`: every front matter reader then reads the same title — * whether it matches keys case-sensitively (Jekyll) or not, keeps the later * of duplicate keys (Psych, PyYAML) or rejects them (js-yaml) — and it is the @@ -140,9 +143,13 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } // emits the held frontmatter without its empty title entries, keeping - // the nested ones + // the nested and the usable ones suspend fun flushWithoutEmptyTitles() { - flushHeld(titleSlots.filterNot { it.hasChildren }.associateWith { emptyList() }) + flushHeld( + titleSlots + .filterNot { it.hasChildren || it.isHeadMetadata } + .associateWith { emptyList() } + ) } suspend fun commitHeading() { @@ -171,16 +178,17 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s .filter { it.isHeadMetadata } .reduceOrNull { read, next -> if (frontMatterKeySupersedes(read.key, next.key)) next else read } state = when { + // a nested title is content, not ours to replace or drop; a + // sequence has no place for a title entry + titleSlots.any { it.hasChildren } || reader.isSequence -> { + flushWithoutEmptyTitles() + PassThrough + } usable != null -> { val events = frontmatterEvents!!.subList(usable.start, usable.end + 1) flushWithSingleTitle(usable, respelled(events)) PassThrough } - // a nested title is content, not ours to replace - titleSlots.any { it.hasChildren } -> { - flushWithoutEmptyTitles() - PassThrough - } else -> AwaitingHeading } } diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt index 2ac519a..14d4055 100644 --- a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -52,6 +52,10 @@ internal class FrontMatterEntryReader( private var openHasChildren = false private val openText = StringBuilder() + // Whether the root is a sequence: a top-level `item` was read. + var isSequence: Boolean = false + private set + // Reads the next event of the subtree, the frontmatter mark first; true // once the frontmatter's own unmark is read. fun read(event: SemanticEvent): Boolean { @@ -60,6 +64,7 @@ internal class FrontMatterEntryReader( is Mark -> { depth++ if (depth == 2) { + if (event.name == "item") isSequence = true open = if (event.name == "entry" && event["key"] != null) event else null openStart = index openHasChildren = false diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index f39b62e..1c17885 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -41,6 +41,10 @@ internal fun isMetadataValue(value: String): Boolean = value.any { !it.isInvisib // zero-width space or a byte order mark. private fun Char.isInvisible(): Boolean = isWhitespace() || category == FORMAT +// This value with the chars [isMetadataValue] finds invisible trimmed from +// its edges. +internal fun String.trimInvisible(): String = trim { it.isInvisible() } + // A title as `document.title` reads a `<title>` — HTML whitespace stripped // and collapsed — with any other invisible char at its edges (the NBSP // padding an icon often leaves, a byte order mark) trimmed too: inside, NBSP @@ -49,10 +53,10 @@ private fun Char.isInvisible(): Boolean = isWhitespace() || category == FORMAT // separator, a next line char) collapses like HTML whitespace, as a title is // one line. internal fun String.normalizeTitle(): String = - map { if (it in LINE_BREAKING_WHITESPACE) ' ' else it } - .joinToString("") + CharArray(length) { if (this[it] in LINE_BREAKING_WHITESPACE) ' ' else this[it] } + .concatToString() .stripAndCollapseHtmlWhitespace() - .trim { it.isInvisible() } + .trimInvisible() private const val LINE_BREAKING_WHITESPACE = "\u000B\u0085\u2028\u2029" @@ -66,7 +70,8 @@ internal fun frontMatterKeySupersedes(existing: String, candidate: String): Bool return candidate == name || existing != name } -// A front matter entry: the name as first spelled, and its value. +// A metadata entry: its name, spelled as where its value was read, and the +// value. internal data class MetadataEntry(val key: String, val value: String) // The `<head>` metadata a front matter holds, in insertion order and keyed diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 59ba06d..1cc9197 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -137,12 +137,9 @@ import kotlinx.coroutines.flow.Flow * `apple-*`) are dropped so they don't inflate the frontmatter, and so are a * value with nothing visible in it (whitespace, NBSP included, or invisible * format chars such as a zero-width space — `<html lang>` too) and - * application state that single-page apps ship in `<meta>`: a JSON object, a - * JSON array holding an object or a nested array, JSON state serialised into - * a JSON string — raw or percent-encoded — and a value over 4096 chars that - * does not read as text, judged as written (a percent-encoded one by its - * escapes) except for a flat JSON array, whose length and text are those of - * its elements joined — not the quotes and commas serialising them. Meta names are + * application state that single-page apps ship in `<meta>` — structured JSON + * (raw, percent-encoded or serialised into a JSON string) and opaque blobs + * over 4096 chars that do not read as text. Meta names are * ASCII case-insensitive, so of several names differing only in letter case * the first one (spelling and value) wins, as HTML resolves duplicate * `<meta>` elements — unlike [wrapInHtmlDocument], which resolves duplicate diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index da808dc..bdd3bd4 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -20,6 +20,7 @@ import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase import com.xemantic.markanywhere.html.spec.isHtmlBlank +import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import kotlinx.coroutines.flow.Flow /** @@ -31,7 +32,8 @@ import kotlinx.coroutines.flow.Flow * inverse of [simplifyHtml]'s head-to-frontmatter extraction: * * - the `title` entry becomes `<title>` - * - the `lang` entry becomes the `lang` attribute on `<html>` + * - the `lang` entry, its HTML whitespace stripped, becomes the `lang` + * attribute on `<html>` * - every other top-level scalar entry becomes a void `<meta name content>` * * Only top-level scalar entries are interpreted — exactly the shape @@ -82,7 +84,8 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman mark( "html", attributes = metadata["lang"] - ?.let { mapOf("lang" to it.value) } + // trimmed, as simplifyHtml reads it + ?.let { mapOf("lang" to it.value.stripHtmlWhitespace()) } ?: emptyMap() ) "head" { diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index f11581d..4ffe117 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -695,6 +695,63 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should keep a nested title entry beside a usable title variant`() = runTest { + // given — the variant is what wrapInHtmlDocument reads, but keeping a + // single title entry would delete the localized titles + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + "entry"("key" to "de") { +"Hallo" } + } + "entry"("key" to "Title") { +"Hello" } + "entry"("key" to "TITLE") { +" " } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { + "entry"("key" to "en") { +"Hello" } + "entry"("key" to "de") { +"Hallo" } + } + "entry"("key" to "Title") { +"Hello" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should not add a title entry to a frontmatter whose root is a sequence`() = runTest { + // given — a title entry beside the items would make the root neither + // a mapping nor a sequence, which no YAML reader loads + val input = semanticEvents { + "frontmatter" { + "item" { +"a" } + "item" { +"b" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "item" { +"a" } + "item" { +"b" } + } + "h1" { +"Heading" } + } + } + @Test fun `should respell a title variant following a null title entry dropping the null one`() = runTest { // given — wrapInHtmlDocument skips the null entry and reads the diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 3d88dc8..2823c6c 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -998,6 +998,61 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop state with invisible chars at its edges`() = runTest { + // given — a byte order mark, NBSP or zero-width space carries nothing + // for a reader, so it must not hide the JSON behind it + val state = "{\"user\":{\"id\":1}}" + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "init", "content" to "\uFEFF$state") { } + "meta"("name" to "init-2", "content" to "\u00A0$state\u200B") { } + "meta"("name" to "init-3", "content" to "\"\uFEFF${state.replace("\"", "\\\"")}\"") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "p" { +"text" } + } + } + + @Test + fun `should undo at most eight layers of encoding in all`() = runTest { + // given — JSON-string and percent layers alternating count against one + // budget, so a value past it costs no more passes and is kept + val state = "{\"a\":1}" + fun String.jsonString() = "\"" + replace("\\", "\\\\").replace("\"", "\\\"") + "\"" + val eight = (1..4).fold(state) { value, _ -> value.jsonString().percentEncoded() } + val nine = eight.jsonString() + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "eight", "content" to eight) { } + "meta"("name" to "nine", "content" to nine) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "nine") { +nine } + } + "p" { +"text" } + } + } + @Test fun `should keep long text written in supplementary-plane letters or emoji`() = runTest { // given — each of these letters and emoji is a UTF-16 surrogate pair diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index dcd49fe..1ab9a52 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -103,6 +103,30 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should trim HTML whitespace around the lang`() = runTest { + // given — as simplifyHtml trims it on the way in + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +" en\n" } + } + "p" { +"Body." } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html"("lang" to "en") { + "head" { } + "body" { + "p" { +"Body." } + } + } + } + } + @Test fun `should use scalar text verbatim including decoded quoting`() = runTest { // given — the parser has already decoded the YAML; the value carries a From aaac6fa19b4f15dd108235e0c6291d955086557d Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 12:53:08 +0200 Subject: [PATCH 20/25] Drop a frontmatter emptied of its titles; judge an unclosed one (#82) - ensureFrontmatterTitle drops a frontmatter left empty by dropping its empty title entries (it would render as `---` twice, two thematic breaks on re-parse), and judges the title entries of a frontmatter the stream ends inside, an open entry included - isMetadataValue treats control chars (next line, C0) as invisible - clarify the over-long nested-array comment in isApplicationStateMeta Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../commonMain/kotlin/ApplicationStateMeta.kt | 4 +- .../kotlin/EnsureFrontmatterTitle.kt | 47 ++++++++--- .../src/commonMain/kotlin/HeadMetadata.kt | 7 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 82 +++++++++++++++++++ 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 91eb4cc..853664e 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -59,8 +59,8 @@ internal fun isApplicationStateMeta(content: String): Boolean { // parses or decodes as, so a blob — megabytes of JSON or of escapes, // say — is dropped unparsed and undecoded. Only a raw array must be // parsed first, to be read by its words — unless an element of it is an - // object or an array: then it is state if it parses and a blob if it - // does not, dropped either way. + // object or an array: then it is state if it parses, and otherwise + // dropped unparsed unless it reads as text, as any over-long value. val readByWords = written.firstOrNull() == '[' if (long && (!readByWords || written.hasNestedElement()) && !writtenReadsAsText) return true val decoded = written.percentDecodedOrNull(MAX_DECODING_DEPTH) diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 754499d..b8ddad7 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -70,8 +70,11 @@ import kotlinx.coroutines.flow.Flow * When no title can be derived — the first non-blank event after the * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — * everything held is flushed unchanged but for the empty title entries of - * the frontmatter, dropped as above: a stream without a leading `h1` passes - * through untouched and no empty frontmatter is fabricated. + * the frontmatter, dropped as above — the frontmatter too, when they were + * all it held: a stream without a leading `h1` passes through untouched and + * no empty frontmatter is fabricated. A frontmatter the stream ends inside (a + * broken upstream contract) is judged all the same, an entry left open + * included. * * Buffering is bounded to the frontmatter subtree plus one `h1` subtree — * everything after the decision point is forwarded as it arrives. @@ -116,21 +119,31 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // emits the held frontmatter — `prepended` right after its mark, the span // of each slot in `rewrite` (in source order, as `titleSlots` is) replaced - // by its events (none drops the entry) — then the blanks held with it + // by its events (none drops the entry) — then the blanks held with it. + // A frontmatter the rewrite leaves empty is dropped: it would render as a + // `---` pair, which parses back as two thematic breaks. suspend fun flushHeld( rewrite: Map<FrontMatterEntry, List<SemanticEvent>> = emptyMap(), prepended: List<SemanticEvent> = emptyList() ) { frontmatterEvents?.let { held -> - emit(held.first()) - emit(prepended) - var next = 1 - for ((slot, events) in rewrite) { - emit(held.subList(next, slot.start)) - emit(events) - next = slot.end + 1 + val body = buildList { + addAll(prepended) + var next = 1 + for ((slot, events) in rewrite) { + addAll(held.subList(next, slot.start)) + addAll(events) + next = slot.end + 1 + } + addAll(held.subList(next, held.size)) + } + val emptied = rewrite.isNotEmpty() && body.none { + it is Mark || (it is Text && !it.text.isHtmlBlank()) + } + if (!emptied) { + emit(held.first()) + emit(body) } - emit(held.subList(next, held.size)) } frontmatterEvents = null flushBlanks() @@ -253,9 +266,17 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } } - // an unclosed h1 (broken upstream contract) still commits so no buffered - // events are lost; otherwise flush whatever is still held + // an unclosed frontmatter or h1 (broken upstream contract) is still + // committed so no buffered events are lost; otherwise flush whatever is + // still held when (state) { + // judged like a closed one, an entry left open included, as + // wrapInHtmlDocument reads it + InFrontmatter -> { + reader.finish() + judgeFrontmatter() + if (state == AwaitingHeading) flushWithoutEmptyTitles() + } InHeading -> commitHeading() AwaitingHeading -> flushWithoutEmptyTitles() else -> flushHeld() diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 1c17885..37dcea6 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -37,9 +37,10 @@ internal fun isScalarEntryType(type: String?): Boolean = // Whether a value carries anything for a reader: a char that shows. internal fun isMetadataValue(value: String): Boolean = value.any { !it.isInvisible() } -// Whitespace (NBSP included) or an invisible format char such as a -// zero-width space or a byte order mark. -private fun Char.isInvisible(): Boolean = isWhitespace() || category == FORMAT +// Whitespace (NBSP included), a control char (a next line char, a C0 +// control), or an invisible format char such as a zero-width space or a byte +// order mark. +private fun Char.isInvisible(): Boolean = isWhitespace() || category == CONTROL || category == FORMAT // This value with the chars [isMetadataValue] finds invisible trimmed from // its edges. diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 4ffe117..57632a7 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.html import com.xemantic.kotlin.test.sameAs +import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.parse.parse import com.xemantic.markanywhere.render.renderMarkdown @@ -1292,6 +1293,87 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should drop a frontmatter left empty by dropping its empty title entries`() = runTest { + // given — an empty frontmatter renders as `---` twice, which parses + // back as two thematic breaks + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title", "type" to "null") { } + "entry"("key" to "Title") { } + } + "p" { +"Body." } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "p" { +"Body." } + } + } + + @Test + fun `should round-trip Markdown whose front matter holds only a null title`() = runTest { + // given + val markdown = "---\ntitle:\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs "Body." + } + + @Test + fun `should treat a title entry of control chars as blank`() = runTest { + // given — a next line char or a C0 control shows nothing, and the + // title read back from `<title>` would be empty + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\u0085\u0001" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } + } + "h1" { +"Heading" } + } + } + + @Test + fun `should judge the title entries of a frontmatter the stream ends inside`() = runTest { + // given — a broken upstream contract: neither the last entry nor the + // frontmatter is closed, yet wrapInHtmlDocument reads the open entry. + // Built from raw events on purpose: the balanced builders cannot + // express an unclosed mark. + val input = flowOf( + SemanticEvent.Mark(name = "frontmatter", isTagged = false), + SemanticEvent.Mark(name = "entry", isTagged = false, attributes = mapOf("key" to "title", "type" to "null")), + SemanticEvent.Unmark(name = "entry", isTagged = false), + SemanticEvent.Mark(name = "entry", isTagged = false, attributes = mapOf("key" to "Title")), + SemanticEvent.Text("Hello"), + ) + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs flowOf( + SemanticEvent.Mark(name = "frontmatter", isTagged = false), + SemanticEvent.Mark(name = "entry", isTagged = false, attributes = mapOf("key" to "title")), + SemanticEvent.Text("Hello"), + ) + } + @Test fun `should collapse line-breaking whitespace in the derived title`() = runTest { // given — a line or paragraph separator, a vertical tab or a next From 2f9420565e1ef4616ef24d50fff40a96a6815037 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 14:12:01 +0200 Subject: [PATCH 21/25] Keep only the read title beside a nested one; cap parsed arrays (#82) - ensureFrontmatterTitle keeps, beside a nested title entry, only the title variant wrapInHtmlDocument reads (via HeadMetadata, now tracking the source entry), and drops a frontmatter arriving empty even when no title can be derived - share the frontmatter-opens-the-stream rule (opensFrontmatter) between wrapInHtmlDocument and ensureFrontmatterTitle - isApplicationStateMeta judges a flat array past MAX_PARSED_LENGTH as written instead of parsing it; restructure into named helpers - normalizeLang trims invisible chars (BOM, zero-width space) around the lang in simplifyHtml and wrapInHtmlDocument Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../commonMain/kotlin/ApplicationStateMeta.kt | 59 +++++++++------ .../kotlin/EnsureFrontmatterTitle.kt | 68 +++++++++--------- .../kotlin/FrontMatterEntryReader.kt | 17 ++++- .../src/commonMain/kotlin/HeadMetadata.kt | 17 +++-- .../src/commonMain/kotlin/SimplifyHtml.kt | 5 +- .../commonMain/kotlin/WrapInHtmlDocument.kt | 12 ++-- .../kotlin/EnsureFrontmatterTitleTest.kt | 52 ++++++++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 71 +++++++++++++++++++ .../kotlin/WrapInHtmlDocumentTest.kt | 26 +++++++ 9 files changed, 253 insertions(+), 74 deletions(-) diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 853664e..0b54738 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -50,34 +50,47 @@ import kotlinx.serialization.json.jsonPrimitive // elements joined, not the quotes and commas serialising them — its length // as well as whether it reads as text — so it is judged as the same list // written plainly would be: a long list of words is kept, a long list of -// hashes is not, and one whose elements fit under the cap is short. +// hashes is not, and one whose elements fit under the cap is short. Past +// [MAX_PARSED_LENGTH] it is judged as written, like any other shape. internal fun isApplicationStateMeta(content: String): Boolean { val written = content.trimInvisible() - val long = written.length > MAX_META_VALUE_LENGTH - val writtenReadsAsText by lazy { written.readsAsText() } - // An over-long value is kept only if it reads as text, whatever it - // parses or decodes as, so a blob — megabytes of JSON or of escapes, - // say — is dropped unparsed and undecoded. Only a raw array must be - // parsed first, to be read by its words — unless an element of it is an - // object or an array: then it is state if it parses, and otherwise - // dropped unparsed unless it reads as text, as any over-long value. - val readByWords = written.firstOrNull() == '[' - if (long && (!readByWords || written.hasNestedElement()) && !writtenReadsAsText) return true - val decoded = written.percentDecodedOrNull(MAX_DECODING_DEPTH) - val json = (decoded?.value ?: written).parseJsonCandidateOrNull() - if (json?.isState(decoded?.layersLeft ?: MAX_DECODING_DEPTH) == true) return true - if (!long || !readByWords) return false - if (json !is JsonArray) return !writtenReadsAsText - // a flat array is measured by its elements joined, in length as in text, - // as the same elements written as a plain list would be - val text = json.joinToString(" ") { it.jsonPrimitive.content } - return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() + return when { + written.length <= MAX_META_VALUE_LENGTH -> written.isStructuralState() + written.length <= MAX_PARSED_LENGTH && written.isFlatArray() -> written.isLongFlatArrayState() + // decoded and parsed only once it reads as text, so a blob — + // megabytes of JSON or of escapes, say — is dropped unparsed + else -> !written.readsAsText() || written.isStructuralState() + } } // Real metadata is short: `description` / `og:description` rarely exceed 300 // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 +// Past this, even a flat array is not parsed to be read by its words: its +// tree would cost many times the value's size, for a list no page writes. +private const val MAX_PARSED_LENGTH = 16 * MAX_META_VALUE_LENGTH + +// Whether this value, percent-decoded when it is encoded, parses as state. +private fun String.isStructuralState(): Boolean { + val decoded = percentDecodedOrNull(MAX_DECODING_DEPTH) + val json = (decoded?.value ?: this).parseJsonCandidateOrNull() ?: return false + return json.isState(decoded?.layersLeft ?: MAX_DECODING_DEPTH) +} + +// A raw array none of whose elements is an object or an array. +private fun String.isFlatArray(): Boolean = firstOrNull() == '[' && !hasNestedElement() + +// An over-long flat array: state if it parses as state, a blob if it does not +// parse and does not read as text, and otherwise judged by its elements +// joined, as the same list written plainly would be. +private fun String.isLongFlatArrayState(): Boolean { + val json = parseJsonOrNull() as? JsonArray ?: return !readsAsText() + if (json.isState(MAX_DECODING_DEPTH)) return true + val text = json.joinToString(" ") { it.jsonPrimitive.content } + return text.length > MAX_META_VALUE_LENGTH && !text.readsAsText() +} + // Whether this (raw, possibly malformed) array holds an object or an array // — a bracket past the opening one outside a JSON string — found by a scan, // without parsing. @@ -130,9 +143,9 @@ private fun JsonElement.isEncodedState(layers: Int): Boolean { } // Text is made of words: at least half the chars are letters (hex and -// number lists are mostly digits) — a combining mark counting as one, since -// scripts like Devanagari and vowel-marked Arabic write vowels as marks —, and at least one char in sixteen breaks a -// word — whitespace, a list separator (`,` `;`, so `a,b,c` keywords count), +// number lists are mostly digits), a combining mark counting as one, since +// scripts like Devanagari and vowel-marked Arabic write vowels as marks; and +// at least one char in sixteen breaks a word — whitespace, a list separator (`,` `;`, so `a,b,c` keywords count), // or a letter of a script written without spaces (anything past ASCII — // base64 and hex never contain one) — sparse enough for a list of long // compound words, while the punctuation of serialised data (quotes, diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index b8ddad7..f01655e 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -38,10 +38,11 @@ import kotlinx.coroutines.flow.Flow * A title entry holding a nested structure (localized titles, say) is no * title for [wrapInHtmlDocument], but it is content: no title is derived, * no other title entry respelled, and the frontmatter passes through with - * only its empty title entries — `null`, blank, an empty collection — - * dropped, as they carry nothing and a duplicate key makes js-yaml reject the - * whole front matter. A usable title variant beside it is kept as spelled, - * so readers may disagree on the title as they did on the input. A + * its other title entries dropped — the empty ones (`null`, blank, an empty + * collection) carry nothing, and a duplicate key makes js-yaml reject the + * whole front matter — but for the usable one [wrapInHtmlDocument] picks, + * kept as spelled, so readers may disagree on the title as they did on the + * input. A * frontmatter whose root is a sequence (top-level `item` marks) has no place * for a title entry and passes through unchanged. * @@ -70,9 +71,9 @@ import kotlinx.coroutines.flow.Flow * When no title can be derived — the first non-blank event after the * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — * everything held is flushed unchanged but for the empty title entries of - * the frontmatter, dropped as above — the frontmatter too, when they were - * all it held: a stream without a leading `h1` passes through untouched and - * no empty frontmatter is fabricated. A frontmatter the stream ends inside (a + * the frontmatter, dropped as above — the frontmatter too, when it is left + * empty or came so: a stream without a leading `h1` passes through + * untouched but for that, and no empty frontmatter is fabricated. A frontmatter the stream ends inside (a * broken upstream contract) is judged all the same, an entry left open * included. * @@ -90,16 +91,18 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // the top-level title entries (in any letter case) within // `frontmatterEvents`, in source order, their spans indexing it val titleSlots = mutableListOf<FrontMatterEntry>() + // the title as wrapInHtmlDocument reads it, which of them it is read from + val titles = HeadMetadata() val reader = FrontMatterEntryReader { entry -> - if (entry.key.asciiLowercase() == "title") titleSlots += entry + if (entry.key.asciiLowercase() == "title") { + titleSlots += entry + titles.addFromFrontMatter(entry) + } } // blank text held before the first h1 (ahead of the frontmatter, or // between it and the h1), replayed in source order on commit val blanks = mutableListOf<SemanticEvent>() - // whether all of `blanks` is HTML whitespace, which alone lets a - // frontmatter following it open the stream - var blanksAreHtmlWhitespace = true // the h1 subtree, buffered between its mark and balanced unmark so the // title can be derived from the flattened text @@ -112,16 +115,11 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s blanks.clear() } - fun holdBlank(event: SemanticEvent.Text) { - blanks += event - if (!event.text.isHtmlBlank()) blanksAreHtmlWhitespace = false - } - // emits the held frontmatter — `prepended` right after its mark, the span // of each slot in `rewrite` (in source order, as `titleSlots` is) replaced // by its events (none drops the entry) — then the blanks held with it. - // A frontmatter the rewrite leaves empty is dropped: it would render as a - // `---` pair, which parses back as two thematic breaks. + // A frontmatter left empty, by the rewrite or as it came, is dropped: it + // would render as a `---` pair, which parses back as two thematic breaks. suspend fun flushHeld( rewrite: Map<FrontMatterEntry, List<SemanticEvent>> = emptyMap(), prepended: List<SemanticEvent> = emptyList() @@ -137,10 +135,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } addAll(held.subList(next, held.size)) } - val emptied = rewrite.isNotEmpty() && body.none { + val empty = body.none { it is Mark || (it is Text && !it.text.isHtmlBlank()) } - if (!emptied) { + if (!empty) { emit(held.first()) emit(body) } @@ -155,12 +153,14 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s flushHeld(titleSlots.associateWith { if (it == slot) entry else emptyList() }) } - // emits the held frontmatter without its empty title entries, keeping - // the nested and the usable ones - suspend fun flushWithoutEmptyTitles() { + // emits the held frontmatter keeping, of its title entries, only the + // nested ones and the one wrapInHtmlDocument reads: the empty ones carry + // nothing, and another usable one would be a duplicate key + suspend fun flushWithReadTitleOnly() { + val read = titles["title"]?.source flushHeld( titleSlots - .filterNot { it.hasChildren || it.isHeadMetadata } + .filterNot { it.hasChildren || it == read } .associateWith { emptyList() } ) } @@ -169,7 +169,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s val title = headingText.toString().normalizeTitle() val slot = titleSlots.firstOrNull() when { - !isMetadataValue(title) -> flushWithoutEmptyTitles() + !isMetadataValue(title) -> flushWithReadTitleOnly() frontmatterEvents == null -> { "frontmatter" { emit(titleEntry(title)) @@ -187,14 +187,12 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // decides, once the frontmatter closes, whether it holds a usable title suspend fun judgeFrontmatter() { // the entry wrapInHtmlDocument reads - val usable = titleSlots - .filter { it.isHeadMetadata } - .reduceOrNull { read, next -> if (frontMatterKeySupersedes(read.key, next.key)) next else read } + val usable = titles["title"]?.source state = when { // a nested title is content, not ours to replace or drop; a // sequence has no place for a title entry titleSlots.any { it.hasChildren } || reader.isSequence -> { - flushWithoutEmptyTitles() + flushWithReadTitleOnly() PassThrough } usable != null -> { @@ -218,13 +216,13 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // only a frontmatter opening the stream (but for HTML // whitespace text) is the page's metadata; anywhere else it // is content - event is Mark && !event.isTagged && event.name == "frontmatter" && blanksAreHtmlWhitespace -> { + event.opensFrontmatter(blanks) -> { frontmatterEvents = mutableListOf(event) reader.read(event) state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) - event is Text && event.text.isBlank() -> holdBlank(event) + event is Text && event.text.isBlank() -> blanks += event else -> { flushBlanks() emit(event) @@ -236,10 +234,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s if (reader.read(event)) judgeFrontmatter() } AwaitingHeading -> when (event) { - is Text if event.text.isBlank() -> holdBlank(event) + is Text if event.text.isBlank() -> blanks += event is Mark if event.name == "h1" -> startHeading(event) else -> { - flushWithoutEmptyTitles() + flushWithReadTitleOnly() emit(event) state = PassThrough } @@ -275,10 +273,10 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s InFrontmatter -> { reader.finish() judgeFrontmatter() - if (state == AwaitingHeading) flushWithoutEmptyTitles() + if (state == AwaitingHeading) flushWithReadTitleOnly() } InHeading -> commitHeading() - AwaitingHeading -> flushWithoutEmptyTitles() + AwaitingHeading -> flushWithReadTitleOnly() else -> flushHeld() } } diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt index 14d4055..a211407 100644 --- a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -17,6 +17,20 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent +import com.xemantic.markanywhere.html.spec.isHtmlBlank + +// Whether this event opens the frontmatter that is the page's metadata, the +// `preceding` events being all the stream held before it: an untagged +// `frontmatter` mark, nothing but [insignificant][mayPrecedeFrontmatter] +// text ahead of it. The one rule wrapInHtmlDocument reads the head by and +// ensureFrontmatterTitle judges the title by — anywhere else a frontmatter +// is content. +internal fun SemanticEvent.opensFrontmatter(preceding: List<SemanticEvent>): Boolean = + this is Mark && !isTagged && name == "frontmatter" && preceding.all { it.mayPrecedeFrontmatter() } + +// Whether this event may come ahead of the frontmatter without keeping it +// from opening the stream: text of HTML whitespace (an NBSP is content). +internal fun SemanticEvent.mayPrecedeFrontmatter(): Boolean = this is Text && text.isHtmlBlank() // A top-level front matter `entry`: its key as spelled, `type`, text, whether // it holds nested marks, and the span of its events among those read. @@ -31,8 +45,7 @@ internal class FrontMatterEntry( // Whether it holds head metadata: scalar text with visible content — // not a nested structure, an empty collection, `null` or blank text. - val isHeadMetadata: Boolean - get() = !hasChildren && isScalarEntryType(type) && isMetadataValue(text) + val isHeadMetadata: Boolean = !hasChildren && isScalarEntryType(type) && isMetadataValue(text) } diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 37dcea6..0916397 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -61,6 +61,11 @@ internal fun String.normalizeTitle(): String = private const val LINE_BREAKING_WHITESPACE = "\u000B\u0085\u2028\u2029" +// A language tag as `<html lang>` carries it: HTML strips its whitespace, and +// any other invisible char at its edges (a byte order mark, a zero-width +// space) is trimmed too — no valid BCP 47 tag holds one. +internal fun String.normalizeLang(): String = trimInvisible() + // Whether a front matter entry spelled `candidate` supersedes an earlier one // of the same name spelled `existing`, as front matter readers resolve a // duplicate key: the later one wins (Psych — Jekyll's — and PyYAML), except @@ -71,9 +76,13 @@ internal fun frontMatterKeySupersedes(existing: String, candidate: String): Bool return candidate == name || existing != name } -// A metadata entry: its name, spelled as where its value was read, and the -// value. -internal data class MetadataEntry(val key: String, val value: String) +// A metadata entry: its name, spelled as where its value was read, the +// value, and the front matter entry it was read from, if any. +internal data class MetadataEntry( + val key: String, + val value: String, + val source: FrontMatterEntry? = null +) // The `<head>` metadata a front matter holds, in insertion order and keyed // the way HTML reads `<meta>` names: ASCII case-insensitively (HTML §4.2.5). @@ -115,7 +124,7 @@ internal class HeadMetadata { val name = key.asciiLowercase() val existing = entries[name] if (existing == null || frontMatterKeySupersedes(existing.key, key)) { - entries[name] = MetadataEntry(key, entry.text) + entries[name] = MetadataEntry(key, entry.text, source = entry) } } diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index 1cc9197..f9bfd12 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -19,7 +19,6 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.dump.AccessibilityAnnotations import com.xemantic.markanywhere.html.spec.asciiLowercase -import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import com.xemantic.markanywhere.transform.MatcherScope import com.xemantic.markanywhere.transform.transform import kotlinx.coroutines.flow.Flow @@ -266,7 +265,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // --- metadata extraction (explicit per-tag) ------------------------- match("html") { event -> - event["lang"]?.let { metadata.addFromHtml("lang", it.stripHtmlWhitespace()) } + event["lang"]?.let { metadata.addFromHtml("lang", it.normalizeLang()) } children() } @@ -317,7 +316,7 @@ public fun Flow<SemanticEvent>.simplifyHtml( // as a <title> element's text reads "title" -> content.normalizeTitle() // as <html lang> is read above - "lang" -> content.stripHtmlWhitespace() + "lang" -> content.normalizeLang() else -> content } // the keys wrapInHtmlDocument turns back into <title> and diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index bdd3bd4..7c70b42 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -19,8 +19,6 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.flow.semanticEvents import com.xemantic.markanywhere.html.spec.asciiLowercase -import com.xemantic.markanywhere.html.spec.isHtmlBlank -import com.xemantic.markanywhere.html.spec.stripHtmlWhitespace import kotlinx.coroutines.flow.Flow /** @@ -32,8 +30,8 @@ import kotlinx.coroutines.flow.Flow * inverse of [simplifyHtml]'s head-to-frontmatter extraction: * * - the `title` entry becomes `<title>` - * - the `lang` entry, its HTML whitespace stripped, becomes the `lang` - * attribute on `<html>` + * - the `lang` entry, its HTML whitespace and any other invisible char at + * its edges stripped, becomes the `lang` attribute on `<html>` * - every other top-level scalar entry becomes a void `<meta name content>` * * Only top-level scalar entries are interpreted — exactly the shape @@ -85,7 +83,7 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman "html", attributes = metadata["lang"] // trimmed, as simplifyHtml reads it - ?.let { mapOf("lang" to it.value.stripHtmlWhitespace()) } + ?.let { mapOf("lang" to it.value.normalizeLang()) } ?: emptyMap() ) "head" { @@ -109,10 +107,10 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman when { reader != null -> if (reader.read(event)) openDocument() !opened -> when { - event is Mark && !event.isTagged && event.name == "frontmatter" -> { + event.opensFrontmatter(blanks) -> { frontmatter = FrontMatterEntryReader(metadata::addFromFrontMatter).also { it.read(event) } } - event is Text && event.text.isHtmlBlank() -> blanks += event + event.mayPrecedeFrontmatter() -> blanks += event else -> { openDocument() emit(event) diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 57632a7..381deab 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -728,6 +728,37 @@ class EnsureFrontmatterTitleTest { } } + + @Test + fun `should keep only the title variant wrapInHtmlDocument reads beside a nested title entry`() = runTest { + // given — two variants spelled alike are a duplicate key, which makes + // js-yaml reject the whole front matter + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "title") { +"First" } + "entry"("key" to "title") { +"Second" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "Title") { + "entry"("key" to "en") { +"Hello" } + } + "entry"("key" to "title") { +"Second" } + } + "h1" { +"Heading" } + } + } + @Test fun `should not add a title entry to a frontmatter whose root is a sequence`() = runTest { // given — a title entry beside the items would make the root neither @@ -1314,6 +1345,27 @@ class EnsureFrontmatterTitleTest { } } + + @Test + fun `should drop a frontmatter arriving empty when no title can be derived`() = runTest { + // given — it renders as `---` twice all the same, which parses back + // as two thematic breaks + val input = semanticEvents { + "frontmatter" { + +"\n" + } + "p" { +"Body." } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "p" { +"Body." } + } + } + @Test fun `should round-trip Markdown whose front matter holds only a null title`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 2823c6c..b9846af 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1227,6 +1227,30 @@ class SimplifyHtmlTest { } } + + @Test + fun `should judge a flat JSON array too long to parse as written`() = runTest { + // given — words, but far past anything metadata holds: parsing it + // would cost many times its size, so its quotes and commas count + val words = (1..10000).joinToString(",", "[", "]") { "\"lorem\"" } + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to words) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "p" { +"text" } + } + } + @Test fun `should drop a long array holding a nested element unless it reads as text`() = runTest { // given — state if it parses, a blob if it does not; neither is text @@ -1434,6 +1458,53 @@ class SimplifyHtmlTest { } } + + @Test + fun `should trim invisible chars around the html lang`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html"("lang" to "\uFEFFen\u200B") { + "head" { } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"en" } + } + "p" { +"text" } + } + } + + @Test + fun `should trim invisible chars around a lang meta`() = runTest { + // given + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "lang", "content" to "\u200Bde\uFEFF") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"de" } + } + "p" { +"text" } + } + } + @Test fun `should trim HTML whitespace around the html lang`() = runTest { // given diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index 1ab9a52..da86f74 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -127,6 +127,32 @@ class WrapInHtmlDocumentTest { } } + + @Test + fun `should trim invisible chars around the lang`() = runTest { + // given — a byte order mark or a zero-width space makes no valid + // language tag, as they are trimmed from a title + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "lang") { +"\uFEFFen\u200B" } + } + "p" { +"Body." } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html"("lang" to "en") { + "head" { } + "body" { + "p" { +"Body." } + } + } + } + } + @Test fun `should use scalar text verbatim including decoded quoting`() = runTest { // given — the parser has already decoded the YAML; the value carries a From 958a48748aeb26f951f0a2d033f078752c544f39 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 14:38:53 +0200 Subject: [PATCH 22/25] Balance held output; treat invisible text as blank; keep URL lists (#82) - ensureFrontmatterTitle closes a frontmatter or h1 the stream ends inside, so its output stays balanced, and treats text showing nothing (BOM, zero-width space) around the h1 as blank, as a title is judged - pick the title entry via FrontMatterEntry.readOver, shared with HeadMetadata, instead of tracking the source entry in MetadataEntry - wrapInHtmlDocument normalizes the title as simplifyHtml reads a <title> - isApplicationStateMeta counts `/` as a word break, so a long list of URLs is kept while base64 stays dropped - drop the now unused stripHtmlWhitespace from markanywhere-html-spec - restructure the ensureFrontmatterTitle and wrapInHtmlDocument KDoc Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- markanywhere-html-spec/README.md | 1 - .../api/markanywhere-html-spec.api | 1 - .../src/commonMain/kotlin/HtmlWhitespace.kt | 8 -- .../commonTest/kotlin/HtmlWhitespaceTest.kt | 9 -- .../commonMain/kotlin/ApplicationStateMeta.kt | 14 ++- .../kotlin/EnsureFrontmatterTitle.kt | 116 ++++++++++-------- .../src/commonMain/kotlin/HeadMetadata.kt | 32 +++-- .../commonMain/kotlin/WrapInHtmlDocument.kt | 30 +++-- .../kotlin/EnsureFrontmatterTitleTest.kt | 56 ++++++++- .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 28 +++++ .../kotlin/WrapInHtmlDocumentTest.kt | 37 +++++- 11 files changed, 220 insertions(+), 112 deletions(-) diff --git a/markanywhere-html-spec/README.md b/markanywhere-html-spec/README.md index c879372..7c4d66a 100644 --- a/markanywhere-html-spec/README.md +++ b/markanywhere-html-spec/README.md @@ -11,7 +11,6 @@ so code that reads a semantic event stream carrying HTML attributes can use it w |---------------------------------------------------|------------------------------------------------------------------------------------------------| | `HTML_WHITESPACE_CHARS`, `Char.isHtmlWhitespace()` | HTML "ASCII whitespace": TAB, LF, FF, CR, SPACE — not NBSP, which HTML treats as content | | `String.isHtmlBlank()` | Empty or HTML whitespace only | -| `String.stripHtmlWhitespace()` | Trimmed of HTML whitespace at both ends, NBSP kept | | `String.stripAndCollapseHtmlWhitespace()` | Trimmed of HTML whitespace, inner runs collapsed to one space, as `document.title` reads it | | `Char.asciiLowercase()`, `String.asciiLowercase()` | ASCII lowercase: only `A`–`Z` fold, as HTML compares names "ASCII case-insensitively" | | `HTML_VOID_ELEMENTS` | Elements with no content and no closing tag, including the obsolete `keygen` and `param` | diff --git a/markanywhere-html-spec/api/markanywhere-html-spec.api b/markanywhere-html-spec/api/markanywhere-html-spec.api index 71fb6d8..9cb4fbe 100644 --- a/markanywhere-html-spec/api/markanywhere-html-spec.api +++ b/markanywhere-html-spec/api/markanywhere-html-spec.api @@ -12,7 +12,6 @@ public final class com/xemantic/markanywhere/html/spec/HtmlWhitespaceKt { public static final fun isHtmlBlank (Ljava/lang/String;)Z public static final fun isHtmlWhitespace (C)Z public static final fun stripAndCollapseHtmlWhitespace (Ljava/lang/String;)Ljava/lang/String; - public static final fun stripHtmlWhitespace (Ljava/lang/String;)Ljava/lang/String; } public final class com/xemantic/markanywhere/html/spec/MarkClassListKt { diff --git a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt index 3f469f7..33a1b97 100644 --- a/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt +++ b/markanywhere-html-spec/src/commonMain/kotlin/HtmlWhitespace.kt @@ -42,14 +42,6 @@ public fun Char.isHtmlWhitespace(): Boolean = this in HTML_WHITESPACE_CHARS */ public fun String.isHtmlBlank(): Boolean = all { it.isHtmlWhitespace() } -/** - * This string with leading and trailing [HTML whitespace][isHtmlWhitespace] - * removed — the WHATWG Infra "strip leading and trailing ASCII whitespace", - * which HTML applies to attribute values such as `lang`. NBSP is content and - * stays. - */ -public fun String.stripHtmlWhitespace(): String = trim { it.isHtmlWhitespace() } - /** * This string with leading and trailing [HTML whitespace][isHtmlWhitespace] * removed and every inner run of it replaced by a single space — the WHATWG diff --git a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt index d218ea9..43f2e65 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt @@ -97,13 +97,4 @@ class HtmlWhitespaceTest { assert(result == "") } - @Test - fun `should strip leading and trailing HTML whitespace keeping inner runs and NBSP`() { - // when - val result = "\n\t\u00A0a b\u00A0 \u000C\r".stripHtmlWhitespace() - - // then - assert(result == "\u00A0a b\u00A0") - } - } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index 0b54738..ce5a38d 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -145,11 +145,13 @@ private fun JsonElement.isEncodedState(layers: Int): Boolean { // Text is made of words: at least half the chars are letters (hex and // number lists are mostly digits), a combining mark counting as one, since // scripts like Devanagari and vowel-marked Arabic write vowels as marks; and -// at least one char in sixteen breaks a word — whitespace, a list separator (`,` `;`, so `a,b,c` keywords count), -// or a letter of a script written without spaces (anything past ASCII — -// base64 and hex never contain one) — sparse enough for a list of long -// compound words, while the punctuation of serialised data (quotes, -// brackets, `=`, `|`, `\`) stays rare. +// at least one char in sixteen breaks a word — whitespace, a list separator +// (`,` `;`, so `a,b,c` keywords count), a path separator (`/`, so a list of +// URLs counts — base64 holds one in 64 chars, too few), or a letter of a +// script written without spaces (anything past ASCII — base64 and hex never +// contain one) — sparse enough for a list of long compound words, while the +// punctuation of serialised data (quotes, brackets, `=`, `|`, `\`) stays +// rare. // Chars are counted as code points: a surrogate pair — a letter of a // supplementary-plane script (CJK Extension B, historic scripts) or an emoji, // which the common stdlib cannot classify — counts once, as a letter of a @@ -170,7 +172,7 @@ private fun String.readsAsText(): Boolean { continue } if (c.isLetter() || c.category in COMBINING_MARKS) letters++ - if (c.isWhitespace() || c == ',' || c == ';' || c.code > 0x7F && c.isLetter()) wordBreaks++ + if (c.isWhitespace() || c == ',' || c == ';' || c == '/' || c.code > 0x7F && c.isLetter()) wordBreaks++ else if (c in DATA_PUNCTUATION) dataPunctuation++ i++ } diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index f01655e..8d881be 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -35,47 +35,48 @@ import kotlinx.coroutines.flow.Flow * scalar text with visible content is usable — entries holding a nested * structure, an empty collection, `null` or blank text are not, as there. * - * A title entry holding a nested structure (localized titles, say) is no - * title for [wrapInHtmlDocument], but it is content: no title is derived, - * no other title entry respelled, and the frontmatter passes through with - * its other title entries dropped — the empty ones (`null`, blank, an empty - * collection) carry nothing, and a duplicate key makes js-yaml reject the - * whole front matter — but for the usable one [wrapInHtmlDocument] picks, - * kept as spelled, so readers may disagree on the title as they did on the - * input. A - * frontmatter whose root is a sequence (top-level `item` marks) has no place - * for a title entry and passes through unchanged. - * - * Otherwise, whenever a title comes out, the frontmatter holds exactly one title entry, + * Whenever a title comes out, the frontmatter holds exactly one title entry, * spelled `title`: every front matter reader then reads the same title — * whether it matches keys case-sensitively (Jekyll) or not, keeps the later * of duplicate keys (Psych, PyYAML) or rejects them (js-yaml) — and it is the - * one [wrapInHtmlDocument] reads. With a usable title entry that is the one - * [wrapInHtmlDocument] picks, respelled `title` in place; every other title - * entry is dropped. + * one [wrapInHtmlDocument] reads. It comes out in one of two ways: + * + * - **A usable title entry exists**: the one [wrapInHtmlDocument] picks is + * respelled `title` in place, and every other title entry is dropped. + * - **None exists**: the frontmatter is held back and the title is derived + * from the very first `h1` following it, only text showing nothing + * (whitespace, an NBSP, a zero-width char) intervening. The `h1` subtree's + * flattened text — its text events plus the `alt` of every `img` mark, in + * document order, the way an accessible name is computed from content — + * normalized as [normalizeTitle] reads a `<title>` becomes the `title`, and + * only then are the frontmatter and the buffered `h1` emitted, in source + * order. The derived entry takes the place of the first title entry, every + * other one dropped, or is injected as the first `entry` of a frontmatter + * without any. When no frontmatter exists at all, one carrying just the + * derived `title` is synthesized as the **first** event, ahead of any text + * that preceded the `h1`. * - * Otherwise the frontmatter is held back and the title is derived from the - * very first `h1` following it (only blank text may intervene — an NBSP - * included, which keeps a frontmatter *following* it from opening the - * stream, but not a synthesized one from going first): the - * `h1` subtree's flattened text — its text events plus the `alt` of every - * `img` mark, in document order, the way an accessible name is computed from - * content — normalized as [normalizeTitle] reads a `<title>` — becomes the - * `title`, and only then the frontmatter and the - * buffered `h1` are emitted, in source order. The derived entry takes the - * place of the first title entry, every other one dropped, or is injected as - * the first `entry` of a frontmatter without any. When no frontmatter exists - * at all, one carrying just the derived `title` is synthesized as the - * **first** event — ahead of any whitespace text that preceded the `h1`. + * Two shapes are left without a title: * - * When no title can be derived — the first non-blank event after the - * frontmatter is not an `h1`, the `h1` yields no text, or the stream ends — - * everything held is flushed unchanged but for the empty title entries of - * the frontmatter, dropped as above — the frontmatter too, when it is left - * empty or came so: a stream without a leading `h1` passes through - * untouched but for that, and no empty frontmatter is fabricated. A frontmatter the stream ends inside (a - * broken upstream contract) is judged all the same, an entry left open - * included. + * - A title entry holding a nested structure (localized titles, say) is no + * title for [wrapInHtmlDocument], but it is content: no title is derived + * and no title entry respelled. Of the other title entries only the usable + * one [wrapInHtmlDocument] picks is kept, as spelled, so readers may + * disagree on the title as they did on the input; the empty ones (`null`, + * blank, an empty collection) carry nothing, and another usable one would + * be a duplicate key, which makes js-yaml reject the whole front matter. + * A frontmatter whose root is a sequence (top-level `item` marks) has no + * place for a title entry and passes through unchanged. + * - When no title can be derived — the first event after the frontmatter + * that shows something is not an `h1`, the `h1` yields no text, or the + * stream ends — everything held is flushed unchanged but for the title + * entries dropped as above. + * + * A frontmatter left empty, by dropping its title entries or as it came, is + * dropped too (it would render as two thematic breaks), so no empty + * frontmatter is ever emitted. A frontmatter or `h1` the stream ends inside + * (a broken upstream contract) is judged all the same, an entry left open + * included, and emitted closed, so the output stays balanced. * * Buffering is bounded to the frontmatter subtree plus one `h1` subtree — * everything after the decision point is forwarded as it arrives. @@ -91,17 +92,18 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // the top-level title entries (in any letter case) within // `frontmatterEvents`, in source order, their spans indexing it val titleSlots = mutableListOf<FrontMatterEntry>() - // the title as wrapInHtmlDocument reads it, which of them it is read from - val titles = HeadMetadata() + // the one of them wrapInHtmlDocument reads the title from, if any + var read: FrontMatterEntry? = null val reader = FrontMatterEntryReader { entry -> if (entry.key.asciiLowercase() == "title") { titleSlots += entry - titles.addFromFrontMatter(entry) + if (entry.readOver(read?.key)) read = entry } } - // blank text held before the first h1 (ahead of the frontmatter, or - // between it and the h1), replayed in source order on commit + // text showing nothing held before the first h1 (ahead of the + // frontmatter, or between it and the h1), replayed in source order on + // commit val blanks = mutableListOf<SemanticEvent>() // the h1 subtree, buffered between its mark and balanced unmark so the @@ -139,8 +141,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s it is Mark || (it is Text && !it.text.isHtmlBlank()) } if (!empty) { - emit(held.first()) - emit(body) + emit((listOf(held.first()) + body).closed()) } } frontmatterEvents = null @@ -157,7 +158,6 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // nested ones and the one wrapInHtmlDocument reads: the empty ones carry // nothing, and another usable one would be a duplicate key suspend fun flushWithReadTitleOnly() { - val read = titles["title"]?.source flushHeld( titleSlots .filterNot { it.hasChildren || it == read } @@ -179,15 +179,14 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s slot == null -> flushHeld(prepended = titleEntry(title)) else -> flushWithSingleTitle(slot, titleEntry(title)) } - emit(headingEvents) + emit(headingEvents.closed()) headingEvents.clear() state = PassThrough } // decides, once the frontmatter closes, whether it holds a usable title suspend fun judgeFrontmatter() { - // the entry wrapInHtmlDocument reads - val usable = titles["title"]?.source + val usable = read state = when { // a nested title is content, not ours to replace or drop; a // sequence has no place for a title entry @@ -222,7 +221,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s state = InFrontmatter } event is Mark && event.name == "h1" -> startHeading(event) - event is Text && event.text.isBlank() -> blanks += event + event.showsNothing() -> blanks += event else -> { flushBlanks() emit(event) @@ -234,7 +233,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s if (reader.read(event)) judgeFrontmatter() } AwaitingHeading -> when (event) { - is Text if event.text.isBlank() -> blanks += event + is Text if event.showsNothing() -> blanks += event is Mark if event.name == "h1" -> startHeading(event) else -> { flushWithReadTitleOnly() @@ -281,6 +280,25 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } } +// Whether this event is text a reader sees nothing of — the one test of +// blankness around the h1, the one a title is judged by ([isMetadataValue]). +// Which text may precede a frontmatter is a different, HTML question +// ([mayPrecedeFrontmatter]). +private fun SemanticEvent.showsNothing(): Boolean = this is Text && !isMetadataValue(text) + +// These events with an unmark appended for every mark left open, innermost +// first. +private fun List<SemanticEvent>.closed(): List<SemanticEvent> { + val open = ArrayDeque<SemanticEvent.Mark>() + for (event in this) when (event) { + is Mark -> open.addLast(event) + is Unmark -> open.removeLastOrNull() + else -> {} + } + return if (open.isEmpty()) this + else this + open.reversed().map { SemanticEvent.Unmark(it.name, it.isTagged) } +} + private enum class State { AtStart, InFrontmatter, AwaitingHeading, InHeading, PassThrough } diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index 0916397..ad53138 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -76,13 +76,16 @@ internal fun frontMatterKeySupersedes(existing: String, candidate: String): Bool return candidate == name || existing != name } -// A metadata entry: its name, spelled as where its value was read, the -// value, and the front matter entry it was read from, if any. -internal data class MetadataEntry( - val key: String, - val value: String, - val source: FrontMatterEntry? = null -) +// Whether a front matter reader reads this entry's value over that of an +// earlier entry of the same name spelled `existingKey`, if any: it must be +// head metadata ([FrontMatterEntry.isHeadMetadata]), and must then +// [supersede][frontMatterKeySupersedes] the earlier one. +internal fun FrontMatterEntry.readOver(existingKey: String?): Boolean = + isHeadMetadata && (existingKey == null || frontMatterKeySupersedes(existingKey, key)) + +// A metadata entry: its name, spelled as where its value was read, and the +// value. +internal data class MetadataEntry(val key: String, val value: String) // The `<head>` metadata a front matter holds, in insertion order and keyed // the way HTML reads `<meta>` names: ASCII case-insensitively (HTML §4.2.5). @@ -114,17 +117,12 @@ internal class HeadMetadata { } // Adds a front matter entry the way its readers resolve a duplicate key - // ([frontMatterKeySupersedes]), unless it is no head metadata - // ([FrontMatterEntry.isHeadMetadata], the rule ensureFrontmatterTitle - // judges title entries by). The name keeps the position of its first - // occurrence. + // ([readOver], the rule ensureFrontmatterTitle picks the title entry by). + // The name keeps the position of its first occurrence. fun addFromFrontMatter(entry: FrontMatterEntry) { - if (!entry.isHeadMetadata) return - val key = entry.key - val name = key.asciiLowercase() - val existing = entries[name] - if (existing == null || frontMatterKeySupersedes(existing.key, key)) { - entries[name] = MetadataEntry(key, entry.text, source = entry) + val name = entry.key.asciiLowercase() + if (entry.readOver(entries[name]?.key)) { + entries[name] = MetadataEntry(entry.key, entry.text) } } diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index 7c70b42..d33d96f 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -29,7 +29,8 @@ import kotlinx.coroutines.flow.Flow * YAML `---` front matter, holding `entry` marks) feeds the `head` — the * inverse of [simplifyHtml]'s head-to-frontmatter extraction: * - * - the `title` entry becomes `<title>` + * - the `title` entry, normalized as [simplifyHtml] reads a `<title>`, + * becomes `<title>` * - the `lang` entry, its HTML whitespace and any other invisible char at * its edges stripped, becomes the `lang` attribute on `<html>` * - every other top-level scalar entry becomes a void `<meta name content>` @@ -49,20 +50,24 @@ import kotlinx.coroutines.flow.Flow * (above) and values [simplifyHtml] drops (noise names such as `viewport`, * application state such as a JSON object or an opaque over-long blob) are * gone, case-variant duplicates are merged, the `title` and `lang` keys come - * back spelled in lowercase, the title with its whitespace stripped and - * collapsed (as `document.title` reads it) and `lang` trimmed, and a typed - * scalar comes back as a string. + * back spelled in lowercase, the title and `lang` normalized as above, and a + * typed scalar comes back as a string. + * * Text of HTML whitespace ahead of the frontmatter is insignificant — it is * moved to the start of `body` (a non-breaking space is content, as - * everywhere in HTML, and opens the body instead), as [ensureFrontmatterTitle] moves it after the - * frontmatter. A `frontmatter` mark appearing past any other event is - * ordinary content and flows into `body` verbatim. + * everywhere in HTML, and opens the body instead), as + * [ensureFrontmatterTitle] moves it after the frontmatter. A `frontmatter` + * mark appearing past any other event is ordinary content and flows into + * `body` verbatim. * * Only leading whitespace text and the frontmatter subtree are read ahead - * (bounded); without a frontmatter the document opening is emitted on the - * first other event and body content streams through untouched. All - * synthetic marks are untagged, consistent with the parser's `frontmatter` mark and [simplifyHtml] output. An empty input - * stream still yields the full document skeleton. + * (bounded); body content streams through untouched. Holding the leading + * whitespace delays nothing a reader would see — it renders as nothing, and + * so does the skeleton opened ahead of it — while opening the document on it + * would push a frontmatter following it into `body`. All synthetic marks are + * untagged, consistent with the parser's `frontmatter` mark and + * [simplifyHtml] output. An empty input stream still yields the full + * document skeleton. */ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = semanticEvents { @@ -89,7 +94,8 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman "head" { metadata["title"]?.let { "title" { - +it.value + // as simplifyHtml reads a <title> + +it.value.normalizeTitle() } } for ((key, value) in metadata.values) { diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 381deab..c14b456 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -1418,12 +1418,60 @@ class EnsureFrontmatterTitleTest { // when val output = input.ensureFrontmatterTitle() - // then - output sameAs flowOf( - SemanticEvent.Mark(name = "frontmatter", isTagged = false), - SemanticEvent.Mark(name = "entry", isTagged = false, attributes = mapOf("key" to "title")), + // then — closed, so the stream stays balanced + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + } + } + } + + @Test + fun `should close an h1 the stream ends inside`() = runTest { + // given — a broken upstream contract; built from raw events on + // purpose, as the balanced builders cannot express an unclosed mark + val input = flowOf( + SemanticEvent.Mark(name = "h1", isTagged = false), + SemanticEvent.Mark(name = "em", isTagged = false), SemanticEvent.Text("Hello"), ) + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + } + "h1" { + "em" { +"Hello" } + } + } + } + + @Test + fun `should derive a title past invisible format chars before the h1`() = runTest { + // given — a byte order mark or a zero-width space shows nothing, as + // in a title + val input = semanticEvents { + +"\uFEFF" + +"\u200B" + "h1" { +"Hello" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Hello" } + } + +"\uFEFF" + +"\u200B" + "h1" { +"Hello" } + } } @Test diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index b9846af..52c7909 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1170,6 +1170,34 @@ class SimplifyHtmlTest { } } + @Test + fun `should keep a long list of URLs`() = runTest { + // given — path separators break a URL into words; base64, which + // holds a slash only now and then, stays dropped + val urls = (1..150).joinToString(" ") { "https://www.example.org/reference/article-$it" } + val base64 = "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo/abc+".repeat(120) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "citation_references", "content" to urls) { } + "meta"("name" to "blob", "content" to base64) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "citation_references") { +urls } + } + "p" { +"text" } + } + } + @Test fun `should judge a long flat JSON array by its words and not its quotes and commas`() = runTest { // given — both past the cap; the quotes and commas serialising the diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index da86f74..a99a791 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -127,7 +127,6 @@ class WrapInHtmlDocumentTest { } } - @Test fun `should trim invisible chars around the lang`() = runTest { // given — a byte order mark or a zero-width space makes no valid @@ -153,13 +152,40 @@ class WrapInHtmlDocumentTest { } } + @Test + fun `should normalize the title as simplifyHtml reads it`() = runTest { + // given — invisible chars at the edges, a line separator inside + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\uFEFF Foo\u2028Bar baz " } + } + "p" { +"Body." } + } + + // when + val output = input.wrapInHtmlDocument() + + // then + output sameAs semanticEvents { + "html" { + "head" { + "title" { +"Foo Bar baz" } + } + "body" { + "p" { +"Body." } + } + } + } + } + @Test fun `should use scalar text verbatim including decoded quoting`() = runTest { - // given — the parser has already decoded the YAML; the value carries a - // colon, quotes and a newline as plain text + // given — the parser has already decoded the YAML; the values carry a + // colon, quotes and a newline as plain text (a title is one line) val input = semanticEvents { "frontmatter" { - "entry"("key" to "title") { +"He said: \"hi\"\nBye" } + "entry"("key" to "title") { +"He said: \"hi\"" } + "entry"("key" to "description") { +"Hello\nBye" } "entry"("key" to "og:image") { +"https://example.com/img.png" } } } @@ -171,7 +197,8 @@ class WrapInHtmlDocumentTest { output sameAs semanticEvents { "html" { "head" { - "title" { +"He said: \"hi\"\nBye" } + "title" { +"He said: \"hi\"" } + "meta"("name" to "description", "content" to "Hello\nBye") { } "meta"("name" to "og:image", "content" to "https://example.com/img.png") { } } "body" { } From 47f22f8e430dabe4abf4afdf02a93c37ea9b4751 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 14:56:18 +0200 Subject: [PATCH 23/25] Keep frontmatter detectable; tighten state and blank checks (#82) - ensureFrontmatterTitle no longer drops a leading title entry when that would leave a verbatim line first (front matter detection needs a key on line 2): a kept title variant moves into the first slot instead, or the first slot stays - isMetadataValue treats blank-rendering glyphs (Hangul fillers, blank Braille pattern) as invisible - isApplicationStateMeta never parses a value past MAX_PARSED_LENGTH, dropping one that opens like encoded state, and counts only letters of scripts written without spaces as word breaks - dropHtmlStructuralWhitespace reuses stripAndCollapseHtmlWhitespace - wrapInHtmlDocument checks the frontmatter mark alone (isFrontmatterMark) - html-spec tests use sameAs for string comparisons Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../src/commonTest/kotlin/AsciiCaseTest.kt | 11 ++- .../commonTest/kotlin/HtmlWhitespaceTest.kt | 7 +- .../commonMain/kotlin/ApplicationStateMeta.kt | 49 ++++++++-- .../kotlin/EnsureFrontmatterTitle.kt | 93 ++++++++++++++----- .../kotlin/FrontMatterEntryReader.kt | 7 +- .../src/commonMain/kotlin/HeadMetadata.kt | 13 ++- .../kotlin/HtmlWhitespaceNormalization.kt | 21 +---- .../commonMain/kotlin/WrapInHtmlDocument.kt | 4 +- .../kotlin/EnsureFrontmatterTitleTest.kt | 60 ++++++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 90 ++++++++++++++++++ 10 files changed, 292 insertions(+), 63 deletions(-) diff --git a/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt index 548d6fb..de3098e 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/AsciiCaseTest.kt @@ -18,6 +18,7 @@ package com.xemantic.markanywhere.html.spec import com.xemantic.kotlin.test.assert +import com.xemantic.kotlin.test.sameAs import kotlin.test.Test class AsciiCaseTest { @@ -28,7 +29,7 @@ class AsciiCaseTest { val lowered = "Og:TITLE-1_x".asciiLowercase() // then - assert(lowered == "og:title-1_x") + lowered sameAs "og:title-1_x" } @Test @@ -39,8 +40,8 @@ class AsciiCaseTest { val kelvin = 'K'.asciiLowercase() // then - assert(dotless == "tıtle") - assert(umlaut == "Ärger") + dotless sameAs "tıtle" + umlaut sameAs "Ärger" assert(kelvin == 'K') } @@ -53,7 +54,7 @@ class AsciiCaseTest { val lowered = kelvinKey.asciiLowercase() // then - assert(kelvinKey.lowercase() == "key") - assert(lowered == kelvinKey) + kelvinKey.lowercase() sameAs "key" + lowered sameAs kelvinKey } } diff --git a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt index 43f2e65..06d0f4e 100644 --- a/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt +++ b/markanywhere-html-spec/src/commonTest/kotlin/HtmlWhitespaceTest.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.html.spec import com.xemantic.kotlin.test.assert +import com.xemantic.kotlin.test.sameAs import kotlin.test.Test class HtmlWhitespaceTest { @@ -76,7 +77,7 @@ class HtmlWhitespaceTest { val result = "\n Foo\t\r\n Bar \u000C".stripAndCollapseHtmlWhitespace() // then - assert(result == "Foo Bar") + result sameAs "Foo Bar" } @Test @@ -85,7 +86,7 @@ class HtmlWhitespaceTest { val result = "\u00A0 a \u00A0b ".stripAndCollapseHtmlWhitespace() // then - assert(result == "\u00A0 a \u00A0b") + result sameAs "\u00A0 a \u00A0b" } @Test @@ -94,7 +95,7 @@ class HtmlWhitespaceTest { val result = " \t\n".stripAndCollapseHtmlWhitespace() // then - assert(result == "") + result sameAs "" } } diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index ce5a38d..e8445fa 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -50,19 +50,31 @@ import kotlinx.serialization.json.jsonPrimitive // elements joined, not the quotes and commas serialising them — its length // as well as whether it reads as text — so it is judged as the same list // written plainly would be: a long list of words is kept, a long list of -// hashes is not, and one whose elements fit under the cap is short. Past -// [MAX_PARSED_LENGTH] it is judged as written, like any other shape. +// hashes is not, and one whose elements fit under the cap is short. +// Past [MAX_PARSED_LENGTH] nothing is decoded or parsed: a value is state +// unless it reads as text, as written, and does not open like encoded state +// (a JSON object, array or string, a percent escape) — no page writes +// metadata that long, and a page-controlled value must not cost a tree many +// times its size. internal fun isApplicationStateMeta(content: String): Boolean { val written = content.trimInvisible() return when { written.length <= MAX_META_VALUE_LENGTH -> written.isStructuralState() - written.length <= MAX_PARSED_LENGTH && written.isFlatArray() -> written.isLongFlatArrayState() - // decoded and parsed only once it reads as text, so a blob — - // megabytes of JSON or of escapes, say — is dropped unparsed - else -> !written.readsAsText() || written.isStructuralState() + written.length <= MAX_PARSED_LENGTH -> when { + written.isFlatArray() -> written.isLongFlatArrayState() + // decoded and parsed only once it reads as text, so a blob is + // dropped unparsed + else -> !written.readsAsText() || written.isStructuralState() + } + // never parsed: opening like encoded state is state enough + else -> !written.readsAsText() || written.opensLikeEncodedState() } } +// Whether this value opens the way a value [isStructuralState] parses does: +// like a JSON object, array or string, or with a percent escape. +private fun String.opensLikeEncodedState(): Boolean = firstOrNull()?.let { it in "{[\"%" } == true + // Real metadata is short: `description` / `og:description` rarely exceed 300 // characters. Past this, a value must read as text to be kept. private const val MAX_META_VALUE_LENGTH = 4096 @@ -148,8 +160,9 @@ private fun JsonElement.isEncodedState(layers: Int): Boolean { // at least one char in sixteen breaks a word — whitespace, a list separator // (`,` `;`, so `a,b,c` keywords count), a path separator (`/`, so a list of // URLs counts — base64 holds one in 64 chars, too few), or a letter of a -// script written without spaces (anything past ASCII — base64 and hex never -// contain one) — sparse enough for a list of long compound words, while the +// script written without spaces ([isUnspacedScriptLetter] — base64 and hex +// never contain one; a Latin, Greek or Cyrillic letter, accented or not, +// breaks no word) — sparse enough for a list of long compound words, while the // punctuation of serialised data (quotes, brackets, `=`, `|`, `\`) stays // rare. // Chars are counted as code points: a surrogate pair — a letter of a @@ -172,13 +185,31 @@ private fun String.readsAsText(): Boolean { continue } if (c.isLetter() || c.category in COMBINING_MARKS) letters++ - if (c.isWhitespace() || c == ',' || c == ';' || c == '/' || c.code > 0x7F && c.isLetter()) wordBreaks++ + if (c.isWhitespace() || c == ',' || c == ';' || c == '/' || c.isUnspacedScriptLetter()) wordBreaks++ else if (c in DATA_PUNCTUATION) dataPunctuation++ i++ } return letters * 2 >= chars && wordBreaks * 16 >= chars && dataPunctuation * 20 < chars } +// Whether this is a letter of a script written without spaces between words: +// Thai, Lao, Tibetan, Myanmar, Khmer, Japanese kana, Bopomofo, the CJK +// ideographs and Yi. (A supplementary-plane letter, which the common stdlib +// cannot classify, is counted as one by [readsAsText] directly.) +private fun Char.isUnspacedScriptLetter(): Boolean = + isLetter() && UNSPACED_SCRIPT_RANGES.any { code in it } + +private val UNSPACED_SCRIPT_RANGES = listOf( + 0x0E00..0x0FFF, // Thai, Lao, Tibetan + 0x1000..0x109F, // Myanmar + 0x1780..0x17FF, // Khmer + 0x3040..0x31FF, // Hiragana, Katakana, Bopomofo, Katakana extensions + 0x3400..0x4DBF, // CJK Extension A + 0x4E00..0xA4CF, // CJK Unified Ideographs, Yi + 0xF900..0xFAFF, // CJK Compatibility Ideographs + 0xFF66..0xFF9F, // halfwidth Katakana +) + private val COMBINING_MARKS = setOf( CharCategory.NON_SPACING_MARK, CharCategory.COMBINING_SPACING_MARK, diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index 8d881be..fb7c4b2 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -74,9 +74,16 @@ import kotlinx.coroutines.flow.Flow * * A frontmatter left empty, by dropping its title entries or as it came, is * dropped too (it would render as two thematic breaks), so no empty - * frontmatter is ever emitted. A frontmatter or `h1` the stream ends inside - * (a broken upstream contract) is judged all the same, an entry left open - * included, and emitted closed, so the output stays balanced. + * frontmatter is ever emitted. Nor is a title entry dropped from the front + * of a frontmatter where that would leave a verbatim line (one outside the + * YAML subset) first — front matter detection needs a key there, so the + * block would parse back as a thematic break and a paragraph: the title + * entry kept moves into the first title entry's place instead, or, none + * being kept, the first one stays. + * + * A frontmatter or `h1` the stream ends inside (a broken upstream contract) + * is judged all the same, an entry left open included, and emitted closed, + * so the output stays balanced. * * Buffering is bounded to the frontmatter subtree plus one `h1` subtree — * everything after the decision point is forwarded as it arrives. @@ -117,25 +124,63 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s blanks.clear() } - // emits the held frontmatter — `prepended` right after its mark, the span - // of each slot in `rewrite` (in source order, as `titleSlots` is) replaced - // by its events (none drops the entry) — then the blanks held with it. - // A frontmatter left empty, by the rewrite or as it came, is dropped: it - // would render as a `---` pair, which parses back as two thematic breaks. + // the held events of this slot, as they came + fun FrontMatterEntry.events(): List<SemanticEvent> = frontmatterEvents!!.subList(start, end + 1) + + // the held frontmatter's body — `prepended` first, the span of each slot + // in `rewrite` (in source order, as `titleSlots` is) replaced by its + // events (none drops the entry) + fun heldBody( + rewrite: Map<FrontMatterEntry, List<SemanticEvent>>, + prepended: List<SemanticEvent> + ): List<SemanticEvent> = buildList { + val held = frontmatterEvents!! + addAll(prepended) + var next = 1 + for ((slot, events) in rewrite) { + addAll(held.subList(next, slot.start)) + addAll(events) + next = slot.end + 1 + } + addAll(held.subList(next, held.size)) + } + + // `rewrite` keeping the first title slot: the one scalar title entry it + // keeps elsewhere moves there, or, keeping none, the slot stays as it + // came + fun keepingFirstSlot( + rewrite: Map<FrontMatterEntry, List<SemanticEvent>> + ): Map<FrontMatterEntry, List<SemanticEvent>> { + val first = titleSlots.first() + val kept = rewrite.entries.firstOrNull { (slot, events) -> + slot != first && !slot.hasChildren && events.isNotEmpty() + } + return rewrite.mapValues { (slot, events) -> + when (slot) { + first -> kept?.value ?: first.events() + kept?.key -> emptyList() + else -> events + } + } + } + + // emits the held frontmatter rewritten as [heldBody] does, then the + // blanks held with it. Dropping leading title entries must not leave a + // verbatim line (one outside the YAML subset) first: front matter + // detection needs a key on the line after the `---`, so the block would + // parse back as a thematic break and a paragraph — the first title slot + // is kept then ([keepingFirstSlot]). A frontmatter left empty, by the + // rewrite or as it came, is dropped: it would render as a `---` pair, + // which parses back as two thematic breaks. suspend fun flushHeld( rewrite: Map<FrontMatterEntry, List<SemanticEvent>> = emptyMap(), prepended: List<SemanticEvent> = emptyList() ) { frontmatterEvents?.let { held -> - val body = buildList { - addAll(prepended) - var next = 1 - for ((slot, events) in rewrite) { - addAll(held.subList(next, slot.start)) - addAll(events) - next = slot.end + 1 - } - addAll(held.subList(next, held.size)) + val body = heldBody(rewrite, prepended).let { + if (it.opensWithVerbatimLine() && !held.drop(1).opensWithVerbatimLine()) { + heldBody(keepingFirstSlot(rewrite), prepended) + } else it } val empty = body.none { it is Mark || (it is Text && !it.text.isHtmlBlank()) @@ -159,9 +204,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // nothing, and another usable one would be a duplicate key suspend fun flushWithReadTitleOnly() { flushHeld( - titleSlots - .filterNot { it.hasChildren || it == read } - .associateWith { emptyList() } + titleSlots.associateWith { + if (it.hasChildren || it == read) it.events() else emptyList() + } ) } @@ -195,8 +240,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s PassThrough } usable != null -> { - val events = frontmatterEvents!!.subList(usable.start, usable.end + 1) - flushWithSingleTitle(usable, respelled(events)) + flushWithSingleTitle(usable, respelled(usable.events())) PassThrough } else -> AwaitingHeading @@ -286,6 +330,11 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // ([mayPrecedeFrontmatter]). private fun SemanticEvent.showsNothing(): Boolean = this is Text && !isMetadataValue(text) +// Whether a frontmatter body opens with a verbatim line: text, past any of +// HTML whitespace, ahead of the first entry. +private fun List<SemanticEvent>.opensWithVerbatimLine(): Boolean = + firstOrNull { !(it is Text && it.text.isHtmlBlank()) } is Text + // These events with an unmark appended for every mark left open, innermost // first. private fun List<SemanticEvent>.closed(): List<SemanticEvent> { diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt index a211407..ab35168 100644 --- a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -26,7 +26,12 @@ import com.xemantic.markanywhere.html.spec.isHtmlBlank // ensureFrontmatterTitle judges the title by — anywhere else a frontmatter // is content. internal fun SemanticEvent.opensFrontmatter(preceding: List<SemanticEvent>): Boolean = - this is Mark && !isTagged && name == "frontmatter" && preceding.all { it.mayPrecedeFrontmatter() } + isFrontmatterMark() && preceding.all { it.mayPrecedeFrontmatter() } + +// Whether this is the mark of a frontmatter that is metadata wherever it +// stands first: an untagged `frontmatter` mark (a tagged one is content). +internal fun SemanticEvent.isFrontmatterMark(): Boolean = + this is Mark && !isTagged && name == "frontmatter" // Whether this event may come ahead of the frontmatter without keeping it // from opening the stream: text of HTML whitespace (an NBSP is content). diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index ad53138..d52140a 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -38,9 +38,16 @@ internal fun isScalarEntryType(type: String?): Boolean = internal fun isMetadataValue(value: String): Boolean = value.any { !it.isInvisible() } // Whitespace (NBSP included), a control char (a next line char, a C0 -// control), or an invisible format char such as a zero-width space or a byte -// order mark. -private fun Char.isInvisible(): Boolean = isWhitespace() || category == CONTROL || category == FORMAT +// control), an invisible format char such as a zero-width space or a byte +// order mark, or a letter or symbol that renders blank ([BLANK_GLYPHS]). +private fun Char.isInvisible(): Boolean = + isWhitespace() || category == CONTROL || category == FORMAT || this in BLANK_GLYPHS + +// Letters and symbols with no visible glyph — the Hangul fillers (choseong, +// jungseong, compatibility, halfwidth) and the blank Braille pattern: being no +// whitespace or format char, they are the common trick for a name that shows +// nothing. +private const val BLANK_GLYPHS = "\u115F\u1160\u3164\uFFA0\u2800" // This value with the chars [isMetadataValue] finds invisible trimmed from // its edges. diff --git a/markanywhere-html/src/commonMain/kotlin/HtmlWhitespaceNormalization.kt b/markanywhere-html/src/commonMain/kotlin/HtmlWhitespaceNormalization.kt index d7c616f..fbbac15 100644 --- a/markanywhere-html/src/commonMain/kotlin/HtmlWhitespaceNormalization.kt +++ b/markanywhere-html/src/commonMain/kotlin/HtmlWhitespaceNormalization.kt @@ -21,6 +21,7 @@ import com.xemantic.markanywhere.dump.AccessibilityAnnotations import com.xemantic.markanywhere.flow.mergeAdjacentText import com.xemantic.markanywhere.html.spec.isHtmlBlank import com.xemantic.markanywhere.html.spec.isHtmlWhitespace +import com.xemantic.markanywhere.html.spec.stripAndCollapseHtmlWhitespace import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow @@ -109,7 +110,7 @@ public fun Flow<SemanticEvent>.dropHtmlStructuralWhitespace(): Flow<SemanticEven // content, dropped at a block boundary). if (event.text.first().isHtmlWhitespace()) pending = true resolvePending(rightQualifies = true) - emit(SemanticEvent.Text(event.text.collapseWhitespace())) + emit(SemanticEvent.Text(event.text.stripAndCollapseHtmlWhitespace())) leftQualifies = true pending = event.text.last().isHtmlWhitespace() } @@ -149,24 +150,6 @@ public fun Flow<SemanticEvent>.dropHtmlStructuralWhitespace(): Flow<SemanticEven // End-of-stream (a block boundary): any trailing whitespace run is dropped. }.mergeAdjacentText() -// Collapses every run of ASCII whitespace to a single space and trims the ends — -// the caller re-attaches a separating space via the block/inline gate when one -// is warranted. Non-ASCII spaces (NBSP ` `, narrow NBSP, en/em space, …) -// are content, not structural whitespace: HTML never collapses them, so they -// pass through verbatim (e.g. legal citations like `§ 823`). -private fun String.collapseWhitespace(): String = buildString { - var inWhitespace = false - for (c in this@collapseWhitespace) { - if (c.isHtmlWhitespace()) { - inWhitespace = true - } else { - if (isNotEmpty() && inWhitespace) append(' ') - inWhitespace = false - append(c) - } - } -} - // Opens a whitespace-preserving region, whose watermark covers the whole // subtree — so a `code` nested in a `pre` needs no rule of its own. // diff --git a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt index d33d96f..191162b 100644 --- a/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt +++ b/markanywhere-html/src/commonMain/kotlin/WrapInHtmlDocument.kt @@ -112,8 +112,10 @@ public fun Flow<SemanticEvent>.wrapInHtmlDocument(): Flow<SemanticEvent> = seman val reader = frontmatter when { reader != null -> if (reader.read(event)) openDocument() + // `blanks` only ever holds text that may precede a frontmatter, + // so the mark alone decides whether it opens the stream !opened -> when { - event.opensFrontmatter(blanks) -> { + event.isFrontmatterMark() -> { frontmatter = FrontMatterEntryReader(metadata::addFromFrontMatter).also { it.read(event) } } event.mayPrecedeFrontmatter() -> blanks += event diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index c14b456..12ffe6f 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -1204,6 +1204,28 @@ class EnsureFrontmatterTitleTest { } } + @Test + fun `should treat a title entry of blank-rendering letters as blank`() = runTest { + // given — a Hangul filler is a letter, yet shows nothing + val input = semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\u3164" } + } + "h1" { +"Heading" } + } + + // when + val output = input.ensureFrontmatterTitle() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Heading" } + } + "h1" { +"Heading" } + } + } + @Test fun `should keep a non-breaking space before the frontmatter as content`() = runTest { // given — NBSP is content in HTML, so the frontmatter does not open @@ -1378,6 +1400,44 @@ class EnsureFrontmatterTitleTest { output sameAs "Body." } + @Test + fun `should keep a leading empty title entry that a verbatim line follows`() = runTest { + // given — dropping the entry would make the verbatim line the first + // one, which front matter detection rejects: the block would parse + // back as a thematic break and a paragraph + val markdown = "---\ntitle:\nfoo bar baz\nx: y\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs markdown + } + + @Test + fun `should move a usable title variant ahead of a verbatim line following a dropped empty title`() = runTest { + // given + val markdown = "---\ntitle:\nfoo bar baz\nTitle: Real\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs "---\ntitle: Real\nfoo bar baz\n---\n\nBody." + } + + @Test + fun `should keep a frontmatter led by empty title entries and a verbatim line a front matter on re-parse`() = runTest { + // given + val markdown = "---\ntitle: \"\"\nTitle:\n? complex\nx: y\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs "---\ntitle: \"\"\n? complex\nx: y\n---\n\nBody." + } + @Test fun `should treat a title entry of control chars as blank`() = runTest { // given — a next line char or a C0 control shows nothing, and the diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index 52c7909..de48974 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1081,6 +1081,39 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop a long unspaced run of letters of a script written with spaces`() = runTest { + // given — Latin, Greek and Cyrillic are written with spaces, so a + // letter of theirs breaks no word, accented or not; Thai is not, so a + // long Thai text without spaces is kept + val latin = "àéîõüçñ".repeat(700) + val greek = "αβγδεζη".repeat(700) + val cyrillic = "абвгдеж".repeat(700) + val thai = "ภาษาไทยเขียนติดกันโดยไม่เว้นวรรค".repeat(150) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "latin", "content" to latin) { } + "meta"("name" to "greek", "content" to greek) { } + "meta"("name" to "cyrillic", "content" to cyrillic) { } + "meta"("name" to "description", "content" to thai) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +thai } + } + "p" { +"text" } + } + } + @Test fun `should keep long text in scripts writing vowels as combining marks`() = runTest { // given — Devanagari matras and viramas, and Arabic harakat, are marks @@ -1366,6 +1399,36 @@ class SimplifyHtmlTest { } } + @Test + fun `should drop a value too long to parse opening like JSON without parsing it`() = runTest { + // given — far past anything metadata holds, a value opening like a + // JSON object, array or string, or a percent escape is judged by + // that alone, so no page-controlled value is parsed whatever its size; + // plain prose that long is still kept + val prose = "A study of how language models behave in long dialogues. ".repeat(1200) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "quoted", "content" to "\"$prose\"") { } + "meta"("name" to "state", "content" to "{\"text\": \"$prose\"}") { } + "meta"("name" to "description", "content" to prose) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +prose } + } + "p" { +"text" } + } + } + @Test fun `should not decode a percent escape made of non-ASCII digits`() = runTest { // given — an Arabic-Indic seven is no hex digit, so this is not a @@ -1945,6 +2008,33 @@ class SimplifyHtmlTest { } } + @Test + fun `should skip a title of blank-rendering letters only`() = runTest { + // given — a Hangul filler or a blank Braille pattern is a letter or a + // symbol, yet shows nothing: the common trick for a blank name + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +"\u3164" } + "meta"("name" to "title", "content" to "Real Page") { } + "meta"("name" to "description", "content" to "\u115F\u1160\uFFA0\u2800") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"Real Page" } + } + "p" { +"x" } + } + } + @Test fun `should strip and collapse a title meta like a title element`() = runTest { // given From 826f6a0fe8457bd075778b3aac521b4217b2b34f Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 15:14:11 +0200 Subject: [PATCH 24/25] Keep verbatim titles and continuation lines; strict JSON state (#82) - FrontMatterEntryReader spans each top-level entry over the indented verbatim lines continuing it, and reads a verbatim line defining a key as an entry of unknown value (yamlKeyLineKeyOrNull, new in markanywhere-yaml), never head metadata - ensureFrontmatterTitle derives no title beside a verbatim title line (a multi-line quoted scalar), which made a duplicate key, and drops a title entry together with its continuation lines, which otherwise continued the entry before it; the verbatim-first guard is decided without building the body twice - normalizeTitle keeps bidi controls at the title's edges - isApplicationStateMeta judges only strict JSON as state, so bracketed prose such as [[Wiki]] is kept, and classifies supplementary-plane chars by plane, so a blob of emoji no longer reads as text - HeadMetadata.acceptsFromHtml is the one first-wins and blank rule, shared with simplifyHtml's meta pre-check Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CLAUDE.md | 2 + markanywhere-html/build.gradle.kts | 1 + .../commonMain/kotlin/ApplicationStateMeta.kt | 66 +++++++++++--- .../kotlin/EnsureFrontmatterTitle.kt | 66 ++++++++++---- .../kotlin/FrontMatterEntryReader.kt | 90 +++++++++++++++---- .../src/commonMain/kotlin/HeadMetadata.kt | 26 ++++-- .../src/commonMain/kotlin/SimplifyHtml.kt | 12 +-- .../kotlin/EnsureFrontmatterTitleTest.kt | 38 ++++++++ .../src/commonTest/kotlin/SimplifyHtmlTest.kt | 88 ++++++++++++++++++ markanywhere-yaml/api/markanywhere-yaml.api | 1 + .../src/commonMain/kotlin/YamlKeyLine.kt | 31 +++++-- .../src/commonTest/kotlin/YamlKeyLineTest.kt | 16 ++++ 12 files changed, 365 insertions(+), 72 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 741f3f9..80634c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -417,6 +417,8 @@ If the fix is bounded (one inline construct, one or two lines of lookahead), it' Repeat a character class instead (`[0-9_]*`) and check what the group expressed by hand (`YamlScalarType.kt`). - Facts of the HTML standard needed by more than one of `parse` / `render` / `html` — HTML whitespace (`isHtmlWhitespace` / `isHtmlBlank`), void and raw text elements, `Mark.classList` — live in the leaf module `markanywhere-html-spec` (depends only on `api`), never in `markanywhere-api` (which describes only the event model) and never as private copies. Only spec facts go there, not project policy (e.g. which tags render as blocks in Markdown); the parser's `isFlankWhitespace` is deliberately a different set (CommonMark "Unicode whitespace", which includes NBSP). +- kotlinx `Json.parseToJsonElement` accepts a **bare word as a value** even with `isLenient = false` (`[[Wiki]]`, `{"a": tru}` parse; only keys must be quoted), so "it parses as JSON" is not evidence that a value is serialised data. + `ApplicationStateMeta.kt` checks the parsed tree with `isStrictJson` before judging a `<meta>` value as state — otherwise bracketed prose like `[foo, [bar]]` was dropped as a nested array. - `MutableMap.putIfAbsent` is **JVM-only** — not in the common stdlib, so it can't be used in `commonMain`. For first-wins ("put only if key absent") semantics across all KMP targets, prefer **`getOrPutIfMissing(key) { value }`** (Kotlin 2.4, `@ExperimentalStdlibApi`): it computes & stores the default *only when the key is genuinely absent*. Do NOT reach for plain `getOrPut` as the `putIfAbsent` replacement — it (and its explicit alias `getOrPutIfNull`) also recompute when the stored value is **null**, conflating "missing" with "null" (the footgun [KEEP-0457](https://github.com/Kotlin/KEEP/blob/main/proposals/stdlib/KEEP-0457-alternative-behavior-for-map-getOrElse.md) fixes). diff --git a/markanywhere-html/build.gradle.kts b/markanywhere-html/build.gradle.kts index 51664b5..ac41366 100644 --- a/markanywhere-html/build.gradle.kts +++ b/markanywhere-html/build.gradle.kts @@ -41,6 +41,7 @@ kotlin { api(project(":markanywhere-transform")) implementation(project(":markanywhere-dump")) implementation(project(":markanywhere-html-spec")) + implementation(project(":markanywhere-yaml")) api(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.serialization.json) implementation(libs.xemantic.kotlin.core) diff --git a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt index e8445fa..53313e6 100644 --- a/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt +++ b/markanywhere-html/src/commonMain/kotlin/ApplicationStateMeta.kt @@ -19,6 +19,7 @@ package com.xemantic.markanywhere.html import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonPrimitive @@ -34,7 +35,9 @@ import kotlinx.serialization.json.jsonPrimitive // order mark, NBSP) at its edges or those of a JSON string within: // - a value that parses as a JSON object. Parsing, not a look at the first // and last char, is what keeps human text that merely starts with a -// bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`); +// bracket (`[Solved] …`, `{Draft} …`, `[2024] Annual report [PDF]`) — +// parsing as strict JSON, since the parser takes a bare word for a value +// and would read `[[Wiki]]` as a nested array; // - a JSON array holding an object or a nested array. A flat list of // scalars — an empty one included — is metadata a person writes // (`keywords`, `article:tag`, `citation_volume`), whether a bare word in it @@ -130,12 +133,39 @@ private fun String.hasNestedElement(): Boolean { // a pass over itself per layer. private const val MAX_DECODING_DEPTH = 8 -// Only a value opening like a JSON object, array or string is parsed. +// Only a value opening like a JSON object, array or string is parsed, and +// only strict JSON counts ([isStrictJson]). private fun String.parseJsonCandidateOrNull(): JsonElement? { val first = firstOrNull() - return if (first == '{' || first == '[' || first == '"') parseJsonOrNull() else null + return if (first == '{' || first == '[' || first == '"') { + parseJsonOrNull()?.takeIf { it.isStrictJson() } + } else null } +// Whether every scalar in this tree is a JSON value — a string, a number, +// `true`, `false` or `null` — and not a bare word, which the parser accepts +// even when not lenient (keys it does require quoted). Walked with a stack +// of its own, as a crafted value nests deeper than the call stack reaches. +private fun JsonElement.isStrictJson(): Boolean { + val pending = ArrayDeque<JsonElement>() + pending.addLast(this) + while (pending.isNotEmpty()) { + when (val element = pending.removeLast()) { + is JsonObject -> pending.addAll(element.values) + is JsonArray -> pending.addAll(element) + is JsonPrimitive -> if (!element.isString + && element.content != "true" + && element.content != "false" + && element !is JsonNull + && !JSON_NUMBER.matches(element.content) + ) return false + } + } + return true +} + +private val JSON_NUMBER = Regex("^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?$") + // Recursion unwraps one JSON string per level, `layers` bounding the // layers of encoding still to be undone ([MAX_DECODING_DEPTH]). private fun JsonElement.isState(layers: Int): Boolean = when (this) { @@ -165,10 +195,8 @@ private fun JsonElement.isEncodedState(layers: Int): Boolean { // breaks no word) — sparse enough for a list of long compound words, while the // punctuation of serialised data (quotes, brackets, `=`, `|`, `\`) stays // rare. -// Chars are counted as code points: a surrogate pair — a letter of a -// supplementary-plane script (CJK Extension B, historic scripts) or an emoji, -// which the common stdlib cannot classify — counts once, as a letter of a -// script without spaces; serialised data never holds one. +// Chars are counted as code points: a surrogate pair, which the common stdlib +// cannot classify, counts once, judged by its plane. private fun String.readsAsText(): Boolean { var chars = 0 var letters = 0 @@ -178,9 +206,19 @@ private fun String.readsAsText(): Boolean { while (i < length) { val c = this[i] chars++ - if (c.isHighSurrogate() && getOrNull(i + 1)?.isLowSurrogate() == true) { - letters++ - wordBreaks++ + val low = getOrNull(i + 1) + if (c.isHighSurrogate() && low != null && low.isLowSurrogate()) { + when (supplementaryCodePoint(c, low)) { + // the CJK ideograph extensions: letters of a script written + // without spaces + in 0x20000..0x3FFFF -> { letters++; wordBreaks++ } + // historic scripts and styled mathematical letters, written + // with spaces + in 0x10000..0x1DFFF -> letters++ + // emoji, other symbols, private use: no letter, so a blob + // encoded as emoji does not read as text + else -> {} + } i += 2 continue } @@ -192,10 +230,14 @@ private fun String.readsAsText(): Boolean { return letters * 2 >= chars && wordBreaks * 16 >= chars && dataPunctuation * 20 < chars } +// The code point a surrogate pair encodes. +private fun supplementaryCodePoint(high: Char, low: Char): Int = + 0x10000 + ((high.code - 0xD800) shl 10) + (low.code - 0xDC00) + // Whether this is a letter of a script written without spaces between words: // Thai, Lao, Tibetan, Myanmar, Khmer, Japanese kana, Bopomofo, the CJK -// ideographs and Yi. (A supplementary-plane letter, which the common stdlib -// cannot classify, is counted as one by [readsAsText] directly.) +// ideographs and Yi. (A supplementary-plane char, which the common stdlib +// cannot classify, is judged by its plane in [readsAsText] directly.) private fun Char.isUnspacedScriptLetter(): Boolean = isLetter() && UNSPACED_SCRIPT_RANGES.any { code in it } diff --git a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt index fb7c4b2..04a8b0c 100644 --- a/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt +++ b/markanywhere-html/src/commonMain/kotlin/EnsureFrontmatterTitle.kt @@ -34,6 +34,11 @@ import kotlinx.coroutines.flow.Flow * `entry` marks with `key="title"` in any ASCII letter case, and one holding * scalar text with visible content is usable — entries holding a nested * structure, an empty collection, `null` or blank text are not, as there. + * A verbatim line defining a title key (a YAML shape outside the parsed + * subset, such as a multi-line quoted scalar) is a title entry too, one whose + * value only the front matter readers know. Each title entry spans the + * indented verbatim lines continuing it, so dropping one drops them too, + * rather than leaving them to continue the entry before it. * * Whenever a title comes out, the frontmatter holds exactly one title entry, * spelled `title`: every front matter reader then reads the same title — @@ -58,9 +63,9 @@ import kotlinx.coroutines.flow.Flow * * Two shapes are left without a title: * - * - A title entry holding a nested structure (localized titles, say) is no - * title for [wrapInHtmlDocument], but it is content: no title is derived - * and no title entry respelled. Of the other title entries only the usable + * - A title entry holding a nested structure (localized titles, say) or kept + * as a verbatim line is no title for [wrapInHtmlDocument], but it is + * content: no title is derived and no title entry respelled. Of the other title entries only the usable * one [wrapInHtmlDocument] picks is kept, as spelled, so readers may * disagree on the title as they did on the input; the empty ones (`null`, * blank, an empty collection) carry nothing, and another usable one would @@ -145,6 +150,32 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s addAll(held.subList(next, held.size)) } + // Whether the held frontmatter's body, rewritten as [heldBody] would, + // opens with a verbatim line: text, past any of HTML whitespace, ahead of + // the first entry — found without building the body. + fun opensWithVerbatimLine( + rewrite: Map<FrontMatterEntry, List<SemanticEvent>>, + prepended: List<SemanticEvent> + ): Boolean { + if (prepended.isNotEmpty()) return false + val held = frontmatterEvents!! + val slots = rewrite.keys.associateBy { it.start } + var i = 1 + while (i < held.size) { + val slot = slots[i] + if (slot != null) { + val events = rewrite.getValue(slot) + if (events.isNotEmpty()) return events.first() is Text + i = slot.end + 1 + continue + } + val event = held[i] + if (!(event is Text && event.text.isHtmlBlank())) return event is Text + i++ + } + return false + } + // `rewrite` keeping the first title slot: the one scalar title entry it // keeps elsewhere moves there, or, keeping none, the slot stays as it // came @@ -153,7 +184,7 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s ): Map<FrontMatterEntry, List<SemanticEvent>> { val first = titleSlots.first() val kept = rewrite.entries.firstOrNull { (slot, events) -> - slot != first && !slot.hasChildren && events.isNotEmpty() + slot != first && !slot.isUnreadable && events.isNotEmpty() } return rewrite.mapValues { (slot, events) -> when (slot) { @@ -177,11 +208,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s prepended: List<SemanticEvent> = emptyList() ) { frontmatterEvents?.let { held -> - val body = heldBody(rewrite, prepended).let { - if (it.opensWithVerbatimLine() && !held.drop(1).opensWithVerbatimLine()) { - heldBody(keepingFirstSlot(rewrite), prepended) - } else it - } + val leavesVerbatimLineFirst = opensWithVerbatimLine(rewrite, prepended) + && !opensWithVerbatimLine(emptyMap(), emptyList()) + val body = heldBody(if (leavesVerbatimLineFirst) keepingFirstSlot(rewrite) else rewrite, prepended) val empty = body.none { it is Mark || (it is Text && !it.text.isHtmlBlank()) } @@ -200,12 +229,12 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s } // emits the held frontmatter keeping, of its title entries, only the - // nested ones and the one wrapInHtmlDocument reads: the empty ones carry - // nothing, and another usable one would be a duplicate key + // unreadable ones and the one wrapInHtmlDocument reads: the empty ones + // carry nothing, and another usable one would be a duplicate key suspend fun flushWithReadTitleOnly() { flushHeld( titleSlots.associateWith { - if (it.hasChildren || it == read) it.events() else emptyList() + if (it.isUnreadable || it == read) it.events() else emptyList() } ) } @@ -233,9 +262,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s suspend fun judgeFrontmatter() { val usable = read state = when { - // a nested title is content, not ours to replace or drop; a - // sequence has no place for a title entry - titleSlots.any { it.hasChildren } || reader.isSequence -> { + // a nested or verbatim title is content, not ours to replace or + // drop; a sequence has no place for a title entry + titleSlots.any { it.isUnreadable } || reader.isSequence -> { flushWithReadTitleOnly() PassThrough } @@ -330,10 +359,9 @@ public fun Flow<SemanticEvent>.ensureFrontmatterTitle(): Flow<SemanticEvent> = s // ([mayPrecedeFrontmatter]). private fun SemanticEvent.showsNothing(): Boolean = this is Text && !isMetadataValue(text) -// Whether a frontmatter body opens with a verbatim line: text, past any of -// HTML whitespace, ahead of the first entry. -private fun List<SemanticEvent>.opensWithVerbatimLine(): Boolean = - firstOrNull { !(it is Text && it.text.isHtmlBlank()) } is Text +// Whether this title entry holds a title [wrapInHtmlDocument] cannot read, +// though front matter readers do: a nested structure or a verbatim line. +private val FrontMatterEntry.isUnreadable: Boolean get() = hasChildren || isVerbatim // These events with an unmark appended for every mark left open, innermost // first. diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt index ab35168..0701998 100644 --- a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -18,6 +18,7 @@ package com.xemantic.markanywhere.html import com.xemantic.markanywhere.SemanticEvent import com.xemantic.markanywhere.html.spec.isHtmlBlank +import com.xemantic.markanywhere.yaml.yamlKeyLineKeyOrNull // Whether this event opens the frontmatter that is the page's metadata, the // `preceding` events being all the stream held before it: an untagged @@ -38,26 +39,36 @@ internal fun SemanticEvent.isFrontmatterMark(): Boolean = internal fun SemanticEvent.mayPrecedeFrontmatter(): Boolean = this is Text && text.isHtmlBlank() // A top-level front matter `entry`: its key as spelled, `type`, text, whether -// it holds nested marks, and the span of its events among those read. +// it holds nested marks, and the span of its events among those read — the +// indented verbatim lines continuing it included, so dropping the span drops +// them too, rather than leaving them to continue the entry before it. A +// verbatim line defining a key (one outside the YAML subset, such as a +// multi-line quoted scalar) is an entry too, [isVerbatim]: its value is +// unknown, so it is never head metadata, yet it holds the key for readers. internal class FrontMatterEntry( val key: String, val type: String?, val text: String, val hasChildren: Boolean, + val isVerbatim: Boolean, val start: Int, val end: Int ) { // Whether it holds head metadata: scalar text with visible content — - // not a nested structure, an empty collection, `null` or blank text. - val isHeadMetadata: Boolean = !hasChildren && isScalarEntryType(type) && isMetadataValue(text) + // not a nested structure, an empty collection, `null`, blank text or a + // verbatim line. + val isHeadMetadata: Boolean = + !hasChildren && !isVerbatim && isScalarEntryType(type) && isMetadataValue(text) } // Reads the top-level entries of a `frontmatter` subtree, reporting each to -// `onEntry` as it closes. The one reader of front matter as head metadata — -// wrapInHtmlDocument turns the entries into `<head>`, ensureFrontmatterTitle -// judges the title entries among them, and the two must agree on both. +// `onEntry` once its span is known: when the next top-level entry or verbatim +// line opens, or the frontmatter closes. The one reader of front matter as +// head metadata — wrapInHtmlDocument turns the entries into `<head>`, +// ensureFrontmatterTitle judges the title entries among them, and the two +// must agree on both. internal class FrontMatterEntryReader( private val onEntry: (FrontMatterEntry) -> Unit ) { @@ -70,6 +81,9 @@ internal class FrontMatterEntryReader( private var openHasChildren = false private val openText = StringBuilder() + // the last top-level entry read, held until its span is known + private var pending: FrontMatterEntry? = null + // Whether the root is a sequence: a top-level `item` was read. var isSequence: Boolean = false private set @@ -82,6 +96,7 @@ internal class FrontMatterEntryReader( is Mark -> { depth++ if (depth == 2) { + reportPending() if (event.name == "item") isSequence = true open = if (event.name == "entry" && event["key"] != null) event else null openStart = index @@ -91,10 +106,16 @@ internal class FrontMatterEntryReader( openHasChildren = true } } - is Text -> if (depth == 2) openText.append(event.text) + is Text -> when (depth) { + 1 -> readVerbatimLine(event.text) + 2 -> openText.append(event.text) + } is Unmark -> { if (--depth == 1) closeEntry() - return depth == 0 + if (depth == 0) { + reportPending() + return true + } } } return false @@ -104,22 +125,57 @@ internal class FrontMatterEntryReader( // upstream contract) — its text is complete by then. fun finish() { closeEntry() + reportPending() + } + + // A verbatim line (text outside any entry, as the YAML parser emits a + // line outside its subset): an indented one continues the entry before + // it, one at the margin ends it, and defines an entry itself when it + // opens with a key. A blank line decides nothing. + private fun readVerbatimLine(text: String) { + if (text.isHtmlBlank()) return + val entry = pending + if (text.first() == ' ' || text.first() == '\t') { + if (entry != null) pending = entry.extendedTo(index) + return + } + reportPending() + pending = yamlKeyLineKeyOrNull(text.substringBefore('\n'))?.let { key -> + FrontMatterEntry( + key = key, + type = null, + text = "", + hasChildren = false, + isVerbatim = true, + start = index, + end = index + ) + } } private fun closeEntry() { open?.let { open = null - onEntry( - FrontMatterEntry( - key = it["key"]!!, - type = it["type"], - text = openText.toString(), - hasChildren = openHasChildren, - start = openStart, - end = index - ) + pending = FrontMatterEntry( + key = it["key"]!!, + type = it["type"], + text = openText.toString(), + hasChildren = openHasChildren, + isVerbatim = false, + start = openStart, + end = index ) } } + private fun reportPending() { + pending?.let { + pending = null + onEntry(it) + } + } + } + +private fun FrontMatterEntry.extendedTo(end: Int) = + FrontMatterEntry(key, type, text, hasChildren, isVerbatim, start, end) diff --git a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt index d52140a..5000c05 100644 --- a/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt +++ b/markanywhere-html/src/commonMain/kotlin/HeadMetadata.kt @@ -57,6 +57,8 @@ internal fun String.trimInvisible(): String = trim { it.isInvisible() } // and collapsed — with any other invisible char at its edges (the NBSP // padding an icon often leaves, a byte order mark) trimmed too: inside, NBSP // is content and stays, at an edge it only forces the title into quotes. +// A bidi control ([BIDI_CONTROLS]) is kept at an edge as well: invisible, yet +// it is what keeps the punctuation of a right-to-left title on its side. // Whitespace that breaks a line (a vertical tab, a line or paragraph // separator, a next line char) collapses like HTML whitespace, as a title is // one line. @@ -64,10 +66,16 @@ internal fun String.normalizeTitle(): String = CharArray(length) { if (this[it] in LINE_BREAKING_WHITESPACE) ' ' else this[it] } .concatToString() .stripAndCollapseHtmlWhitespace() - .trimInvisible() + .trim { it.isInvisible() && it !in BIDI_CONTROLS } private const val LINE_BREAKING_WHITESPACE = "\u000B\u0085\u2028\u2029" +// The bidi marks, embeddings, overrides and isolates: the Arabic letter mark, +// the left-to-right and right-to-left marks, U+202A..U+202E and +// U+2066..U+2069. +private const val BIDI_CONTROLS = + "\u061C\u200E\u200F\u202A\u202B\u202C\u202D\u202E\u2066\u2067\u2068\u2069" + // A language tag as `<html lang>` carries it: HTML strips its whitespace, and // any other invisible char at its edges (a byte order mark, a zero-width // space) is trimmed too — no valid BCP 47 tag holds one. @@ -114,13 +122,17 @@ internal class HeadMetadata { operator fun get(name: String): MetadataEntry? = entries[name.asciiLowercase()] - // Adds a `<meta>` read from HTML, where the first of duplicate elements - // is the one a query finds: the first occurrence of a name, in any letter - // case, wins — spelling and value. + // Whether [addFromHtml] would add this `<meta>`: it carries something + // for a reader and, the first of duplicate elements being the one a + // query finds, no earlier one of its name, in any letter case, was added. + // Lets a caller skip judging a value that would not be added anyway. + fun acceptsFromHtml(key: String, value: String): Boolean = + key !in this && isMetadataValue(value) + + // Adds a `<meta>` read from HTML when it [acceptsFromHtml] — the first + // occurrence of a name wins, spelling and value. fun addFromHtml(key: String, value: String) { - if (!isMetadataValue(value)) return - @OptIn(ExperimentalStdlibApi::class) - entries.getOrPutIfMissing(key.asciiLowercase()) { MetadataEntry(key, value) } + if (acceptsFromHtml(key, value)) entries[key.asciiLowercase()] = MetadataEntry(key, value) } // Adds a front matter entry the way its readers resolve a duplicate key diff --git a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt index f9bfd12..a0c2184 100644 --- a/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt +++ b/markanywhere-html/src/commonMain/kotlin/SimplifyHtml.kt @@ -303,15 +303,11 @@ public fun Flow<SemanticEvent>.simplifyHtml( match("meta") { event -> val name = event["name"] val content = event["content"] - // blank by the rule addFromHtml applies; cheapest checks first, so - // the JSON parse runs only for a name that would otherwise be kept - // (an already present one loses, the first of duplicates winning) - if (name != null && content != null && isMetadataValue(content)) { + // cheapest checks first, so the JSON parse runs only for a meta + // that would otherwise be added + if (name != null && content != null && metadata.acceptsFromHtml(name, content)) { val normalizedName = name.asciiLowercase() - if (normalizedName !in metadata - && !isNoiseMetaName(normalizedName) - && !isApplicationStateMeta(content) - ) { + if (!isNoiseMetaName(normalizedName) && !isApplicationStateMeta(content)) { val value = when (normalizedName) { // as a <title> element's text reads "title" -> content.normalizeTitle() diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index 12ffe6f..b9bc9d3 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -1438,6 +1438,44 @@ class EnsureFrontmatterTitleTest { output sameAs "---\ntitle: \"\"\n? complex\nx: y\n---\n\nBody." } + @Test + fun `should derive no title beside a title line outside the YAML subset`() = runTest { + // given — the multi-line quoted title comes through as verbatim lines, + // yet readers read it: deriving one would make a duplicate key + val markdown = "---\ntitle: \"A long\n title\"\nx: y\n---\n\n# Heading" + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs markdown + } + + @Test + fun `should drop the continuation lines of a dropped title entry`() = runTest { + // given — left behind, the indented line would continue `foo` + val markdown = "---\nfoo: a\nTitle: x\n cont\n more\ntitle: Real\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs "---\nfoo: a\ntitle: Real\n---\n\nBody." + } + + @Test + fun `should keep the continuation lines of a verbatim line after a title entry`() = runTest { + // given — the indented line continues the verbatim line, not the + // title entry dropped before it + val markdown = "---\nfoo: a\nTitle:\nbar: [a:, b]\n x\n---\n\nBody." + + // when + val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() + + // then + output sameAs "---\nfoo: a\nbar: [a:, b]\n x\n---\n\nBody." + } + @Test fun `should treat a title entry of control chars as blank`() = runTest { // given — a next line char or a C0 control shows nothing, and the diff --git a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt index de48974..da02b29 100644 --- a/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/SimplifyHtmlTest.kt @@ -1429,6 +1429,66 @@ class SimplifyHtmlTest { } } + @Test + fun `should keep bracketed human text holding a nested bracket`() = runTest { + // given — a bare word is no JSON value, so these are no nested JSON + // arrays however leniently a parser reads them + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "keywords", "content" to "[foo, [bar]]") { } + "meta"("name" to "description", "content" to "[[Wiki]]") { } + "meta"("name" to "state", "content" to "[[\"a\", 1, true, null, -2.5e3]]") { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "keywords") { +"[foo, [bar]]" } + "entry"("key" to "description") { +"[[Wiki]]" } + } + "p" { +"text" } + } + } + + @Test + fun `should drop a long value made of emoji`() = runTest { + // given — an emoji is no letter, so a blob encoded as emoji does not + // read as text, while styled letters of the supplementary planes and + // CJK Extension B ideographs do + val emoji = "\uD83D\uDE00\uD83D\uDC4D\uD83C\uDF89 ".repeat(1500) + val styled = "\uD835\uDC07\uD835\uDC1E\uD835\uDC25\uD835\uDC25\uD835\uDC28 ".repeat(1000) + val cjk = "\uD840\uDC00\uD840\uDC01\uD840\uDC02".repeat(1500) + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "meta"("name" to "state", "content" to emoji) { } + "meta"("name" to "description", "content" to styled) { } + "meta"("name" to "abstract", "content" to cjk) { } + } + "body" { "p" { +"text" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "description") { +styled } + "entry"("key" to "abstract") { +cjk } + } + "p" { +"text" } + } + } + @Test fun `should not decode a percent escape made of non-ASCII digits`() = runTest { // given — an Arabic-Indic seven is no hex digit, so this is not a @@ -1823,6 +1883,34 @@ class SimplifyHtmlTest { } } + @Test + fun `should keep the bidi marks at the edges of a title`() = runTest { + // given — a right-to-left title ends in a right-to-left mark on + // purpose, keeping its punctuation on the right side; an isolate + // likewise opens the title of an embedded direction + val input = semanticEvents(tagged = true) { + "html" { + "head" { + "title" { +" \uFEFF\u05E9\u05DC\u05D5\u05DD!\u200F " } + "meta"("name" to "og:title", "content" to "\u2067\u05E9\u05DC\u05D5\u05DD\u2069") { } + } + "body" { "p" { +"x" } } + } + } + + // when + val output = input.simplifyHtml() + + // then + output sameAs semanticEvents { + "frontmatter" { + "entry"("key" to "title") { +"\u05E9\u05DC\u05D5\u05DD!\u200F" } + "entry"("key" to "og:title") { +"\u2067\u05E9\u05DC\u05D5\u05DD\u2069" } + } + "p" { +"x" } + } + } + @Test fun `should trim invisible format chars at the edges of a title`() = runTest { // given — a byte order mark and a zero-width space show nothing, at diff --git a/markanywhere-yaml/api/markanywhere-yaml.api b/markanywhere-yaml/api/markanywhere-yaml.api index 9d5e9ae..5b6ab85 100644 --- a/markanywhere-yaml/api/markanywhere-yaml.api +++ b/markanywhere-yaml/api/markanywhere-yaml.api @@ -1,5 +1,6 @@ public final class com/xemantic/markanywhere/yaml/YamlKeyLineKt { public static final fun isYamlKeyLine (Ljava/lang/String;)Z + public static final fun yamlKeyLineKeyOrNull (Ljava/lang/String;)Ljava/lang/String; } public final class com/xemantic/markanywhere/yaml/YamlKt { diff --git a/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt b/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt index 89bdedc..1623f01 100644 --- a/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt +++ b/markanywhere-yaml/src/commonMain/kotlin/YamlKeyLine.kt @@ -32,19 +32,32 @@ package com.xemantic.markanywhere.yaml * A manual scan rather than a regex: `\p{L}` classes are not portable * across the Kotlin/JS and Kotlin/Native regex engines. */ -public fun isYamlKeyLine(line: String): Boolean { - if (line.isEmpty()) return false +public fun isYamlKeyLine(line: String): Boolean = yamlKeyLineKeyOrNull(line) != null + +/** + * The key of [line] when it is a YAML block mapping key line by the rule of + * [isYamlKeyLine] — a quoted key decoded — or null when it is not one. + * + * Lets a consumer of a front matter line kept verbatim (one outside the + * subset `YamlParser` parses) still tell which key it defines. + */ +public fun yamlKeyLineKeyOrNull(line: String): String? { + if (line.isEmpty()) return null val first = line[0] - var i = if (first == '"' || first == '\'') { - scanQuoted(line)?.second ?: return false + val key: String + var i: Int + if (first == '"' || first == '\'') { + val (content, end) = scanQuoted(line) ?: return null + key = content + i = end } else { - if (!isIdentifierKeyStart(first)) return false - var j = 1 - while (j < line.length && isIdentifierKeyChar(line[j])) j++ - j + if (!isIdentifierKeyStart(first)) return null + i = 1 + while (i < line.length && isIdentifierKeyChar(line[i])) i++ + key = line.substring(0, i) } while (i < line.length && line[i] == ' ') i++ - return i < line.length && line.isMappingColonAt(i) + return if (i < line.length && line.isMappingColonAt(i)) key else null } // A key [YamlWriter] may write plain: identifier-shaped, so it passes diff --git a/markanywhere-yaml/src/commonTest/kotlin/YamlKeyLineTest.kt b/markanywhere-yaml/src/commonTest/kotlin/YamlKeyLineTest.kt index f03d9fd..3f0628c 100644 --- a/markanywhere-yaml/src/commonTest/kotlin/YamlKeyLineTest.kt +++ b/markanywhere-yaml/src/commonTest/kotlin/YamlKeyLineTest.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.yaml import com.xemantic.kotlin.test.assert +import com.xemantic.kotlin.test.sameAs import kotlin.test.Test /** @@ -51,4 +52,19 @@ class YamlKeyLineTest { // then for (line in lines) assert(!isYamlKeyLine(line)) } + + @Test + fun `should read the key of a mapping key line`() { + // when + val plain = yamlKeyLineKeyOrNull("title: \"A long") + val doubleQuoted = yamlKeyLineKeyOrNull("\"og:\\u0074itle\" : x") + val singleQuoted = yamlKeyLineKeyOrNull("'it''s':") + val none = yamlKeyLineKeyOrNull("title:x") + + // then + plain sameAs "title" + doubleQuoted sameAs "og:title" + singleQuoted sameAs "it's" + assert(none == null) + } } From 957692de10fa88319c71608f0ac2c5b4599fdf17 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda <morisil@xemantic.com> Date: Wed, 30 Sep 2026 15:31:44 +0200 Subject: [PATCH 25/25] Read no value from front matter entries continued by verbatim lines (#82) An entry continued on indented lines, or holding indented verbatim lines under a bare `key:`, has a value front matter readers read but the YAML subset keeps verbatim. FrontMatterEntryReader marks it verbatim, so wrapInHtmlDocument emits no truncated or indented <title>/<meta> and ensureFrontmatterTitle keeps it as content. The proper fix, marking such entries in YamlParser, is tracked in #85. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../kotlin/FrontMatterEntryReader.kt | 35 +++++--- .../kotlin/EnsureFrontmatterTitleTest.kt | 7 +- .../kotlin/WrapInHtmlDocumentTest.kt | 79 ++++++++++++++++++- 3 files changed, 106 insertions(+), 15 deletions(-) diff --git a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt index 0701998..11f21c3 100644 --- a/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt +++ b/markanywhere-html/src/commonMain/kotlin/FrontMatterEntryReader.kt @@ -41,10 +41,12 @@ internal fun SemanticEvent.mayPrecedeFrontmatter(): Boolean = this is Text && te // A top-level front matter `entry`: its key as spelled, `type`, text, whether // it holds nested marks, and the span of its events among those read — the // indented verbatim lines continuing it included, so dropping the span drops -// them too, rather than leaving them to continue the entry before it. A -// verbatim line defining a key (one outside the YAML subset, such as a -// multi-line quoted scalar) is an entry too, [isVerbatim]: its value is -// unknown, so it is never head metadata, yet it holds the key for readers. +// them too, rather than leaving them to continue the entry before it. An +// entry whose value is unknown is [isVerbatim], never head metadata, yet it +// holds the key for readers: a verbatim line defining a key (one outside the +// YAML subset, such as a multi-line quoted scalar), an entry continued by +// indented verbatim lines (readers join them into the value), and one +// holding indented verbatim lines under a bare `key:`. internal class FrontMatterEntry( val key: String, val type: String?, @@ -79,6 +81,7 @@ internal class FrontMatterEntryReader( private var open: SemanticEvent.Mark? = null private var openStart = 0 private var openHasChildren = false + private var openIsVerbatim = false private val openText = StringBuilder() // the last top-level entry read, held until its span is known @@ -101,6 +104,7 @@ internal class FrontMatterEntryReader( open = if (event.name == "entry" && event["key"] != null) event else null openStart = index openHasChildren = false + openIsVerbatim = false openText.clear() } else if (depth > 2) { openHasChildren = true @@ -108,7 +112,14 @@ internal class FrontMatterEntryReader( } is Text -> when (depth) { 1 -> readVerbatimLine(event.text) - 2 -> openText.append(event.text) + 2 -> { + // the YAML parser strips a scalar's indentation, never + // a verbatim line's, which also keeps its `\n` + if (event.text.startsWithIndentation() && event.text.endsWith('\n')) { + openIsVerbatim = true + } + openText.append(event.text) + } } is Unmark -> { if (--depth == 1) closeEntry() @@ -135,8 +146,8 @@ internal class FrontMatterEntryReader( private fun readVerbatimLine(text: String) { if (text.isHtmlBlank()) return val entry = pending - if (text.first() == ' ' || text.first() == '\t') { - if (entry != null) pending = entry.extendedTo(index) + if (text.startsWithIndentation()) { + if (entry != null) pending = entry.continuedTo(index) return } reportPending() @@ -161,7 +172,7 @@ internal class FrontMatterEntryReader( type = it["type"], text = openText.toString(), hasChildren = openHasChildren, - isVerbatim = false, + isVerbatim = openIsVerbatim, start = openStart, end = index ) @@ -177,5 +188,9 @@ internal class FrontMatterEntryReader( } -private fun FrontMatterEntry.extendedTo(end: Int) = - FrontMatterEntry(key, type, text, hasChildren, isVerbatim, start, end) +// The entry continued by the verbatim line ending at [end]: its value is +// no longer known. +private fun FrontMatterEntry.continuedTo(end: Int) = + FrontMatterEntry(key, type, text, hasChildren, isVerbatim = true, start, end) + +private fun String.startsWithIndentation() = startsWith(' ') || startsWith('\t') diff --git a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt index b9bc9d3..cb56deb 100644 --- a/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/EnsureFrontmatterTitleTest.kt @@ -1452,15 +1452,16 @@ class EnsureFrontmatterTitleTest { } @Test - fun `should drop the continuation lines of a dropped title entry`() = runTest { - // given — left behind, the indented line would continue `foo` + fun `should keep a title variant continued on indented lines as content`() = runTest { + // given — readers join the continuation lines into the value, which + // the YAML subset keeps verbatim, so the variant's title is unknown val markdown = "---\nfoo: a\nTitle: x\n cont\n more\ntitle: Real\n---\n\nBody." // when val output = flowOf(markdown).parse().ensureFrontmatterTitle().renderMarkdown() // then - output sameAs "---\nfoo: a\ntitle: Real\n---\n\nBody." + output sameAs markdown } @Test diff --git a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt index a99a791..4235749 100644 --- a/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt +++ b/markanywhere-html/src/commonTest/kotlin/WrapInHtmlDocumentTest.kt @@ -105,10 +105,12 @@ class WrapInHtmlDocumentTest { @Test fun `should trim HTML whitespace around the lang`() = runTest { - // given — as simplifyHtml trims it on the way in + // given — as simplifyHtml trims it on the way in (text that opens + // with indentation and ends with a newline would read as a verbatim + // line of unknown value) val input = semanticEvents { "frontmatter" { - "entry"("key" to "lang") { +" en\n" } + "entry"("key" to "lang") { +"\n en " } } "p" { +"Body." } } @@ -436,6 +438,79 @@ class WrapInHtmlDocumentTest { """.trimIndent() } + @Test + fun `should read no value from an entry continued on indented lines`() = runTest { + // given — readers join the continuation lines into the value, which + // the YAML subset keeps verbatim, so the first line alone is not it + val markdown = """ + --- + title: A long + title + description: first + second + author: Alice + --- + Body. + """.trimIndent() + + // when + val html = flowOf(markdown) + .parse() + .wrapInHtmlDocument() + .renderHtml() + + // then + html sameAsHtml """ + <html> + <head> + <meta name="author" content="Alice"/> + </head> + <body> + <p> + Body. + </p> + </body> + </html> + """.trimIndent() + } + + @Test + fun `should read no value from indented verbatim lines under a bare key`() = runTest { + // given — the lines are outside the YAML subset, kept verbatim with + // their indentation, and readers disagree on what they hold + val markdown = """ + --- + title: + [a:, b] + description: + first line + second + author: Alice + --- + Body. + """.trimIndent() + + // when + val html = flowOf(markdown) + .parse() + .wrapInHtmlDocument() + .renderHtml() + + // then + html sameAsHtml """ + <html> + <head> + <meta name="author" content="Alice"/> + </head> + <body> + <p> + Body. + </p> + </body> + </html> + """.trimIndent() + } + @Test fun `should reconstruct head metadata extracted by simplifyHtml`() = runTest { // given — a captured (tagged) HTML document whose head simplifyHtml