fix(reader): short pages kept their whole sidebar

.github/workflows/ci.yml

@@ -20,4 +20,5 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm lint
       - run: pnpm exec tsc -b
+      - run: pnpm check:reader
       - run: pnpm build

package.json

@@ -15,6 +15,7 @@
     "build": "tsc -b && vite build && vite build --ssr src/entry-ssr.tsx --outDir dist-ssr && node scripts/prerender.mjs",
     "build:client": "tsc -b && vite build",
     "lint": "oxlint",
+    "check:reader": "node --experimental-strip-types scripts/check-reader.mjs",
     "preview": "vite preview"
   },
   "dependencies": {

scripts/check-reader.mjs

@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+/**
+ * check-reader.mjs — regression guard for the reader's markdown cleanup.
+ *
+ * cleanReaderMarkdown() decides where a scraped page's navigation ends and its
+ * body begins. That boundary is a heuristic, and it has broken in both
+ * directions: too eager and a link-dense homepage is wiped down to its footer;
+ * too shy and a short page keeps its entire sidebar. Both shapes are pinned
+ * below so neither can come back unnoticed.
+ *
+ * Offline and deterministic — the cases are hand-written, not fetched. For the
+ * live acceptance test against real URLs see verify-samples.mjs.
+ *
+ * Usage: node --experimental-strip-types scripts/check-reader.mjs
+ * Exit code: 0 if every case passes, 1 otherwise.
+ */
+
+import assert from 'node:assert/strict'
+
+const { cleanReaderMarkdown } = await import('../src/core/content.ts')
+
+const lines = (markdown) =>
+  cleanReaderMarkdown(markdown)
+    .split('\n')
+    .filter((line) => line.trim())
+
+/** A wiki sidebar: dozens of short links, the article's own H1 dropped in the
+ *  middle of it, then the skin's appearance panel — and a body too short to
+ *  contain a single long paragraph to anchor on. */
+const WIKI_SHORT = `Title: Moyu
+URL Source: https://en.wikipedia.org/wiki/Moyu
+
+Markdown Content:
+- [x] Main menu
+
+*   [Main page](https://en.wikipedia.org/wiki/Main_Page)
+*   [Contents](https://en.wikipedia.org/wiki/Wikipedia:Contents)
+*   [Current events](https://en.wikipedia.org/wiki/Portal:Current_events)
+*   [Random article](https://en.wikipedia.org/wiki/Special:Random)
+*   [About Wikipedia](https://en.wikipedia.org/wiki/Wikipedia:About)
+*   [Contact us](https://en.wikipedia.org/wiki/Wikipedia:Contact_us)
+
+[![Image 2](https://en.wikipedia.org/static/images/icons/enwiki.svg)![Image 3: Wikipedia](https://en.wikipedia.org/static/images/wordmark.svg)](https://en.wikipedia.org/wiki/Main_Page)
+
+[Search](https://en.wikipedia.org/wiki/Special:Search)
+
+Search
+
+# Moyu
+
+- [x] Tools
+
+*   [What links here](https://en.wikipedia.org/wiki/Special:WhatLinksHere/Moyu)
+*   [Permanent link](https://en.wikipedia.org/w/index.php?title=Moyu&oldid=1)
+*   [Page information](https://en.wikipedia.org/w/index.php?title=Moyu&action=info)
+*   [Cite this page](https://en.wikipedia.org/wiki/Special:CiteThisPage)
+
+Appearance
+
+Text
+
+*   Small  Standard  Large
+
+This page always uses small font size
+
+Width
+
+*   Standard  Wide
+
+The content is as wide as possible for your browser window.
+
+From Wikipedia, the free encyclopedia
+
+**Moyu** may refer to:
+
+*   Moyu, the edible corm of the konjac plant
+*   [Karakax County](https://en.wikipedia.org/wiki/Karakax_County), in Xinjiang, China
+`
+
+/** A link-dense homepage: every headline is a link, so the first "paragraph"
+ *  lives down in the footer. Trimming to it would delete the whole page. */
+const HOMEPAGE = `Title: BBC News
+
+Markdown Content:
+## [The student behind the protests that forced an Indian minister to quit, after weeks of unrest](https://www.bbc.com/news/articles/a1)
+## [New hurricane season record as a historic El Nino leaves the Atlantic eerily quiet this year](https://www.bbc.com/news/articles/a2)
+## [Hong Kong activists sentenced to up to seven years in prison over the annual vigil they held](https://www.bbc.com/news/articles/a3)
+## [Coach crash kills several people in the Swiss Alps as rescuers work through the night to help](https://www.bbc.com/news/articles/a4)
+## [Princess Diana's revenge dress goes up for auction with bids expected to reach seven figures](https://www.bbc.com/news/articles/a5)
+`
+
+/** A 404 page: the same site menu, but what follows it is the body, announced
+ *  by a heading. The heading must survive — it is the whole message. */
+const ERROR_PAGE = `Title: Try searching for it instead
+
+Markdown Content:
+*   [Home](https://www.bbc.com/)
+*   [News](https://www.bbc.com/news)
+*   [Sport](https://www.bbc.com/sport)
+*   [Business](https://www.bbc.com/business)
+*   [Technology](https://www.bbc.com/technology)
+*   [Culture](https://www.bbc.com/culture)
+
+Site search
+
+## Error 404 - Oops, the page you're looking for is no longer here
+
+We can't find the page you're looking for. If you typed the address by hand, check it for typos and try again.
+`
+
+/** Nothing but menu. There is no body to find, so the page stays whole rather
+ *  than being trimmed away to nothing. */
+const LINK_LIST = `Title: Sitemap
+
+Markdown Content:
+*   [Home](https://example.com/)
+*   [About](https://example.com/about)
+*   [Blog](https://example.com/blog)
+*   [Careers](https://example.com/careers)
+*   [Contact](https://example.com/contact)
+*   [Privacy](https://example.com/privacy)
+`
+
+const cases = [
+  [
+    'wiki sidebar is dropped from a page with no long paragraph',
+    () => {
+      const out = lines(WIKI_SHORT)
+      assert.equal(out[0], 'From Wikipedia, the free encyclopedia')
+      assert.ok(
+        out.some((line) => line.includes('may refer to')),
+        'the body survived',
+      )
+      assert.ok(
+        !out.some((line) => line.includes('Main page') || line.includes('Permanent link')),
+        'no menu item survived',
+      )
+      assert.ok(
+        !out.some((line) => line.startsWith('Appearance') || line.startsWith('Width')),
+        'no appearance-panel chrome survived',
+      )
+    },
+  ],
+  [
+    'a link-dense homepage keeps every headline',
+    () => {
+      const headlines = lines(HOMEPAGE).filter((line) => line.startsWith('## ['))
+      assert.equal(headlines.length, 5, 'all five headlines survived')
+    },
+  ],
+  [
+    "a 404 page keeps the heading that carries its message",
+    () => {
+      const out = lines(ERROR_PAGE)
+      assert.ok(out[0].startsWith('## Error 404'), `body starts at the heading, got: ${out[0]}`)
+      assert.ok(out.some((line) => line.startsWith("We can't find the page")))
+      assert.ok(!out.some((line) => line.includes('[Business]')), 'the menu is gone')
+    },
+  ],
+  [
+    'a page that is nothing but links is left alone',
+    () => {
+      const out = lines(LINK_LIST)
+      assert.ok(
+        out.some((line) => line.includes('[Privacy]')),
+        'the link list survived',
+      )
+    },
+  ],
+]
+
+let failed = 0
+for (const [name, run] of cases) {
+  try {
+    run()
+    console.log(`  ok    ${name}`)
+  } catch (err) {
+    failed++
+    console.log(`  FAIL  ${name}\n        ${err.message.split('\n').join('\n        ')}`)
+  }
+}
+console.log(`\n${cases.length - failed}/${cases.length} passed`)
+process.exit(failed ? 1 : 0)

src/core/content.ts

@@ -363,24 +363,78 @@ function trimLeadingNav(lines: string[]) {
   // ("[Watch the final live…](url)") is real content, not navigation —
   // otherwise link-dense homepages have their first prose line down in the
   // footer and everything above it gets eaten.
-  const proseLength = (raw: string) => {
-    const t = raw.trim()
-    if (/^([#>|]|[-*+]\s|!\[)/.test(t)) return 0
-    const visible = t
+  const visibleText = (raw: string) =>
+    raw
       .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
       .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
       .trim()
-    return visible.length
+  const proseLength = (raw: string) => {
+    const t = raw.trim()
+    if (/^([#>|]|[-*+]\s|!\[)/.test(t)) return 0
+    return visibleText(t).length
+  }
+  // The filler a menu is padded with: a short bare label between two groups
+  // ("Tools", "Appearance"), a collapsible group's own toggle ("- [x] Tools"),
+  // or a logo — a link carrying no text of its own, image or not. No sentence,
+  // no link with words in it, so a body of short lines is never mistaken for it.
+  const isChromeLabel = (raw: string) => {
+    const t = raw.trim().replace(/^[-*+]\s+(\[[x ]\]\s*)?/i, '')
+    if (!t) return false
+    if (/\]\(/.test(t)) return visibleText(t) === ''
+    if (/[.!?:]$/.test(t) || /[[\]()#>|]/.test(t)) return false
+    return t.length <= 30 && t.split(/\s+/).length <= 4
   }
   const firstProse = lines.findIndex((line) => proseLength(line) > 120)
-  if (firstProse <= 0) return lines
-  // Safety valve: a preamble is a header, not half the document. If the
-  // heuristic wants to delete most of the page, it is wrong — keep everything.
-  if (firstProse > lines.length * 0.5) return lines
+  // A long paragraph is the best anchor for "the body starts here", but short
+  // pages (disambiguation stubs, link lists) have none, and on a nav-heavy site
+  // the first one can sit below the halfway mark — both cases used to return
+  // the sidebar intact. Fall back to the end of the leading menu run, which
+  // needs no paragraph at all.
+  if (firstProse <= 0 || firstProse > lines.length * 0.5) {
+    return trimNavRun(lines, isNavish, isChromeLabel)
+  }
   const navCount = lines.slice(0, firstProse).filter(isNavish).length
   return navCount >= 5 ? lines.slice(firstProse) : lines
 }
 
+// The run of menu items that opens the page. It starts at the first navish line
+// (skipping the reader's own "Title:/URL Source:" header) and runs while every
+// line is either a menu item or a chrome label between menu groups — so it ends
+// exactly at the first line that could be content, and a body of short bullets
+// is never swallowed. A no-op unless the run holds at least NAV_RUN_MIN items.
+const NAV_RUN_MIN = 5
+function trimNavRun(
+  lines: string[],
+  isNavish: (line: string) => boolean,
+  isChromeLabel: (line: string) => boolean,
+) {
+  const firstNav = lines.findIndex(isNavish)
+  // A menu that only shows up a third of the way down is not a preamble.
+  if (firstNav < 0 || firstNav > lines.length * 0.3) return lines
+  let navCount = 0
+  let end = firstNav
+  // Wikis drop the article's own H1 in the middle of the sidebar, so one
+  // heading may be crossed — but only if the menu really resumes after it.
+  // Otherwise the heading is the start of the body (a 404 page's "Error 404").
+  let headingAt = -1
+  let navBeforeHeading = 0
+  for (; end < lines.length; end++) {
+    const line = lines[end].trim()
+    if (!line) continue
+    if (isNavish(line)) navCount++
+    else if (isChromeLabel(line)) continue
+    else if (headingAt < 0 && /^#{1,6}\s+\S/.test(line) && line.length <= 80) {
+      headingAt = end
+      navBeforeHeading = navCount
+    } else break
+  }
+  if (headingAt >= 0 && navCount === navBeforeHeading) end = headingAt
+  // Fewer items than a menu has, or nothing but menu (the page really is a link
+  // list) — either way, keep the page whole.
+  if (navCount < NAV_RUN_MIN || end >= lines.length) return lines
+  return lines.slice(end)
+}
+
 function readerBoilerplateCandidate(line = '') {
   return line
     .trim()
@@ -413,6 +467,11 @@ function isReaderBoilerplateLine(line: string, nextLine = '') {
     /^Navigation$/i,
     /^Contribute$/i,
     /^Personal tools$/i,
+    // MediaWiki's "Appearance" side panel — the sentences it ships with. The
+    // bare labels around them ("Text", "Width", "Color") are left alone: they
+    // are ordinary section headings on plenty of real articles.
+    /^This page (always uses small font size|is always in (light|dark) mode)\.?$/i,
+    /^The content is as wide as possible for your browser window\.?$/i,
     /^Toggle .*(subsection|section|the table of contents)/i,
     /^\[?edit\]?(\([^)]*\))?( links| source| section)?$/i,
     /^Retrieved from /i,