diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 979fd89..ea49302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,4 +20,5 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm lint - run: pnpm exec tsc -b + - run: pnpm check:reader - run: pnpm build diff --git a/package.json b/package.json index 7fcc1a5..e553b37 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build": "tsc -b && vite build && vite build --ssr src/entry-ssr.tsx --outDir dist-ssr && node scripts/prerender.mjs", "build:client": "tsc -b && vite build", "lint": "oxlint", + "check:reader": "node --experimental-strip-types scripts/check-reader.mjs", "preview": "vite preview" }, "dependencies": { diff --git a/scripts/check-reader.mjs b/scripts/check-reader.mjs new file mode 100644 index 0000000..f23e156 --- /dev/null +++ b/scripts/check-reader.mjs @@ -0,0 +1,183 @@ +#!/usr/bin/env node +/** + * check-reader.mjs — regression guard for the reader's markdown cleanup. + * + * cleanReaderMarkdown() decides where a scraped page's navigation ends and its + * body begins. That boundary is a heuristic, and it has broken in both + * directions: too eager and a link-dense homepage is wiped down to its footer; + * too shy and a short page keeps its entire sidebar. Both shapes are pinned + * below so neither can come back unnoticed. + * + * Offline and deterministic — the cases are hand-written, not fetched. For the + * live acceptance test against real URLs see verify-samples.mjs. + * + * Usage: node --experimental-strip-types scripts/check-reader.mjs + * Exit code: 0 if every case passes, 1 otherwise. + */ + +import assert from 'node:assert/strict' + +const { cleanReaderMarkdown } = await import('../src/core/content.ts') + +const lines = (markdown) => + cleanReaderMarkdown(markdown) + .split('\n') + .filter((line) => line.trim()) + +/** A wiki sidebar: dozens of short links, the article's own H1 dropped in the + * middle of it, then the skin's appearance panel — and a body too short to + * contain a single long paragraph to anchor on. */ +const WIKI_SHORT = `Title: Moyu +URL Source: https://en.wikipedia.org/wiki/Moyu + +Markdown Content: +- [x] Main menu + +* [Main page](https://en.wikipedia.org/wiki/Main_Page) +* [Contents](https://en.wikipedia.org/wiki/Wikipedia:Contents) +* [Current events](https://en.wikipedia.org/wiki/Portal:Current_events) +* [Random article](https://en.wikipedia.org/wiki/Special:Random) +* [About Wikipedia](https://en.wikipedia.org/wiki/Wikipedia:About) +* [Contact us](https://en.wikipedia.org/wiki/Wikipedia:Contact_us) + +[![Image 2](https://en.wikipedia.org/static/images/icons/enwiki.svg)![Image 3: Wikipedia](https://en.wikipedia.org/static/images/wordmark.svg)](https://en.wikipedia.org/wiki/Main_Page) + +[Search](https://en.wikipedia.org/wiki/Special:Search) + +Search + +# Moyu + +- [x] Tools + +* [What links here](https://en.wikipedia.org/wiki/Special:WhatLinksHere/Moyu) +* [Permanent link](https://en.wikipedia.org/w/index.php?title=Moyu&oldid=1) +* [Page information](https://en.wikipedia.org/w/index.php?title=Moyu&action=info) +* [Cite this page](https://en.wikipedia.org/wiki/Special:CiteThisPage) + +Appearance + +Text + +* Small Standard Large + +This page always uses small font size + +Width + +* Standard Wide + +The content is as wide as possible for your browser window. + +From Wikipedia, the free encyclopedia + +**Moyu** may refer to: + +* Moyu, the edible corm of the konjac plant +* [Karakax County](https://en.wikipedia.org/wiki/Karakax_County), in Xinjiang, China +` + +/** A link-dense homepage: every headline is a link, so the first "paragraph" + * lives down in the footer. Trimming to it would delete the whole page. */ +const HOMEPAGE = `Title: BBC News + +Markdown Content: +## [The student behind the protests that forced an Indian minister to quit, after weeks of unrest](https://www.bbc.com/news/articles/a1) +## [New hurricane season record as a historic El Nino leaves the Atlantic eerily quiet this year](https://www.bbc.com/news/articles/a2) +## [Hong Kong activists sentenced to up to seven years in prison over the annual vigil they held](https://www.bbc.com/news/articles/a3) +## [Coach crash kills several people in the Swiss Alps as rescuers work through the night to help](https://www.bbc.com/news/articles/a4) +## [Princess Diana's revenge dress goes up for auction with bids expected to reach seven figures](https://www.bbc.com/news/articles/a5) +` + +/** A 404 page: the same site menu, but what follows it is the body, announced + * by a heading. The heading must survive — it is the whole message. */ +const ERROR_PAGE = `Title: Try searching for it instead + +Markdown Content: +* [Home](https://www.bbc.com/) +* [News](https://www.bbc.com/news) +* [Sport](https://www.bbc.com/sport) +* [Business](https://www.bbc.com/business) +* [Technology](https://www.bbc.com/technology) +* [Culture](https://www.bbc.com/culture) + +Site search + +## Error 404 - Oops, the page you're looking for is no longer here + +We can't find the page you're looking for. If you typed the address by hand, check it for typos and try again. +` + +/** Nothing but menu. There is no body to find, so the page stays whole rather + * than being trimmed away to nothing. */ +const LINK_LIST = `Title: Sitemap + +Markdown Content: +* [Home](https://example.com/) +* [About](https://example.com/about) +* [Blog](https://example.com/blog) +* [Careers](https://example.com/careers) +* [Contact](https://example.com/contact) +* [Privacy](https://example.com/privacy) +` + +const cases = [ + [ + 'wiki sidebar is dropped from a page with no long paragraph', + () => { + const out = lines(WIKI_SHORT) + assert.equal(out[0], 'From Wikipedia, the free encyclopedia') + assert.ok( + out.some((line) => line.includes('may refer to')), + 'the body survived', + ) + assert.ok( + !out.some((line) => line.includes('Main page') || line.includes('Permanent link')), + 'no menu item survived', + ) + assert.ok( + !out.some((line) => line.startsWith('Appearance') || line.startsWith('Width')), + 'no appearance-panel chrome survived', + ) + }, + ], + [ + 'a link-dense homepage keeps every headline', + () => { + const headlines = lines(HOMEPAGE).filter((line) => line.startsWith('## [')) + assert.equal(headlines.length, 5, 'all five headlines survived') + }, + ], + [ + "a 404 page keeps the heading that carries its message", + () => { + const out = lines(ERROR_PAGE) + assert.ok(out[0].startsWith('## Error 404'), `body starts at the heading, got: ${out[0]}`) + assert.ok(out.some((line) => line.startsWith("We can't find the page"))) + assert.ok(!out.some((line) => line.includes('[Business]')), 'the menu is gone') + }, + ], + [ + 'a page that is nothing but links is left alone', + () => { + const out = lines(LINK_LIST) + assert.ok( + out.some((line) => line.includes('[Privacy]')), + 'the link list survived', + ) + }, + ], +] + +let failed = 0 +for (const [name, run] of cases) { + try { + run() + console.log(` ok ${name}`) + } catch (err) { + failed++ + console.log(` FAIL ${name}\n ${err.message.split('\n').join('\n ')}`) + } +} +console.log(`\n${cases.length - failed}/${cases.length} passed`) +process.exit(failed ? 1 : 0) diff --git a/src/core/content.ts b/src/core/content.ts index 6ee783b..3e6e510 100644 --- a/src/core/content.ts +++ b/src/core/content.ts @@ -363,24 +363,78 @@ function trimLeadingNav(lines: string[]) { // ("[Watch the final live…](url)") is real content, not navigation — // otherwise link-dense homepages have their first prose line down in the // footer and everything above it gets eaten. - const proseLength = (raw: string) => { - const t = raw.trim() - if (/^([#>|]|[-*+]\s|!\[)/.test(t)) return 0 - const visible = t + const visibleText = (raw: string) => + raw .replace(/!\[[^\]]*\]\([^)]*\)/g, '') .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') .trim() - return visible.length + const proseLength = (raw: string) => { + const t = raw.trim() + if (/^([#>|]|[-*+]\s|!\[)/.test(t)) return 0 + return visibleText(t).length + } + // The filler a menu is padded with: a short bare label between two groups + // ("Tools", "Appearance"), a collapsible group's own toggle ("- [x] Tools"), + // or a logo — a link carrying no text of its own, image or not. No sentence, + // no link with words in it, so a body of short lines is never mistaken for it. + const isChromeLabel = (raw: string) => { + const t = raw.trim().replace(/^[-*+]\s+(\[[x ]\]\s*)?/i, '') + if (!t) return false + if (/\]\(/.test(t)) return visibleText(t) === '' + if (/[.!?:]$/.test(t) || /[[\]()#>|]/.test(t)) return false + return t.length <= 30 && t.split(/\s+/).length <= 4 } const firstProse = lines.findIndex((line) => proseLength(line) > 120) - if (firstProse <= 0) return lines - // Safety valve: a preamble is a header, not half the document. If the - // heuristic wants to delete most of the page, it is wrong — keep everything. - if (firstProse > lines.length * 0.5) return lines + // A long paragraph is the best anchor for "the body starts here", but short + // pages (disambiguation stubs, link lists) have none, and on a nav-heavy site + // the first one can sit below the halfway mark — both cases used to return + // the sidebar intact. Fall back to the end of the leading menu run, which + // needs no paragraph at all. + if (firstProse <= 0 || firstProse > lines.length * 0.5) { + return trimNavRun(lines, isNavish, isChromeLabel) + } const navCount = lines.slice(0, firstProse).filter(isNavish).length return navCount >= 5 ? lines.slice(firstProse) : lines } +// The run of menu items that opens the page. It starts at the first navish line +// (skipping the reader's own "Title:/URL Source:" header) and runs while every +// line is either a menu item or a chrome label between menu groups — so it ends +// exactly at the first line that could be content, and a body of short bullets +// is never swallowed. A no-op unless the run holds at least NAV_RUN_MIN items. +const NAV_RUN_MIN = 5 +function trimNavRun( + lines: string[], + isNavish: (line: string) => boolean, + isChromeLabel: (line: string) => boolean, +) { + const firstNav = lines.findIndex(isNavish) + // A menu that only shows up a third of the way down is not a preamble. + if (firstNav < 0 || firstNav > lines.length * 0.3) return lines + let navCount = 0 + let end = firstNav + // Wikis drop the article's own H1 in the middle of the sidebar, so one + // heading may be crossed — but only if the menu really resumes after it. + // Otherwise the heading is the start of the body (a 404 page's "Error 404"). + let headingAt = -1 + let navBeforeHeading = 0 + for (; end < lines.length; end++) { + const line = lines[end].trim() + if (!line) continue + if (isNavish(line)) navCount++ + else if (isChromeLabel(line)) continue + else if (headingAt < 0 && /^#{1,6}\s+\S/.test(line) && line.length <= 80) { + headingAt = end + navBeforeHeading = navCount + } else break + } + if (headingAt >= 0 && navCount === navBeforeHeading) end = headingAt + // Fewer items than a menu has, or nothing but menu (the page really is a link + // list) — either way, keep the page whole. + if (navCount < NAV_RUN_MIN || end >= lines.length) return lines + return lines.slice(end) +} + function readerBoilerplateCandidate(line = '') { return line .trim() @@ -413,6 +467,11 @@ function isReaderBoilerplateLine(line: string, nextLine = '') { /^Navigation$/i, /^Contribute$/i, /^Personal tools$/i, + // MediaWiki's "Appearance" side panel — the sentences it ships with. The + // bare labels around them ("Text", "Width", "Color") are left alone: they + // are ordinary section headings on plenty of real articles. + /^This page (always uses small font size|is always in (light|dark) mode)\.?$/i, + /^The content is as wide as possible for your browser window\.?$/i, /^Toggle .*(subsection|section|the table of contents)/i, /^\[?edit\]?(\([^)]*\))?( links| source| section)?$/i, /^Retrieved from /i,