diff --git a/src/lib/domain/segmenter.spec.ts b/src/lib/domain/segmenter.spec.ts index bc50b5c..685664c 100644 --- a/src/lib/domain/segmenter.spec.ts +++ b/src/lib/domain/segmenter.spec.ts @@ -457,6 +457,38 @@ describe('narrated construct segmentation', () => { } }); + it('terminates when a math span opens on the only break token in the window', () => { + // A web article whose figure caption is one long sentence puts the + // comma INSIDE the equation: "…an author sees$, which is why…$". The + // hard cut lands inside that span, so the next window starts exactly + // at the span and its only break token sits at index 0 — where + // `lastIndexOf(token, -1)` used to keep finding the same token for + // ever and spin the main thread. (Regression: importing + // neovand.github.io/Moire/paper hung the tab.) + const prose = 'the heterodyne ratio is drawn live beside the envelope so an author sees '; + const tex = + ', which is why they stop short of the two lobes flanking the centres and the fringe system appears in neither layer'; + const tail = + ' and the prose keeps going past the equation for a good long stretch so that the splitter still has more than one window of text left to walk through after the equation ends'; + // Whether the hard cut lands inside the span depends on where the + // equation starts, so sweep the openers that put it near the boundary. + for (let length = 120; length <= MAX_SEGMENT_CHARS; length += 1) { + const opener = prose.repeat(4).slice(0, length); + const paragraph = block({ + id: 'b9', + text: `${opener}${tex}${tail}`, + inlines: [{ text: opener }, { text: tex, math: true }, { text: tail }] + }); + const segments = segmentBlocks([paragraph]); + expect(segments.length).toBeGreaterThan(1); + const mathEnd = length + tex.length; + for (const segment of segments) { + expect(segment.start > length && segment.start < mathEnd).toBe(false); + expect(segment.end > length && segment.end < mathEnd).toBe(false); + } + } + }); + it('leaves sentences with single-letter math as plain word-highlighted text', () => { const paragraph = block({ id: 'b4', diff --git a/src/lib/domain/segmenter.ts b/src/lib/domain/segmenter.ts index bda59fb..d597ff0 100644 --- a/src/lib/domain/segmenter.ts +++ b/src/lib/domain/segmenter.ts @@ -135,8 +135,11 @@ function splitLongSentence( let breakAt = -1; for (const token of ['; ', ', ', ' — ']) { let index = candidate.lastIndexOf(token); + // `lastIndexOf(token, -1)` searches from 0, not before it, so a + // protected token sitting at index 0 would be found again for + // ever: once there is nothing left of it, there is no break. while (index >= 0 && spanAt(absoluteStart + cursor + index + 1)) { - index = candidate.lastIndexOf(token, index - 1); + index = index > 0 ? candidate.lastIndexOf(token, index - 1) : -1; } breakAt = Math.max(breakAt, index); } diff --git a/src/lib/domain/tex-macros.spec.ts b/src/lib/domain/tex-macros.spec.ts new file mode 100644 index 0000000..7fc3df8 --- /dev/null +++ b/src/lib/domain/tex-macros.spec.ts @@ -0,0 +1,167 @@ +import { DOMParser } from 'linkedom'; +import { describe, expect, it } from 'vitest'; +import { + balancedBraces, + expandMarkdownMacros, + expandTexMacros, + texMacrosFromDocument, + texMacrosFromScripts +} from './tex-macros'; + +function dom(html: string): Document { + return new DOMParser().parseFromString( + `${html}`, + 'text/html' + ) as unknown as Document; +} + +describe('balancedBraces', () => { + it('spans nested braces and ignores braces inside strings', () => { + expect(balancedBraces('x = {"a": "{{{"} tail', 4)).toBe('{"a": "{{{"}'); + expect(balancedBraces('{ a: { b: 1 } }', 0)).toBe('{ a: { b: 1 } }'); + }); + + it('returns null when the group never closes or does not start here', () => { + expect(balancedBraces('{ unterminated', 0)).toBeNull(); + expect(balancedBraces('not a group', 0)).toBeNull(); + }); +}); + +describe('texMacrosFromScripts', () => { + it('reads a KaTeX auto-render table, backslashes and all', () => { + const script = `renderMathInElement(document.body, { + delimiters: [{ left: '$', right: '$', display: false }], + macros: {"\\\\R":"\\\\mathbb{R}","\\\\idx":"\\\\phi","\\\\Rot":"\\\\mathbf{R}_{#1}"}, + strict: false + });`; + expect(texMacrosFromScripts([script])).toEqual({ + R: '\\mathbb{R}', + idx: '\\phi', + Rot: '\\mathbf{R}_{#1}' + }); + }); + + it('reads a MathJax table: bare keys, single quotes, [body, arity] values', () => { + const script = `window.MathJax = { tex: { macros: { + RR: '{\\\\bf R}', + bold: ['{\\\\bf #1}', 1], + } } };`; + expect(texMacrosFromScripts([script])).toEqual({ RR: '{\\bf R}', bold: '{\\bf #1}' }); + }); + + it('ignores tables it cannot read and names it could never match back', () => { + expect(texMacrosFromScripts(['macros: { broken'])).toEqual({}); + expect(texMacrosFromScripts(['macros: [1, 2]'])).toEqual({}); + expect(texMacrosFromScripts([`macros: {"\\\\two words": "x", "\\\\ok": "y"}`])).toEqual({ + ok: 'y' + }); + }); + + it('takes the page scripts in order, later definitions winning', () => { + expect( + texMacrosFromScripts([`macros: {"\\\\R": "first"}`, `macros: {"\\\\R": "second"}`]) + ).toEqual({ R: 'second' }); + }); +}); + +describe('texMacrosFromDocument', () => { + it('reads inline scripts and skips external ones', () => { + const document = dom( + `` + + `` + ); + expect(texMacrosFromDocument(document)).toEqual({ het: '\\eta' }); + }); + + it('is empty for a page with no macro table', () => { + expect(texMacrosFromDocument(dom(''))).toEqual({}); + }); +}); + +describe('expandTexMacros', () => { + const macros = { + R: '\\mathbb{R}', + idx: '\\phi', + ph: '\\psi', + Rot: '\\mathbf{R}_{#1}', + pair: '(#1,\\;#2)' + }; + + it('rewrites bare macros and leaves longer names that merely start alike', () => { + expect(expandTexMacros('p \\in \\R^2', macros)).toBe('p \\in \\mathbb{R}^2'); + expect(expandTexMacros('\\idx(p) - \\ph(p)', macros)).toBe('\\phi(p) - \\psi(p)'); + // \Rotate is a different command, not \Rot followed by "ate". + expect(expandTexMacros('\\Rotate', macros)).toBe('\\Rotate'); + }); + + it('takes braced groups and single tokens as arguments', () => { + expect(expandTexMacros('\\Rot{\\theta}', macros)).toBe('\\mathbf{R}_{\\theta}'); + expect(expandTexMacros('\\Rot n', macros)).toBe('\\mathbf{R}_{n}'); + expect(expandTexMacros('\\Rot\\alpha', macros)).toBe('\\mathbf{R}_{\\alpha}'); + expect(expandTexMacros('\\pair{a}{b}', macros)).toBe('(a,\\;b)'); + }); + + it('reads a group whose contents carry primes and escaped braces', () => { + expect(expandTexMacros("\\Rot{-n'\\theta}", macros)).toBe("\\mathbf{R}_{-n'\\theta}"); + expect(expandTexMacros('\\Rot{\\{a\\}}', macros)).toBe('\\mathbf{R}_{\\{a\\}}'); + }); + + it('expands macros written in terms of other macros', () => { + expect(expandTexMacros('\\field', { ...macros, field: '\\R \\times \\R' })).toBe( + '\\mathbb{R} \\times \\mathbb{R}' + ); + }); + + it('leaves a macro alone when its arguments are not there', () => { + expect(expandTexMacros('\\Rot', macros)).toBe('\\Rot'); + expect(expandTexMacros('\\Rot{unterminated', macros)).toBe('\\Rot{unterminated'); + }); + + it('terminates on a self-referential table', () => { + expect(expandTexMacros('\\loop', { loop: '\\loop' })).toBe('\\loop'); + expect(expandTexMacros('\\fan', { fan: '\\fan\\fan' }).length).toBeGreaterThan(0); + }); + + it('passes through text with no macros to expand', () => { + expect(expandTexMacros('plain words', macros)).toBe('plain words'); + expect(expandTexMacros('\\alpha + \\beta', {})).toBe('\\alpha + \\beta'); + }); +}); + +describe('expandMarkdownMacros', () => { + const macros = { het: '\\eta', R: '\\mathbb{R}' }; + + it('expands inside every maths delimiter family', () => { + expect(expandMarkdownMacros('ratio $\\het$ here', macros)).toBe('ratio $\\eta$ here'); + expect(expandMarkdownMacros('$$\n\\het \\in \\R\n$$', macros)).toBe( + '$$\n\\eta \\in \\mathbb{R}\n$$' + ); + expect(expandMarkdownMacros('\\[\\het\\]', macros)).toBe('\\[\\eta\\]'); + expect(expandMarkdownMacros('\\(\\het\\)', macros)).toBe('\\(\\eta\\)'); + }); + + it('follows inline maths across a wrapped line but not past a blank one', () => { + expect(expandMarkdownMacros('lines: $\\het =\n\\R/s$ here', macros)).toBe( + 'lines: $\\eta =\n\\mathbb{R}/s$ here' + ); + // An unbalanced dollar must not swallow the next paragraph's macros. + expect(expandMarkdownMacros('costs $5\n\nprose \\het and $\\het$', macros)).toBe( + 'costs $5\n\nprose \\het and $\\eta$' + ); + }); + + it('leaves prose and fenced code untouched', () => { + expect(expandMarkdownMacros('a path C:\\het and $\\het$', macros)).toBe( + 'a path C:\\het and $\\eta$' + ); + const fenced = '```tex\n$\\het$\n```\n\nand $\\het$ in prose'; + expect(expandMarkdownMacros(fenced, macros)).toBe( + '```tex\n$\\het$\n```\n\nand $\\eta$ in prose' + ); + }); + + it('is a no-op without macros or without backslashes', () => { + expect(expandMarkdownMacros('$\\het$', {})).toBe('$\\het$'); + expect(expandMarkdownMacros('no maths here', macros)).toBe('no maths here'); + }); +}); diff --git a/src/lib/domain/tex-macros.ts b/src/lib/domain/tex-macros.ts new file mode 100644 index 0000000..d13871f --- /dev/null +++ b/src/lib/domain/tex-macros.ts @@ -0,0 +1,299 @@ +/** + * TeX macro expansion for imported pages. + * + * A paper published as HTML usually renders its own maths in the browser, and + * it hands KaTeX (or MathJax) a table of the shorthands its author writes in: + * `\idx` for `\phi`, `\R` for `\mathbb{R}`, `\Rot{\theta}` for + * `\mathbf{R}_{\theta}`. That table lives in the page's own script, so a + * reader that fetches the HTML and renders the TeX itself has never seen it — + * every custom command comes out as literal `\idx` on the page and as raw + * source in the spoken reading. + * + * Expanding the shorthands into the source at import time fixes all of that at + * once: the equations render, the deterministic verbaliser reads them, and the + * language model rewriting them sees ordinary TeX. Everything here is a pure + * string transform — a page's macro table is data, never code, and is never + * evaluated. + */ + +/** Macro name (without the leading backslash) to its replacement body. */ +export type TexMacros = Record; + +/** A hostile or broken page must not be able to blow up the import. */ +const MAX_MACROS = 256; +const MAX_BODY_CHARS = 2048; +/** Total substitutions across one expansion, so a self-referential table + * (`\a` → `\a\a`) cannot fan out. */ +const MAX_EXPANSIONS = 50_000; +/** Depth of macro-inside-macro expansion before the chain is left alone. */ +const MAX_DEPTH = 16; + +/** Highest `#n` referenced by a macro body — its argument count. */ +function arityOf(body: string): number { + let arity = 0; + for (const match of body.matchAll(/#(\d)/g)) arity = Math.max(arity, Number(match[1])); + return arity; +} + +/** + * Normalizes a parsed macro table: KaTeX writes names with the backslash + * (`"\\R"`), MathJax without it (`RR`), and MathJax lets a value be + * `[body, arity]`. Oversized tables and bodies are dropped rather than + * truncated — a partial macro would expand into broken TeX. + */ +function normalizeMacros(raw: Record): TexMacros { + const macros: TexMacros = {}; + for (const [key, value] of Object.entries(raw)) { + if (Object.keys(macros).length >= MAX_MACROS) break; + const body = typeof value === 'string' ? value : Array.isArray(value) ? value[0] : undefined; + if (typeof body !== 'string' || body.length > MAX_BODY_CHARS) continue; + const name = key.replace(/^\\/, ''); + // Only control words and single-character control symbols can ever be + // matched back out of the source. + if (!/^(?:[a-zA-Z]+|[^a-zA-Z\s])$/.test(name)) continue; + macros[name] = body; + } + return macros; +} + +/** + * Best-effort repair of an almost-JSON object literal: bare identifier keys, + * single-quoted strings and trailing commas are all legal JavaScript and all + * common in hand-written macro tables. Anything still unparseable yields no + * macros — a page whose table we cannot read simply reads as it did before. + */ +function parseObjectLiteral(source: string): Record | null { + const attempts = [source, repairJson(source)]; + for (const attempt of attempts) { + try { + const parsed: unknown = JSON.parse(attempt); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // try the next form + } + } + return null; +} + +function repairJson(source: string): string { + return ( + source + // 'single quoted' → "double quoted", with the escapes read the way + // JavaScript reads them (`\\` is one backslash, not two). + .replace(/'((?:[^'\\]|\\.)*)'/g, (_, body: string) => JSON.stringify(fromSingleQuoted(body))) + // bare identifier keys → quoted keys + .replace(/([{,]\s*)([A-Za-z_$][\w$]*)(\s*:)/g, '$1"$2"$3') + // trailing commas + .replace(/,(\s*[}\]])/g, '$1') + ); +} + +/** + * The contents of a single-quoted JavaScript string, read as JSON: the two + * escape grammars agree except that JSON rejects `\'` and requires a bare `"` + * escaped. A body that still will not parse is taken literally. + */ +function fromSingleQuoted(body: string): string { + const asJson = body.replace(/\\(.)|"/gs, (whole, escaped: string | undefined) => { + if (escaped === undefined) return '\\"'; + return escaped === "'" ? "'" : whole; + }); + try { + return JSON.parse(`"${asJson}"`) as string; + } catch { + return body; + } +} + +/** + * The balanced `{…}` starting at `start`, string literals respected so a brace + * inside a TeX body does not close the object. Returns null if it never + * closes. + */ +export function balancedBraces(source: string, start: number): string | null { + if (source[start] !== '{') return null; + let depth = 0; + let quote: string | null = null; + for (let index = start; index < source.length; index += 1) { + const char = source[index]; + if (quote) { + if (char === '\\') index += 1; + else if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") quote = char; + else if (char === '{') depth += 1; + else if (char === '}') { + depth -= 1; + if (depth === 0) return source.slice(start, index + 1); + } + } + return null; +} + +/** + * The macro table a page hands its own maths renderer. Reads every inline + * script for a `macros:` (KaTeX, MathJax v3) or `Macros:` (MathJax v2) object + * and merges what parses; later scripts win, matching the order the page's own + * renderer would see them in. + */ +export function texMacrosFromScripts(scripts: string[]): TexMacros { + const macros: TexMacros = {}; + for (const script of scripts) { + for (const match of script.matchAll(/\b[Mm]acros\s*:\s*(?=\{)/g)) { + const literal = balancedBraces(script, match.index + match[0].length); + if (!literal) continue; + const parsed = parseObjectLiteral(literal); + if (parsed) Object.assign(macros, normalizeMacros(parsed)); + } + } + return macros; +} + +/** Reads `texMacrosFromScripts` off a parsed page's inline scripts. */ +export function texMacrosFromDocument(document: Document): TexMacros { + const scripts = Array.from(document.querySelectorAll('script:not([src])'), (script) => + String(script.textContent ?? '') + ).filter((text) => text.includes('acros')); + return texMacrosFromScripts(scripts); +} + +/** + * The balanced `{…}` of a TeX group. Unlike an object literal, a quote here is + * prime notation (`{-n'\theta}`) and never opens a string; only a backslash + * escapes the brace that follows it. + */ +function balancedTexGroup(source: string, start: number): string | null { + if (source[start] !== '{') return null; + let depth = 0; + for (let index = start; index < source.length; index += 1) { + const char = source[index]; + if (char === '\\') index += 1; + else if (char === '{') depth += 1; + else if (char === '}') { + depth -= 1; + if (depth === 0) return source.slice(start, index + 1); + } + } + return null; +} + +/** One macro argument after `offset`: a braced group, or the single token + * (character or control sequence) that follows. */ +function readArgument(source: string, offset: number): { value: string; end: number } | null { + let index = offset; + while (index < source.length && /\s/.test(source[index])) index += 1; + if (index >= source.length) return null; + if (source[index] === '{') { + const group = balancedTexGroup(source, index); + if (!group) return null; + return { value: group.slice(1, -1), end: index + group.length }; + } + if (source[index] === '\\') { + const control = /^\\(?:[a-zA-Z]+|.)/.exec(source.slice(index)); + if (!control) return null; + return { value: control[0], end: index + control[0].length }; + } + return { value: source[index], end: index + 1 }; +} + +/** + * Rewrites every `\name` the table defines into its body, substituting `#1`… + * `#9` from the arguments that follow. Expansion continues over the + * substituted text so a macro defined in terms of another resolves, bounded by + * `MAX_DEPTH`; a macro that cannot take its arguments (a truncated equation, + * say) is left exactly as written. + */ +export function expandTexMacros(tex: string, macros: TexMacros): string { + if (!tex.includes('\\') || !Object.keys(macros).length) return tex; + return expandWithin(tex, macros, 0, { spent: 0 }); +} + +function expandWithin( + tex: string, + macros: TexMacros, + depth: number, + budget: { spent: number } +): string { + if (!tex.includes('\\') || depth >= MAX_DEPTH) return tex; + let out = ''; + let index = 0; + while (index < tex.length) { + const backslash = tex.indexOf('\\', index); + if (backslash < 0) { + out += tex.slice(index); + break; + } + out += tex.slice(index, backslash); + const control = /^\\(?:[a-zA-Z]+|[^a-zA-Z])/.exec(tex.slice(backslash)); + if (!control) { + out += tex.slice(backslash); + break; + } + const name = control[0].slice(1); + const body = macros[name]; + if (body === undefined || budget.spent >= MAX_EXPANSIONS) { + out += control[0]; + index = backslash + control[0].length; + continue; + } + const arity = arityOf(body); + const args: string[] = []; + let cursor = backslash + control[0].length; + let complete = true; + for (let n = 0; n < arity; n += 1) { + const argument = readArgument(tex, cursor); + if (!argument) { + complete = false; + break; + } + args.push(argument.value); + cursor = argument.end; + } + if (!complete) { + out += control[0]; + index = backslash + control[0].length; + continue; + } + budget.spent += 1; + const substituted = body.replace(/#(\d)/g, (whole, digit: string) => { + const argument = args[Number(digit) - 1]; + return argument === undefined ? whole : argument; + }); + out += expandWithin(substituted, macros, depth + 1, budget); + index = cursor; + } + return out; +} + +/** + * Every maths region of a markdown document, in either delimiter family. + * Deliberately narrow: expansion must not touch prose, where a lone `\R` + * belongs to a Windows path rather than to the page's macro table. Inline + * `$…$` may wrap across a line — source TeX is hard-wrapped all the time — but + * not across a blank one, which is where the paragraph, and any run of + * unbalanced dollars with it, ends. + */ +const MATH_REGION = + /\$\$[\s\S]*?\$\$|\\\[[\s\S]*?\\\]|\\\([\s\S]*?\\\)|\$(?!\s)(?:[^$\n]|\n(?!\s*\n))*?(? + prose.replace(MATH_REGION, (region) => expandTexMacros(region, macros)); + let out = ''; + let cursor = 0; + for (const fence of markdown.matchAll(FENCE_BLOCK)) { + out += expand(markdown.slice(cursor, fence.index)) + fence[0]; + cursor = fence.index + fence[0].length; + } + return out + expand(markdown.slice(cursor)); +} diff --git a/src/lib/domain/web-article.spec.ts b/src/lib/domain/web-article.spec.ts index a652a46..69ccb8f 100644 --- a/src/lib/domain/web-article.spec.ts +++ b/src/lib/domain/web-article.spec.ts @@ -114,6 +114,33 @@ describe('prepareArticleDom', () => { expect(document.querySelector('img')?.getAttribute('alt')).toBe('A double-slit experiment.'); }); + it('lifts a TeX-bearing caption out of its figure and leaves the alt empty', () => { + // Defuddle reads an alt matching /\\[a-zA-Z]{2,}/ as a rendered + // equation image (the picture is dropped, the caption re-emitted as one + // enormous formula), and flattens a caption left inside the figure to + // plain text. A paper's captions are full of maths, so they move out. + const document = dom( + '
Drawn only where $\\het\\le1/4$.
' + ); + prepareArticleDom(document); + expect(document.querySelector('img')?.getAttribute('alt')).toBe(''); + expect(document.querySelector('figcaption')).toBeNull(); + const moved = document.querySelector('figure')?.nextElementSibling; + expect(moved?.tagName).toBe('P'); + expect(moved?.textContent).toBe('Drawn only where $\\het\\le1/4$.'); + expect(moved?.querySelector('strong')?.textContent).toBe('only'); + }); + + it('still folds in a caption whose only maths is delimiters', () => { + const document = dom( + '
Centres differ by $34$ units.
' + ); + prepareArticleDom(document); + expect(document.querySelector('img')?.getAttribute('alt')).toBe( + 'Centres differ by $34$ units.' + ); + }); + it('keeps an authored alt and images outside figures untouched', () => { const document = dom( '
Authored
Caption.
' diff --git a/src/lib/domain/web-article.ts b/src/lib/domain/web-article.ts index 49a95e6..06bb4d7 100644 --- a/src/lib/domain/web-article.ts +++ b/src/lib/domain/web-article.ts @@ -44,6 +44,11 @@ function normalizedText(value: string | null | undefined): string { return (value ?? '').replace(/\s+/g, ' ').trim(); } +/** A caption written in TeX. Deliberately Defuddle's own `LOOKS_LIKE_LATEX_RE`: + * this is the test it applies to an alt attribute before deciding the image is + * a rendered equation. */ +const TEX_CAPTION = /\\[a-zA-Z]{2,}/; + /** * DOM fixes before extraction, for structures the extractor keeps but * mishandles: @@ -60,7 +65,15 @@ function normalizedText(value: string | null | undefined): string { * the caption text (the footnote pass only reaches body refs) — drop them; * - images whose alt is empty adopt their figcaption, so the narration layer * has a caption to describe (the local engine never invents what it cannot - * see) while the visible caption paragraph reads as usual. + * see) while the visible caption paragraph reads as usual; + * - a figure whose caption is written in TeX takes a different route. The + * extractor reads an alt matching `\command` as a rendered equation image — + * it drops the picture and re-emits the caption as one enormous bogus + * formula — and, left inside the figure, it rebuilds the caption from plain + * text, which strips the emphasis, the links, and the delimiters that made + * the equations equations. So the caption is moved out to a paragraph of its + * own and the alt is left empty: the figure survives, and a paper's captions + * read with their maths intact. Captions with no TeX in them are untouched. */ export function prepareArticleDom(document: Document): void { for (const math of Array.from(document.querySelectorAll('math'))) { @@ -80,10 +93,17 @@ export function prepareArticleDom(document: Document): void { } for (const figure of Array.from(document.querySelectorAll('figure'))) { const image = figure.querySelector('img'); - const caption = normalizedText(figure.querySelector('figcaption')?.textContent); - if (image && caption && !normalizedText(image.getAttribute('alt'))) { - image.setAttribute('alt', caption); + const element = Array.from(figure.children).find((child) => child.tagName === 'FIGCAPTION'); + const caption = normalizedText(element?.textContent); + if (!image || !element || !caption) continue; + if (TEX_CAPTION.test(caption)) { + const paragraph = document.createElement('p'); + while (element.firstChild) paragraph.appendChild(element.firstChild); + element.remove(); + figure.after(paragraph); + continue; } + if (!normalizedText(image.getAttribute('alt'))) image.setAttribute('alt', caption); } } diff --git a/src/lib/services/article-fetch.ts b/src/lib/services/article-fetch.ts index 66627ff..b9871ed 100644 --- a/src/lib/services/article-fetch.ts +++ b/src/lib/services/article-fetch.ts @@ -15,6 +15,7 @@ import { prepareArticleDom, wikipediaRestUrl } from '$lib/domain/web-article'; +import { expandMarkdownMacros, texMacrosFromDocument } from '$lib/domain/tex-macros'; export class ArticleFetchError extends Error { constructor( @@ -155,10 +156,13 @@ export async function fetchWebArticle( options.onStage?.('extracting'); const dom = new DOMParser().parseFromString(html, 'text/html'); + // Read before the prepass: the page's own script is the only place its TeX + // shorthands are defined, and extraction drops scripts. + const macros = texMacrosFromDocument(dom); prepareArticleDom(dom); const Defuddle = await loadDefuddle(); const result = await new Defuddle(dom, { url: url.href, markdown: true }).parseAsync(); - const body = polishArticleMarkdown(result.content ?? ''); + const body = expandMarkdownMacros(polishArticleMarkdown(result.content ?? ''), macros); assertReadable(body); return { url: url.href, diff --git a/vite.config.ts b/vite.config.ts index aef449b..c93b778 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -82,6 +82,7 @@ export default defineConfig({ 'src/lib/domain/pdf-markdown.ts', 'src/lib/domain/segmenter.ts', 'src/lib/domain/study-tree.ts', + 'src/lib/domain/tex-macros.ts', 'src/lib/services/web-research.ts', 'src/lib/domain/speech-words.ts', 'src/lib/domain/web-article.ts',