Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions src/lib/domain/segmenter.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,38 @@ describe('narrated construct segmentation', () => {
}
});

it('terminates when a math span opens on the only break token in the window', () => {
// A web article whose figure caption is one long sentence puts the
// comma INSIDE the equation: "…an author sees$, which is why…$". The
// hard cut lands inside that span, so the next window starts exactly
// at the span and its only break token sits at index 0 — where
// `lastIndexOf(token, -1)` used to keep finding the same token for
// ever and spin the main thread. (Regression: importing
// neovand.github.io/Moire/paper hung the tab.)
const prose = 'the heterodyne ratio is drawn live beside the envelope so an author sees ';
const tex =
', which is why they stop short of the two lobes flanking the centres and the fringe system appears in neither layer';
const tail =
' and the prose keeps going past the equation for a good long stretch so that the splitter still has more than one window of text left to walk through after the equation ends';
// Whether the hard cut lands inside the span depends on where the
// equation starts, so sweep the openers that put it near the boundary.
for (let length = 120; length <= MAX_SEGMENT_CHARS; length += 1) {
const opener = prose.repeat(4).slice(0, length);
const paragraph = block({
id: 'b9',
text: `${opener}${tex}${tail}`,
inlines: [{ text: opener }, { text: tex, math: true }, { text: tail }]
});
const segments = segmentBlocks([paragraph]);
expect(segments.length).toBeGreaterThan(1);
const mathEnd = length + tex.length;
for (const segment of segments) {
expect(segment.start > length && segment.start < mathEnd).toBe(false);
expect(segment.end > length && segment.end < mathEnd).toBe(false);
}
}
});

it('leaves sentences with single-letter math as plain word-highlighted text', () => {
const paragraph = block({
id: 'b4',
Expand Down
5 changes: 4 additions & 1 deletion src/lib/domain/segmenter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,11 @@ function splitLongSentence(
let breakAt = -1;
for (const token of ['; ', ', ', ' — ']) {
let index = candidate.lastIndexOf(token);
// `lastIndexOf(token, -1)` searches from 0, not before it, so a
// protected token sitting at index 0 would be found again for
// ever: once there is nothing left of it, there is no break.
while (index >= 0 && spanAt(absoluteStart + cursor + index + 1)) {
index = candidate.lastIndexOf(token, index - 1);
index = index > 0 ? candidate.lastIndexOf(token, index - 1) : -1;
}
breakAt = Math.max(breakAt, index);
}
Expand Down
167 changes: 167 additions & 0 deletions src/lib/domain/tex-macros.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
import { DOMParser } from 'linkedom';
import { describe, expect, it } from 'vitest';
import {
balancedBraces,
expandMarkdownMacros,
expandTexMacros,
texMacrosFromDocument,
texMacrosFromScripts
} from './tex-macros';

function dom(html: string): Document {
return new DOMParser().parseFromString(
`<html><body>${html}</body></html>`,
'text/html'
) as unknown as Document;
}

describe('balancedBraces', () => {
it('spans nested braces and ignores braces inside strings', () => {
expect(balancedBraces('x = {"a": "{{{"} tail', 4)).toBe('{"a": "{{{"}');
expect(balancedBraces('{ a: { b: 1 } }', 0)).toBe('{ a: { b: 1 } }');
});

it('returns null when the group never closes or does not start here', () => {
expect(balancedBraces('{ unterminated', 0)).toBeNull();
expect(balancedBraces('not a group', 0)).toBeNull();
});
});

describe('texMacrosFromScripts', () => {
it('reads a KaTeX auto-render table, backslashes and all', () => {
const script = `renderMathInElement(document.body, {
delimiters: [{ left: '$', right: '$', display: false }],
macros: {"\\\\R":"\\\\mathbb{R}","\\\\idx":"\\\\phi","\\\\Rot":"\\\\mathbf{R}_{#1}"},
strict: false
});`;
expect(texMacrosFromScripts([script])).toEqual({
R: '\\mathbb{R}',
idx: '\\phi',
Rot: '\\mathbf{R}_{#1}'
});
});

it('reads a MathJax table: bare keys, single quotes, [body, arity] values', () => {
const script = `window.MathJax = { tex: { macros: {
RR: '{\\\\bf R}',
bold: ['{\\\\bf #1}', 1],
} } };`;
expect(texMacrosFromScripts([script])).toEqual({ RR: '{\\bf R}', bold: '{\\bf #1}' });
});

it('ignores tables it cannot read and names it could never match back', () => {
expect(texMacrosFromScripts(['macros: { broken'])).toEqual({});
expect(texMacrosFromScripts(['macros: [1, 2]'])).toEqual({});
expect(texMacrosFromScripts([`macros: {"\\\\two words": "x", "\\\\ok": "y"}`])).toEqual({
ok: 'y'
});
});

it('takes the page scripts in order, later definitions winning', () => {
expect(
texMacrosFromScripts([`macros: {"\\\\R": "first"}`, `macros: {"\\\\R": "second"}`])
).toEqual({ R: 'second' });
});
});

describe('texMacrosFromDocument', () => {
it('reads inline scripts and skips external ones', () => {
const document = dom(
`<script src="https://cdn.example.com/katex.js"></script>` +
`<script>renderMathInElement(document.body, { macros: {"\\\\het": "\\\\eta"} });</script>`
);
expect(texMacrosFromDocument(document)).toEqual({ het: '\\eta' });
});

it('is empty for a page with no macro table', () => {
expect(texMacrosFromDocument(dom('<script>console.log(1);</script>'))).toEqual({});
});
});

describe('expandTexMacros', () => {
const macros = {
R: '\\mathbb{R}',
idx: '\\phi',
ph: '\\psi',
Rot: '\\mathbf{R}_{#1}',
pair: '(#1,\\;#2)'
};

it('rewrites bare macros and leaves longer names that merely start alike', () => {
expect(expandTexMacros('p \\in \\R^2', macros)).toBe('p \\in \\mathbb{R}^2');
expect(expandTexMacros('\\idx(p) - \\ph(p)', macros)).toBe('\\phi(p) - \\psi(p)');
// \Rotate is a different command, not \Rot followed by "ate".
expect(expandTexMacros('\\Rotate', macros)).toBe('\\Rotate');
});

it('takes braced groups and single tokens as arguments', () => {
expect(expandTexMacros('\\Rot{\\theta}', macros)).toBe('\\mathbf{R}_{\\theta}');
expect(expandTexMacros('\\Rot n', macros)).toBe('\\mathbf{R}_{n}');
expect(expandTexMacros('\\Rot\\alpha', macros)).toBe('\\mathbf{R}_{\\alpha}');
expect(expandTexMacros('\\pair{a}{b}', macros)).toBe('(a,\\;b)');
});

it('reads a group whose contents carry primes and escaped braces', () => {
expect(expandTexMacros("\\Rot{-n'\\theta}", macros)).toBe("\\mathbf{R}_{-n'\\theta}");
expect(expandTexMacros('\\Rot{\\{a\\}}', macros)).toBe('\\mathbf{R}_{\\{a\\}}');
});

it('expands macros written in terms of other macros', () => {
expect(expandTexMacros('\\field', { ...macros, field: '\\R \\times \\R' })).toBe(
'\\mathbb{R} \\times \\mathbb{R}'
);
});

it('leaves a macro alone when its arguments are not there', () => {
expect(expandTexMacros('\\Rot', macros)).toBe('\\Rot');
expect(expandTexMacros('\\Rot{unterminated', macros)).toBe('\\Rot{unterminated');
});

it('terminates on a self-referential table', () => {
expect(expandTexMacros('\\loop', { loop: '\\loop' })).toBe('\\loop');
expect(expandTexMacros('\\fan', { fan: '\\fan\\fan' }).length).toBeGreaterThan(0);
});

it('passes through text with no macros to expand', () => {
expect(expandTexMacros('plain words', macros)).toBe('plain words');
expect(expandTexMacros('\\alpha + \\beta', {})).toBe('\\alpha + \\beta');
});
});

describe('expandMarkdownMacros', () => {
const macros = { het: '\\eta', R: '\\mathbb{R}' };

it('expands inside every maths delimiter family', () => {
expect(expandMarkdownMacros('ratio $\\het$ here', macros)).toBe('ratio $\\eta$ here');
expect(expandMarkdownMacros('$$\n\\het \\in \\R\n$$', macros)).toBe(
'$$\n\\eta \\in \\mathbb{R}\n$$'
);
expect(expandMarkdownMacros('\\[\\het\\]', macros)).toBe('\\[\\eta\\]');
expect(expandMarkdownMacros('\\(\\het\\)', macros)).toBe('\\(\\eta\\)');
});

it('follows inline maths across a wrapped line but not past a blank one', () => {
expect(expandMarkdownMacros('lines: $\\het =\n\\R/s$ here', macros)).toBe(
'lines: $\\eta =\n\\mathbb{R}/s$ here'
);
// An unbalanced dollar must not swallow the next paragraph's macros.
expect(expandMarkdownMacros('costs $5\n\nprose \\het and $\\het$', macros)).toBe(
'costs $5\n\nprose \\het and $\\eta$'
);
});

it('leaves prose and fenced code untouched', () => {
expect(expandMarkdownMacros('a path C:\\het and $\\het$', macros)).toBe(
'a path C:\\het and $\\eta$'
);
const fenced = '```tex\n$\\het$\n```\n\nand $\\het$ in prose';
expect(expandMarkdownMacros(fenced, macros)).toBe(
'```tex\n$\\het$\n```\n\nand $\\eta$ in prose'
);
});

it('is a no-op without macros or without backslashes', () => {
expect(expandMarkdownMacros('$\\het$', {})).toBe('$\\het$');
expect(expandMarkdownMacros('no maths here', macros)).toBe('no maths here');
});
});
Loading