From 4005b767a9e409cad00d538172f74319af5215af Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 22:56:09 -0500 Subject: [PATCH 01/15] feat: place spoken passages onto original PDF coordinates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader's passages come from the markdown LiteParse extracted; a native page view has to draw the PDF itself. Reconciling the two is the whole problem, and LiteParse's own word boxes (`emitWordBoxes`) are the bridge — both sides descend from the same glyphs. Placement is a monotonic affine-gap alignment of the page's words against the words of the passages anchored to it, with a second pass over whatever region a run of failures was bracketed into. Measured on "Attention Is All You Need": 90% of passages placed, 82% of words boxed individually, under 70ms for the densest page. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 + src/lib/domain/pdf-layout.spec.ts | 341 ++++++++++++++++ src/lib/domain/pdf-layout.ts | 620 ++++++++++++++++++++++++++++++ vite.config.ts | 1 + 4 files changed, 963 insertions(+) create mode 100644 src/lib/domain/pdf-layout.spec.ts create mode 100644 src/lib/domain/pdf-layout.ts diff --git a/.gitignore b/.gitignore index 6c5f3c9..e503f4a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ skills-lock.json # Local test fixtures (user-provided, not committed) test_documents/ +.probe/ diff --git a/src/lib/domain/pdf-layout.spec.ts b/src/lib/domain/pdf-layout.spec.ts new file mode 100644 index 0000000..cd69004 --- /dev/null +++ b/src/lib/domain/pdf-layout.spec.ts @@ -0,0 +1,341 @@ +import { describe, expect, it } from 'vitest'; +import { + alignWordStreams, + chunkText, + matchKey, + mergeWordRects, + pageWordBoxes, + placeSegments, + segmentsForPage, + type PageWordBox, + type PlaceableSegment +} from './pdf-layout'; +import type { SpeechSegment } from './types'; + +/** Word boxes for a run of words laid out on one line, 10pt apart per + * character — enough geometry for the merge and hit-test rules to bite. */ +function line(text: string, y: number, startX = 0): PageWordBox[] { + const boxes: PageWordBox[] = []; + let x = startX; + for (const word of text.split(' ')) { + boxes.push({ text: word, x, y, width: word.length * 10, height: 12 }); + x += word.length * 10 + 5; + } + return boxes; +} + +function words(text: string): Array<{ start: number; end: number }> { + return [...text.matchAll(/\S+/g)].map((match) => ({ + start: match.index ?? 0, + end: (match.index ?? 0) + match[0].length + })); +} + +function placeable(id: string, text: string, extra: Partial = {}) { + return { id, text, words: words(text), page: 1, ...extra }; +} + +describe('matchKey', () => { + it('folds the differences between a page and its extraction', () => { + expect(matchKey('Attention,')).toBe('attention'); + // The page sets a ligature where the markdown carries two letters. + expect(matchKey('figures')).toBe(matchKey('figures')); + expect(matchKey('Café')).toBe('cafe'); + expect(matchKey('“quoted”')).toBe('quoted'); + }); + + it('is empty for text carrying no letters or digits', () => { + expect(matchKey('—')).toBe(''); + expect(matchKey('·')).toBe(''); + }); +}); + +describe('pageWordBoxes', () => { + it('drops rotated stamps, which are not prose', () => { + const boxes = pageWordBoxes([ + { text: 'arXiv:1706', x: 14, y: 380, width: 22, height: 160, rotation: 270 }, + { text: 'Abstract', x: 100, y: 70, width: 50, height: 12, rotation: 0 } + ]); + expect(boxes.map((box) => box.text)).toEqual(['Abstract']); + }); + + it('falls back to the item box when a parse emitted no word boxes', () => { + const boxes = pageWordBoxes([ + { text: 'one two', x: 10, y: 20, width: 60, height: 12 }, + { text: ' ', x: 10, y: 40, width: 5, height: 12 } + ]); + expect(boxes).toEqual([{ text: 'one two', x: 10, y: 20, width: 60, height: 12 }]); + }); + + it('prefers word boxes when the parse emitted them', () => { + const boxes = pageWordBoxes([ + { + text: 'one two', + x: 10, + y: 20, + width: 60, + height: 12, + words: [ + { text: 'one', x: 10, y: 20, width: 25, height: 12 }, + { text: 'two', x: 40, y: 20, width: 25, height: 12 } + ] + } + ]); + expect(boxes.map((box) => box.text)).toEqual(['one', 'two']); + }); +}); + +describe('chunkText', () => { + it('keys whitespace-separated chunks and remembers where they came from', () => { + expect(chunkText('The Transformer, again')).toEqual([ + { key: 'the', start: 0, end: 3 }, + { key: 'transformer', start: 4, end: 16 }, + { key: 'again', start: 17, end: 22 } + ]); + }); + + it('drops chunks that carry no evidence', () => { + expect(chunkText('a — b').map((chunk) => chunk.key)).toEqual(['a', 'b']); + }); +}); + +describe('alignWordStreams', () => { + it('maps each expected word onto the page word it came from', () => { + const alignment = alignWordStreams( + ['the', 'dominant', 'sequence', 'transduction', 'models'], + ['dominant', 'sequence', 'transduction'] + ); + expect([...alignment]).toEqual([1, 2, 3]); + }); + + it('skips page words the spoken layer never says', () => { + const alignment = alignWordStreams( + ['figure', '1', 'the', 'transformer', 'model', 'architecture'], + ['the', 'transformer'] + ); + expect([...alignment]).toEqual([2, 3]); + }); + + it('reports nothing for expected words the page does not have', () => { + const alignment = alignWordStreams(['alpha', 'gamma'], ['alpha', 'beta', 'gamma']); + expect([...alignment]).toEqual([0, -1, 1]); + }); + + it('accepts the fragment a hyphenated line break leaves behind', () => { + const alignment = alignWordStreams( + ['achieves', 'englishto', 'translation'], + ['achieves', 'englishtogerman', 'translation'] + ); + expect([...alignment]).toEqual([0, 1, 2]); + }); + + it('resumes after a long stretch that belongs to only one side', () => { + // The page's table, which the spoken layer narrates in its own words — + // charged per word, this gap would end the alignment here. + const page = [ + 'opening', + 'sentence', + ...Array.from({ length: 60 }, (_, index) => `cell${index}`), + 'closing', + 'sentence' + ]; + const alignment = alignWordStreams(page, ['opening', 'sentence', 'closing', 'sentence']); + expect([...alignment]).toEqual([0, 1, 62, 63]); + }); + + it('never reorders: a repeated phrase matches the run in sequence', () => { + const alignment = alignWordStreams(['a', 'b', 'a', 'b'], ['a', 'b']); + for (let index = 1; index < alignment.length; index += 1) { + expect(alignment[index]).toBeGreaterThan(alignment[index - 1]); + } + }); + + it('gives up rather than guessing on an oversized page', () => { + const huge = Array.from({ length: 2100 }, (_, index) => `w${index}`); + expect([...alignWordStreams(huge, huge)].every((value) => value === -1)).toBe(true); + }); +}); + +describe('mergeWordRects', () => { + it('merges a run of boxes on one line into a single rectangle', () => { + const boxes = line('one two three', 100); + const last = boxes[boxes.length - 1]; + expect(mergeWordRects(boxes)).toEqual([ + { x: 0, y: 100, width: last.x + last.width, height: 12 } + ]); + }); + + it('starts a new rectangle on the next line', () => { + const merged = mergeWordRects([...line('one two', 100), ...line('three', 120)]); + expect(merged).toHaveLength(2); + expect(merged[1]).toEqual({ x: 0, y: 120, width: 50, height: 12 }); + }); + + it('breaks at a column jump rather than sweeping across the gutter', () => { + const merged = mergeWordRects([ + { text: 'left', x: 40, y: 100, width: 40, height: 12 }, + { text: 'right', x: 300, y: 100, width: 40, height: 12 }, + { text: 'wrapped', x: 40, y: 100, width: 40, height: 12 } + ]); + expect(merged).toHaveLength(2); + }); +}); + +describe('placeSegments', () => { + const page = [ + ...line('The dominant sequence transduction models are based on', 100), + ...line('complex recurrent or convolutional neural networks.', 120), + ...line('We propose a new simple network architecture.', 140) + ]; + + it('places a passage over the words it was extracted from', () => { + const [placement] = placeSegments( + page, + [placeable('s1', 'We propose a new simple network architecture.')], + 1 + ); + expect(placement.segmentId).toBe('s1'); + expect(placement.coverage).toBe(1); + const third = line('We propose a new simple network architecture.', 140); + const last = third[third.length - 1]; + expect(placement.rects).toEqual([{ x: 0, y: 140, width: last.x + last.width, height: 12 }]); + }); + + it('boxes each word of the passage separately', () => { + const [placement] = placeSegments(page, [placeable('s1', 'We propose a new')], 1); + expect(placement.wordRects.map((rect) => rect?.x)).toEqual( + line('We propose a new', 140).map((box) => box.x) + ); + }); + + it('leaves a hole for a word the page does not carry', () => { + const [placement] = placeSegments(page, [placeable('s1', 'We hereby propose a new')], 1); + expect(placement.wordRects[1]).toBeUndefined(); + expect(placement.wordRects[0]).toBeDefined(); + expect(placement.wordRects[2]).toBeDefined(); + }); + + it('refuses a passage that only brushed the page', () => { + expect( + placeSegments(page, [placeable('s1', 'Entirely unrelated prose about a cat')], 1) + ).toEqual([]); + }); + + it('keeps passages in document order across the page', () => { + const placements = placeSegments( + page, + [ + placeable('s1', 'The dominant sequence transduction models'), + placeable('s2', 'We propose a new simple network architecture.') + ], + 1 + ); + expect(placements.map((placement) => placement.segmentId)).toEqual(['s1', 's2']); + expect(placements[0].rects[0].y).toBeLessThan(placements[1].rects[0].y); + }); + + it('looks for what the page prints when that differs from what is spoken', () => { + const table = [ + ...line('Layer Type Complexity per Layer', 200), + ...line('Self-Attention Onnd Restricted', 220) + ]; + // The row is narrated with its header labels folded in; only its cells + // are on the page. + const spoken = 'Layer Type: Self-Attention. Complexity per Layer: Onnd.'; + // Spoken as-is, the header labels pull the row up onto the header line. + const [narrated] = placeSegments(table, [placeable('row', spoken)], 1); + expect(narrated.rects[0].y).toBe(200); + const [placement] = placeSegments( + table, + [placeable('row', spoken, { matchText: 'Self-Attention Onnd Restricted' })], + 1 + ); + expect(placement.rects[0].y).toBe(220); + // The words being spoken are not the words on the page. + expect(placement.wordRects.every((rect) => rect === undefined)).toBe(true); + }); + + it('finds a run of rows once its neighbours have staked out the region', () => { + const withTable = [ + ...line('Opening sentence above the table', 100), + ...line('Alpha ninety Beta eighty', 130), + ...line('Gamma seventy Delta sixty', 150), + ...line('Closing sentence below the table', 180) + ]; + const placements = placeSegments( + withTable, + [ + placeable('s1', 'Opening sentence above the table'), + placeable('r1', 'Alpha ninety Beta eighty'), + placeable('r2', 'Gamma seventy Delta sixty'), + placeable('s2', 'Closing sentence below the table') + ], + 1 + ); + expect(placements.map((placement) => placement.segmentId)).toEqual(['s1', 'r1', 'r2', 's2']); + expect(placements[1].rects[0].y).toBe(130); + expect(placements[2].rects[0].y).toBe(150); + }); + + it('has nothing to say about a page with no words', () => { + expect(placeSegments([], [placeable('s1', 'anything')], 1)).toEqual([]); + }); +}); + +describe('segmentsForPage', () => { + function segment(id: string, text: string, page: number, blockId = 'b1'): SpeechSegment { + return { + id, + blockId, + text, + normalizedText: text, + start: 0, + end: text.length, + words: words(text), + estimatedDuration: 1, + anchor: { page } + }; + } + + it('is empty when nothing is anchored to the page', () => { + expect(segmentsForPage([segment('s1', 'one', 1)], 4)).toEqual([]); + }); + + it('reaches into the neighbouring pages for passages that straddle the break', () => { + const segments = [ + segment('before', 'a sentence ending the previous page', 1), + segment('own', 'a sentence of its own', 2), + segment('after', 'a sentence opening the next page', 3) + ]; + expect(segmentsForPage(segments, 2).map((entry) => entry.id)).toEqual([ + 'before', + 'own', + 'after' + ]); + }); + + it('stops spilling once the word budget runs out', () => { + const long = Array.from({ length: 80 }, (_, index) => + segment(`p1-${index}`, 'ten words here that fill up the spill budget fast', 1) + ); + const ids = segmentsForPage([...long, segment('own', 'the page itself', 2)], 2).map( + (entry) => entry.id + ); + expect(ids).toContain('own'); + expect(ids).not.toContain('p1-0'); + }); + + it('carries the construct source through as what to look for on the page', () => { + const row = segment('row', 'Layer Type: Self-Attention.', 2); + row.start = 4; + row.end = 22; + const placeables = segmentsForPage([row], 2, new Map([['b1', 'xxx Self-Attention Onnd xxx']])); + expect(placeables[0].matchText).toBe('Self-Attention Onn'); + }); + + it('leaves plain prose alone, so its words keep their boxes', () => { + const prose = segment('prose', 'a sentence of its own', 2); + const placeables = segmentsForPage([prose], 2, new Map([['b1', 'a sentence of its own']])); + expect(placeables[0].matchText).toBeUndefined(); + }); +}); diff --git a/src/lib/domain/pdf-layout.ts b/src/lib/domain/pdf-layout.ts new file mode 100644 index 0000000..81c638a --- /dev/null +++ b/src/lib/domain/pdf-layout.ts @@ -0,0 +1,620 @@ +import type { SpeechSegment } from './types'; + +/** + * Placing the spoken layer back onto the original page. + * + * The reader's passages come from the markdown LiteParse extracted; the page + * view draws the PDF itself. To highlight a passage where it actually sits on + * paper, the two have to be reconciled — and the only trustworthy bridge is + * that LiteParse can emit a box for every word it read (`emitWordBoxes`), so + * both sides descend from the same glyphs. + * + * Reconciliation is a monotonic sequence alignment: the page's words in + * reading order against the words of the passages anchored to that page. + * Content the spoken layer skips (equations, figure labels, running heads) + * falls out as gaps, and so does markdown the page renders differently + * (table pipes, heading hashes) — neither derails the words on either side of + * it. Nothing here touches the DOM or pdf.js: it is pure geometry over two + * token streams, so the hard part is testable. + * + * Coordinates throughout are PDF points with a **top-left** origin, matching + * LiteParse's boxes and the CSS the overlay ends up writing. + */ + +export interface PageWordBox { + text: string; + x: number; + y: number; + width: number; + height: number; +} + +/** The slice of LiteParse's TextItem this module needs. */ +export interface PageTextItem { + text: string; + x: number; + y: number; + width: number; + height: number; + /** Degrees off horizontal; rotated stamps (arXiv spines) are not prose. */ + rotation?: number; + words?: PageWordBox[]; +} + +export interface PageRect { + x: number; + y: number; + width: number; + height: number; +} + +/** Where one passage landed on one page. */ +export interface SegmentPlacement { + segmentId: string; + page: number; + /** Line-merged rectangles covering the passage — what the overlay paints. */ + rects: PageRect[]; + /** Index-aligned to `SpeechSegment.words`; a hole is a word that found no + * box (a substituted equation reading, a word the page hyphenated away). */ + wordRects: Array; + /** Share of the passage's words that found a box, 0–1. */ + coverage: number; +} + +/** A passage reduced to what placement needs, so tests need no full segment. */ +export interface PlaceableSegment { + id: string; + text: string; + words: Array<{ start: number; end: number }>; + page: number; + /** + * What to look for on the page, when that differs from what is spoken. A + * table row is narrated as "Layer Type: Self-Attention. Complexity per + * Layer: …" — prose assembled from the header labels, which the page never + * printed in that form. Its cells, on the other hand, are right there. + * Passages placed this way get no per-word boxes: the words being spoken + * are not the words on the page, and only the region is meaningful. + */ + matchText?: string; +} + +/** + * The comparison form of a word: compatibility-decomposed (so a `fi` ligature + * on the page meets the `fi` in the markdown), unaccented, lowercased, and + * stripped of everything that is not a letter or a digit. Punctuation must go + * — the page hyphenates and the markdown does not, and quotes differ on both + * sides of every extraction. + */ +export function matchKey(text: string): string { + return text + .normalize('NFKD') + .replace(/\p{M}+/gu, '') + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, ''); +} + +/** Every word LiteParse read on a page, in its reading order. Items without + * word boxes (older parses, OCR text) contribute their whole box, which still + * highlights at line granularity. */ +export function pageWordBoxes(items: PageTextItem[]): PageWordBox[] { + const boxes: PageWordBox[] = []; + for (const item of items) { + if (item.rotation !== undefined && Math.abs(item.rotation) > 1) continue; + if (item.words?.length) { + for (const word of item.words) if (word.text.trim()) boxes.push(word); + } else if (item.text.trim()) { + boxes.push({ + text: item.text, + x: item.x, + y: item.y, + width: item.width, + height: item.height + }); + } + } + return boxes; +} + +interface Chunk { + key: string; + start: number; + end: number; +} + +/** Whitespace-separated chunks with their character ranges, keyed for + * comparison. Splitting on whitespace (not on letter runs) keeps both streams + * tokenized the same way, so `don't` stays one token against the page's one + * box for it. Chunks that key to nothing — a lone bullet, a stray dash — are + * dropped: they carry no evidence and would only invite false matches. */ +export function chunkText(text: string): Chunk[] { + const chunks: Chunk[] = []; + for (const match of text.matchAll(/\S+/gu)) { + const start = match.index ?? 0; + const key = matchKey(match[0]); + if (key) chunks.push({ key, start, end: start + match[0].length }); + } + return chunks; +} + +/** Cheap enough to be exact for a page; the guard only exists so a pathological + * page (a word list, a dense table) cannot lock the main thread. */ +const MAX_ALIGNMENT_CELLS = 4_000_000; + +const SCORE_MATCH = 1; +/** A hyphenated break leaves the page holding a fragment of the markdown's + * word (`englishto` for `englishtogerman`). Scoring that as most of a match + * keeps the alignment on the diagonal, where treating it as a mismatch would + * derail it. */ +const SCORE_PARTIAL = 0.5; +const SCORE_MISMATCH = -1; +/** + * Skips are priced as one decision, not per word: opening a run of unmatched + * words costs, continuing it barely does. This is what lets the two streams + * survive each other's bulk. A page carries equations, axis labels and + * running heads the spoken layer never says; the spoken layer carries whole + * paragraphs the page never printed (a table's narration, "A table with + * columns: …"). Charged per word, one such stretch would outweigh every match + * after it and the alignment would simply stop there — which is exactly what + * a flat gap penalty did: everything below Table 1 on a page went unplaced. + * + * Opening a gap also has to stay cheaper than a single match is worth, or the + * free ends win instead: an alignment that starts late and matches three + * words cleanly would outscore one that matches five with a skip in the + * middle, and the two words before the skip would be dropped for no reason. + */ +const SCORE_GAP_OPEN = -0.6; +const SCORE_GAP_EXTEND = -0.02; + +function pairScore(left: string, right: string): number { + if (left === right) return SCORE_MATCH; + if (left.length >= 4 && right.length >= 4 && (left.startsWith(right) || right.startsWith(left))) { + return SCORE_PARTIAL; + } + return SCORE_MISMATCH; +} + +/** Traceback states: a pair, an unmatched expected word, an unmatched page + * word. */ +const PAIRED = 0; +const EXPECTED_GAP = 1; +const PAGE_GAP = 2; + +/** + * Monotonic alignment of the expected words (the spoken layer's) against the + * page's, returning the page index each expected word landed on, or -1. + * + * Gotoh's affine-gap alignment with both ends free. Free ends because neither + * stream has to be consumed whole — the expected stream deliberately carries + * a little of the neighbouring pages. Affine gaps because both streams are + * full of material the other lacks, and skipping it has to stay cheap enough + * that the alignment resumes afterwards (see the gap constants above). + * + * Only pairs that actually agree are reported: the path threads through + * mismatches to stay on course, but a mismatch is not evidence of where a + * word sits. + */ +export function alignWordStreams(pageKeys: string[], expectedKeys: string[]): Int32Array { + const rows = expectedKeys.length; + const columns = pageKeys.length; + const result = new Int32Array(rows).fill(-1); + if (!rows || !columns || (rows + 1) * (columns + 1) > MAX_ALIGNMENT_CELLS) return result; + + const width = columns + 1; + // Scores roll row by row; only the traceback needs the whole grid. + let pairedRow = new Float32Array(width); + let expectedGapRow = new Float32Array(width); + let pageGapRow = new Float32Array(width); + let nextPaired = new Float32Array(width); + let nextExpectedGap = new Float32Array(width); + let nextPageGap = new Float32Array(width); + // Which state each cell was reached from: for a pair, the predecessor + // state; for a gap, whether it opened (0) or extended (1). + const fromPaired = new Uint8Array((rows + 1) * width); + const fromExpectedGap = new Uint8Array((rows + 1) * width); + const fromPageGap = new Uint8Array((rows + 1) * width); + + let bestScore = 0; + let bestRow = 0; + let bestColumn = 0; + let bestState = PAIRED; + + for (let row = 1; row <= rows; row += 1) { + // Row 0 and column 0 stay at zero: an alignment may start anywhere in + // either stream without paying for the prefix it skipped. + nextPaired[0] = 0; + nextExpectedGap[0] = 0; + nextPageGap[0] = 0; + const base = row * width; + for (let column = 1; column <= columns; column += 1) { + const previous = Math.max( + pairedRow[column - 1], + expectedGapRow[column - 1], + pageGapRow[column - 1] + ); + nextPaired[column] = previous + pairScore(expectedKeys[row - 1], pageKeys[column - 1]); + fromPaired[base + column] = + previous === pairedRow[column - 1] + ? PAIRED + : previous === expectedGapRow[column - 1] + ? EXPECTED_GAP + : PAGE_GAP; + + const openExpected = pairedRow[column] + SCORE_GAP_OPEN + SCORE_GAP_EXTEND; + const extendExpected = expectedGapRow[column] + SCORE_GAP_EXTEND; + nextExpectedGap[column] = Math.max(openExpected, extendExpected); + fromExpectedGap[base + column] = extendExpected > openExpected ? 1 : 0; + + const openPage = nextPaired[column - 1] + SCORE_GAP_OPEN + SCORE_GAP_EXTEND; + const extendPage = nextPageGap[column - 1] + SCORE_GAP_EXTEND; + nextPageGap[column] = Math.max(openPage, extendPage); + fromPageGap[base + column] = extendPage > openPage ? 1 : 0; + + // Ending is free as well, so the alignment stops at its best point on + // either final edge rather than being dragged into the far corner. + if ((row === rows || column === columns) && nextPaired[column] > bestScore) { + bestScore = nextPaired[column]; + bestRow = row; + bestColumn = column; + bestState = PAIRED; + } + } + [pairedRow, nextPaired] = [nextPaired, pairedRow]; + [expectedGapRow, nextExpectedGap] = [nextExpectedGap, expectedGapRow]; + [pageGapRow, nextPageGap] = [nextPageGap, pageGapRow]; + } + + let row = bestRow; + let column = bestColumn; + let state = bestState; + while (row > 0 && column > 0) { + const cell = row * width + column; + if (state === PAIRED) { + if (pairScore(expectedKeys[row - 1], pageKeys[column - 1]) > 0) { + result[row - 1] = column - 1; + } + state = fromPaired[cell]; + row -= 1; + column -= 1; + } else if (state === EXPECTED_GAP) { + state = fromExpectedGap[cell] === 1 ? EXPECTED_GAP : PAIRED; + row -= 1; + } else { + state = fromPageGap[cell] === 1 ? PAGE_GAP : PAIRED; + column -= 1; + } + } + return result; +} + +function unionRect(rects: PageRect[]): PageRect { + let left = Infinity; + let top = Infinity; + let right = -Infinity; + let bottom = -Infinity; + for (const rect of rects) { + left = Math.min(left, rect.x); + top = Math.min(top, rect.y); + right = Math.max(right, rect.x + rect.width); + bottom = Math.max(bottom, rect.y + rect.height); + } + return { x: left, y: top, width: right - left, height: bottom - top }; +} + +/** + * Collapse a passage's word boxes into one rectangle per line, in reading + * order. Boxes join a run while they sit on the same line (vertical centres + * within half a line height) and do not jump backwards across the page — + * a backwards jump is a column break, which has to start a new rectangle or + * the highlight would sweep across the gutter. + */ +export function mergeWordRects(boxes: PageRect[]): PageRect[] { + const merged: PageRect[] = []; + let run: PageRect[] = []; + const flush = () => { + if (run.length) merged.push(unionRect(run)); + run = []; + }; + for (const box of boxes) { + const previous = run[run.length - 1]; + if (previous) { + const sameLine = + Math.abs(box.y + box.height / 2 - (previous.y + previous.height / 2)) < + Math.max(box.height, previous.height) / 2; + if (!sameLine || box.x + box.width < previous.x) flush(); + } + run.push(box); + } + flush(); + return merged; +} + +/** + * Below this, a placement is guesswork: a couple of stray words matched + * somewhere on the page while the passage itself is elsewhere. Painting those + * would put the highlight in the wrong place, which is worse than not + * painting it. The refinement pass is allowed to be less strict, because by + * then the passage is boxed in by its placed neighbours and cannot land far + * from where it belongs. + */ +const MIN_COVERAGE = 0.35; +const MIN_REFINED_COVERAGE = 0.2; +/** One refinement of each unclaimed gap is enough in practice; the limit is + * only here so a pathological page cannot recurse indefinitely. */ +const MAX_REFINEMENT_DEPTH = 2; + +interface ExpectedChunk extends Chunk { + segment: number; +} + +/** + * Align a slice of the expected stream against a slice of the page, then look + * at what went unplaced. + * + * A run of passages that all failed together usually failed for one reason: + * the page renders them in a form the spoken layer rewrote — a table, whose + * rows are narrated as "Layer Type: Self-Attention. Complexity per Layer: …" + * against a page holding a grid of bare cells. Whole-page alignment cannot + * find those rows among a thousand competing words, but the run is bracketed + * by passages that *did* place, and the page words between those two brackets + * are exactly the table. Aligning the run against that much smaller stretch + * usually lands it. + */ +function assignRange( + boxes: PageWordBox[], + pageKeys: string[], + expected: ExpectedChunk[], + matches: Array, + range: { boxLow: number; boxHigh: number; chunkLow: number; chunkHigh: number }, + depth: number +): void { + const { boxLow, boxHigh, chunkLow, chunkHigh } = range; + if (boxLow > boxHigh || chunkLow > chunkHigh) return; + const alignment = alignWordStreams( + pageKeys.slice(boxLow, boxHigh + 1), + expected.slice(chunkLow, chunkHigh + 1).map((chunk) => chunk.key) + ); + for (let index = 0; index < alignment.length; index += 1) { + const box = alignment[index]; + if (box >= 0) matches[chunkLow + index].push(boxLow + box); + } + + const minimum = depth === 0 ? MIN_COVERAGE : MIN_REFINED_COVERAGE; + // Which of the passages in this range came out placed, so the gaps between + // them can be retried against the page they must lie on. + const segmentLow = expected[chunkLow].segment; + const segmentHigh = expected[chunkHigh].segment; + const placed = new Map(); + for (let segment = segmentLow; segment <= segmentHigh; segment += 1) { + let total = 0; + let first = Infinity; + let last = -Infinity; + for (let index = chunkLow; index <= chunkHigh; index += 1) { + if (expected[index].segment !== segment) continue; + total += 1; + const box = matches[index][matches[index].length - 1]; + if (box === undefined) continue; + first = Math.min(first, box); + last = Math.max(last, box); + } + const covered = countMatched(expected, matches, chunkLow, chunkHigh, segment); + if (total && covered / total >= minimum) placed.set(segment, { first, last }); + } + if (depth >= MAX_REFINEMENT_DEPTH) return; + + let runStart: number | undefined; + for (let segment = segmentLow; segment <= segmentHigh + 1; segment += 1) { + if (segment <= segmentHigh && !placed.has(segment)) { + runStart ??= segment; + continue; + } + if (runStart === undefined) continue; + const runEnd = segment - 1; + const before = previousPlaced(placed, runStart - 1, segmentLow); + const after = nextPlaced(placed, runEnd + 1, segmentHigh); + const nextRange = { + boxLow: before === undefined ? boxLow : before + 1, + boxHigh: after === undefined ? boxHigh : after - 1, + chunkLow: firstChunkOf(expected, chunkLow, chunkHigh, runStart), + chunkHigh: lastChunkOf(expected, chunkLow, chunkHigh, runEnd) + }; + runStart = undefined; + // No narrowing happened: retrying the same range would only repeat this + // pass's answer. + if ( + nextRange.chunkLow === undefined || + nextRange.chunkHigh === undefined || + (nextRange.boxLow === boxLow && nextRange.boxHigh === boxHigh) + ) { + continue; + } + assignRange( + boxes, + pageKeys, + expected, + matches, + nextRange as { boxLow: number; boxHigh: number; chunkLow: number; chunkHigh: number }, + depth + 1 + ); + } +} + +function countMatched( + expected: ExpectedChunk[], + matches: Array, + low: number, + high: number, + segment: number +): number { + let count = 0; + for (let index = low; index <= high; index += 1) { + if (expected[index].segment === segment && matches[index].length) count += 1; + } + return count; +} + +function previousPlaced( + placed: Map, + from: number, + low: number +): number | undefined { + for (let segment = from; segment >= low; segment -= 1) { + const entry = placed.get(segment); + if (entry) return entry.last; + } + return undefined; +} + +function nextPlaced( + placed: Map, + from: number, + high: number +): number | undefined { + for (let segment = from; segment <= high; segment += 1) { + const entry = placed.get(segment); + if (entry) return entry.first; + } + return undefined; +} + +function firstChunkOf( + expected: ExpectedChunk[], + low: number, + high: number, + segment: number +): number | undefined { + for (let index = low; index <= high; index += 1) { + if (expected[index].segment === segment) return index; + } + return undefined; +} + +function lastChunkOf( + expected: ExpectedChunk[], + low: number, + high: number, + segment: number +): number | undefined { + for (let index = high; index >= low; index -= 1) { + if (expected[index].segment === segment) return index; + } + return undefined; +} + +/** + * Place every passage anchored to a page onto that page's words. + * + * `segments` should be given in document order and may include a little of the + * neighbouring pages: a paragraph that starts on one page and finishes on the + * next is one passage, and the alignment's free end gaps let the part that + * belongs elsewhere go unmatched rather than dragging the rest off course. + */ +export function placeSegments( + boxes: PageWordBox[], + segments: PlaceableSegment[], + page: number +): SegmentPlacement[] { + const pageKeys = boxes.map((box) => matchKey(box.text)); + const expected: ExpectedChunk[] = []; + segments.forEach((segment, index) => { + for (const chunk of chunkText(segment.matchText ?? segment.text)) { + expected.push({ ...chunk, segment: index }); + } + }); + if (!expected.length || !pageKeys.length) return []; + const matches: Array = expected.map(() => []); + assignRange( + boxes, + pageKeys, + expected, + matches, + { boxLow: 0, boxHigh: pageKeys.length - 1, chunkLow: 0, chunkHigh: expected.length - 1 }, + 0 + ); + + const placements: SegmentPlacement[] = []; + segments.forEach((segment, index) => { + const matched: Array<{ chunk: Chunk; box: PageWordBox }> = []; + let total = 0; + expected.forEach((chunk, chunkIndex) => { + if (chunk.segment !== index) return; + total += 1; + // The last assignment wins: a refinement pass ran against a narrower + // stretch of the page and knows better than the whole-page sweep. + const box = matches[chunkIndex][matches[chunkIndex].length - 1]; + if (box !== undefined) matched.push({ chunk, box: boxes[box] }); + }); + const coverage = total ? matched.length / total : 0; + if (!matched.length || coverage < MIN_REFINED_COVERAGE) return; + matched.sort((left, right) => left.chunk.start - right.chunk.start); + const wordRects = segment.matchText + ? segment.words.map(() => undefined) + : segment.words.map((word) => { + const overlapping = matched + .filter(({ chunk }) => chunk.start < word.end && chunk.end > word.start) + .map(({ box }) => box); + return overlapping.length ? unionRect(overlapping) : undefined; + }); + placements.push({ + segmentId: segment.id, + page, + rects: mergeWordRects(matched.map(({ box }) => box)), + wordRects, + coverage + }); + }); + return placements; +} + +/** How much of each neighbouring page's passages to align alongside a page's + * own, in words. A paragraph that straddles a page break is one passage list + * anchored to the earlier page, so the later page has to look back far enough + * to find the sentences that actually landed on it. */ +const SPILL_WORDS = 400; + +/** The passages to align against one page: everything anchored to it, plus a + * stretch of each neighbour so a paragraph straddling the page break is placed + * from whichever side is looking. + * + * `blockText` (block id → block text) is what lets a narrated construct be + * looked for as it appears on paper rather than as it is spoken: a segment's + * `start`/`end` index its block's text, so that slice is the construct's own + * source — a table row's cells, an equation's characters — while + * `segment.text` is the reading. Without it, tables and equations simply go + * unplaced. + */ +export function segmentsForPage( + segments: SpeechSegment[], + page: number, + blockText?: ReadonlyMap, + spillWords = SPILL_WORDS +): PlaceableSegment[] { + let first = -1; + let last = -1; + segments.forEach((segment, index) => { + if (segment.anchor.page !== page) return; + if (first < 0) first = index; + last = index; + }); + if (first < 0) return []; + let from = first; + for (let budget = spillWords; from > 0 && budget > 0; from -= 1) { + budget -= segments[from - 1].words.length; + } + let to = last; + for (let budget = spillWords; to < segments.length - 1 && budget > 0; to += 1) { + budget -= segments[to + 1].words.length; + } + const placeable: PlaceableSegment[] = []; + for (let index = from; index <= to; index += 1) { + const segment = segments[index]; + const source = blockText?.get(segment.blockId)?.slice(segment.start, segment.end); + placeable.push({ + id: segment.id, + text: segment.text, + words: segment.words, + page: segment.anchor.page ?? page, + ...(source && source !== segment.text ? { matchText: source } : {}) + }); + } + return placeable; +} diff --git a/vite.config.ts b/vite.config.ts index 09f400c..d02cc67 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -77,6 +77,7 @@ export default defineConfig({ 'src/lib/domain/importers.ts', 'src/lib/domain/model-catalog.ts', 'src/lib/domain/pages.ts', + 'src/lib/domain/pdf-layout.ts', 'src/lib/domain/pdf-markdown.ts', 'src/lib/domain/segmenter.ts', 'src/lib/domain/study-tree.ts', From 4268f24e565f3351fedaab0c1def2b65d35a73c5 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:13:45 -0500 Subject: [PATCH 02/15] feat: read a PDF as its own pages, with the narration painted on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second reader view draws the original PDF and highlights the passage being spoken where it actually sits on the paper, down to the word. The markdown stays underneath as it always was — it is what the voice, the assistant, and the study layer read — so the choice is presentation, not a different import. Geometry is recomputed per page from the stored bytes (LiteParse `targetPages` + `emitWordBoxes`, tens of milliseconds a page) rather than stored per document, as `DocumentPageInfo` anticipated. Pages draw and place lazily as they come near the scrollport and hand their bitmaps back when they leave. Co-Authored-By: Claude Opus 5 --- src/lib/components/PdfPageView.svelte | 393 +++++++++++++++++++++++ src/lib/services/pdf-layout.ts | 191 +++++++++++ src/lib/state/reader-chrome.svelte.ts | 33 ++ src/routes/+layout.svelte | 25 ++ src/routes/read/+page.svelte | 438 ++++++++++++++------------ 5 files changed, 876 insertions(+), 204 deletions(-) create mode 100644 src/lib/components/PdfPageView.svelte create mode 100644 src/lib/services/pdf-layout.ts diff --git a/src/lib/components/PdfPageView.svelte b/src/lib/components/PdfPageView.svelte new file mode 100644 index 0000000..c6ba8e8 --- /dev/null +++ b/src/lib/components/PdfPageView.svelte @@ -0,0 +1,393 @@ + + +
+ {#each sizes as size (size.page)} + {@const scale = pageScale(size.page)} + {@const placed = placements.get(size.page)} +
+ + + +
handleActivate(event, size.page)} + onpointermove={(event) => handleHover(event, size.page)} + onpointerleave={() => (hovered = undefined)} + > + + {#if live.has(size.page)} + + {/if} +
+
+ {/each} +
+ + diff --git a/src/lib/services/pdf-layout.ts b/src/lib/services/pdf-layout.ts new file mode 100644 index 0000000..0b4ee33 --- /dev/null +++ b/src/lib/services/pdf-layout.ts @@ -0,0 +1,191 @@ +import type { NormalizedDocument, SpeechSegment } from '../domain/types'; +import { + pageWordBoxes, + placeSegments, + segmentsForPage, + type PageWordBox, + type SegmentPlacement +} from '../domain/pdf-layout'; +import { getSource } from './repository'; + +/** + * Read-time word geometry for a document's original PDF. + * + * The import already ran LiteParse over these bytes, but deliberately kept + * none of the boxes: storing a rectangle per word for every document in the + * library would dwarf the documents themselves (see `DocumentPageInfo`). + * Re-reading them is cheap when it is asked for by page — LiteParse's + * `targetPages` parses a window in tens of milliseconds — so the page view + * recomputes what it needs as the reader scrolls and keeps it in memory for + * the session. + */ + +export interface PageLayout { + page: number; + /** PDF points, as LiteParse read them — the coordinate space of `boxes`. */ + width: number; + height: number; + boxes: PageWordBox[]; +} + +/** Pages either side of the requested one to parse in the same pass. Reading + * is directional, so the window leans forward. */ +const WINDOW_BEHIND = 1; +const WINDOW_AHEAD = 2; + +let liteparse: Promise | undefined; + +/** The wasm module, initialized once per session. LiteParse's own init guard + * only dedupes after the first init resolves, so the promise is what callers + * share. */ +async function loadLiteparse(): Promise { + liteparse ??= (async () => { + const [glue, wasm] = await Promise.all([ + import('@llamaindex/liteparse-wasm'), + import('@llamaindex/liteparse-wasm/liteparse_wasm_bg.wasm?url') + ]); + await glue.default({ module_or_path: wasm.default }); + return glue; + })(); + return liteparse; +} + +/** + * One document's page geometry, parsed on demand and kept for the session. + * Requests for the same page while a parse is in flight join it rather than + * starting a second one — scrolling asks for the same page many times. + */ +class DocumentLayout { + readonly documentId: string; + #source: Promise; + #pages = new Map(); + #pending = new Map>(); + #placements = new Map>(); + #segments: SpeechSegment[] = []; + + constructor(document: NormalizedDocument) { + this.documentId = document.id; + this.#source = (async () => { + const blob = await getSource(document); + if (!blob) return null; + return new Uint8Array(await blob.arrayBuffer()); + })().catch(() => null); + } + + /** Word boxes for one page, or null when the source is gone (OPFS + * eviction) or LiteParse cannot read it. */ + pageLayout(page: number, pageCount: number): Promise { + const cached = this.#pages.get(page); + if (cached !== undefined) return Promise.resolve(cached); + const inFlight = this.#pending.get(page); + if (inFlight) return inFlight; + const from = Math.max(1, page - WINDOW_BEHIND); + const to = Math.max(from, Math.min(pageCount || page + WINDOW_AHEAD, page + WINDOW_AHEAD)); + const work = this.#parseWindow(from, to).then( + () => this.#pages.get(page) ?? null, + () => null + ); + for (let target = from; target <= to; target += 1) { + if (!this.#pages.has(target)) this.#pending.set(target, work); + } + void work.finally(() => { + for (let target = from; target <= to; target += 1) { + if (this.#pending.get(target) === work) this.#pending.delete(target); + } + }); + return work; + } + + async #parseWindow(from: number, to: number): Promise { + const data = await this.#source; + if (!data) { + for (let page = from; page <= to; page += 1) this.#pages.set(page, null); + return; + } + const glue = await loadLiteparse(); + const parser = new glue.LiteParse({ + ocrEnabled: false, + outputFormat: 'markdown', + // Only the geometry is wanted here; images and links are the import's + // business and decoding them again would dominate the parse. + imageMode: 'off', + extractLinks: false, + skipDiagonalText: true, + emitWordBoxes: true, + quiet: true, + targetPages: from === to ? String(from) : `${from}-${to}` + }); + let parsed: Awaited>; + try { + // LiteParse transfers the buffer to wasm; hand it a copy so the next + // window still has bytes to read. + parsed = await parser.parse(data.slice()); + } finally { + parser.free(); + } + const seen = new Set(); + for (const page of parsed.pages) { + seen.add(page.pageNum); + this.#pages.set(page.pageNum, { + page: page.pageNum, + width: page.width, + height: page.height, + boxes: pageWordBoxes(page.textItems ?? []) + }); + } + // A page the parse skipped has no geometry and never will; recording the + // miss stops every scroll from asking again. + for (let page = from; page <= to; page += 1) if (!seen.has(page)) this.#pages.set(page, null); + } + + /** + * Where each passage sits on one page. Placement is pure but not free + * (tens of milliseconds on a dense page), so results are kept per page and + * dropped wholesale when the passages themselves change — a listening-mode + * switch or a narration swap re-cuts every segment id. + */ + async placements( + page: number, + pageCount: number, + segments: SpeechSegment[], + blockText: ReadonlyMap + ): Promise> { + if (segments !== this.#segments) { + this.#segments = segments; + this.#placements.clear(); + } + const key = String(page); + const cached = this.#placements.get(key); + if (cached) return cached; + const layout = await this.pageLayout(page, pageCount); + const placed = new Map(); + if (layout?.boxes.length) { + const placeable = segmentsForPage(segments, page, blockText); + for (const placement of placeSegments(layout.boxes, placeable, page)) { + placed.set(placement.segmentId, placement); + } + } + // The passages may have been rebound while this page was being parsed; + // caching against a stale set would pin placements for ids nothing uses. + if (segments === this.#segments) this.#placements.set(key, placed); + return placed; + } +} + +let open: DocumentLayout | undefined; + +/** The layout source for a document, kept warm one at a time — opening + * another document releases the previous one, matching the page renderer. */ +export function openPdfLayout(document: NormalizedDocument): DocumentLayout { + if (open?.documentId !== document.id) open = new DocumentLayout(document); + return open; +} + +/** Frees the warm layout source. Safe to call twice or with nothing open. */ +export function releasePdfLayout(documentId?: string): void { + if (!open) return; + if (documentId && open.documentId !== documentId) return; + open = undefined; +} + +export type { DocumentLayout }; diff --git a/src/lib/state/reader-chrome.svelte.ts b/src/lib/state/reader-chrome.svelte.ts index 0ae902c..7734a4d 100644 --- a/src/lib/state/reader-chrome.svelte.ts +++ b/src/lib/state/reader-chrome.svelte.ts @@ -1,11 +1,28 @@ import { DEFAULT_LISTENING_MODE, isListeningMode } from '$lib/domain/listening-modes'; import type { ListeningMode } from '$lib/domain/types'; +/** + * How a document is shown. 'reading' is the reflowed markdown — the only + * option for anything that was never a page. 'page' draws the original PDF + * and paints the spoken passage onto it, for readers who want the paper as + * the authors set it. + */ +export type ReaderView = 'reading' | 'page'; + +export const READER_VIEWS: ReaderView[] = ['reading', 'page']; + +function isReaderView(value: unknown): value is ReaderView { + return READER_VIEWS.includes(value as ReaderView); +} + class ReaderChromeState { /** Contents starts closed — the document is the point. */ outlineOpen = $state(false); menuOpen = $state(false); documentZoom = $state(1); + /** The preferred view, remembered across documents. A document with no + * original pages falls back to reading without changing this. */ + readerView = $state('reading'); /** The listening mode new imports start in. Per-document overrides live on * the document itself and take precedence in the reader. */ defaultListeningMode = $state(DEFAULT_LISTENING_MODE); @@ -30,10 +47,26 @@ class ReaderChromeState { if (Number.isFinite(stored) && stored >= 0.8 && stored <= 1.6) this.documentZoom = stored; const mode = window.localStorage.getItem('voicebook:listening-mode'); if (isListeningMode(mode)) this.defaultListeningMode = mode; + const view = window.localStorage.getItem('voicebook:reader-view'); + if (isReaderView(view)) this.readerView = view; this.assistantCaptions = window.localStorage.getItem('voicebook:assistant-captions') !== '0'; this.spokenChatReplies = window.localStorage.getItem('voicebook:spoken-chat-replies') !== '0'; } + /** Steps to the next view. A cycle rather than a flip, so a third view can + * join without the control changing shape. */ + cycleReaderView(): void { + const next = READER_VIEWS[(READER_VIEWS.indexOf(this.readerView) + 1) % READER_VIEWS.length]; + this.setReaderView(next); + } + + setReaderView(view: ReaderView): void { + this.readerView = view; + if (typeof window !== 'undefined') { + window.localStorage.setItem('voicebook:reader-view', view); + } + } + setAssistantCaptions(on: boolean): void { this.assistantCaptions = on; if (typeof window !== 'undefined') { diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 70124ff..955ff4d 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -9,6 +9,7 @@ import { PanelLeftClose, PanelLeftOpen, + BookOpenText, BrainCircuit, CircleHelp, Fullscreen, @@ -72,6 +73,13 @@ let readerBook = $derived( isReader ? appState.documents.find((document) => document.id === readerDocumentId) : undefined ); + /** The page view needs the original file, which only a PDF import keeps — + * and only while the bytes are still on this device. */ + let originalPagesAvailable = $derived( + readerBook?.sourceKind === 'pdf' && Boolean(readerBook.sourcePath || readerBook.sourceBlob) + ); + const viewLabels = { reading: 'Reading view', page: 'Original pages' } as const; + let tourContext = $derived( isReader ? 'reader' @@ -278,6 +286,23 @@ > + {#if originalPagesAvailable} + + {/if}
{/if} -
-
- {book.sourceKind.toUpperCase()} · Local library -

- {#if titleBlock} - {#each segmentsByBlock.get(titleBlock.id) ?? [] as segment (segment.id)} - {@render renderSegment(titleBlock, segment)} - {/each} - {:else} - {book.title} - {/if} -

-

- {Math.max(1, Math.round(player.totalDuration / 60))} min read · {book.segments.length} - passages{#if documentPageCount} · {documentPageCount} pages{/if} -

-
+ {#if pageViewActive && documentPageCount} + + {:else} +
+
+ {book.sourceKind.toUpperCase()} · Local library +

+ {#if titleBlock} + {#each segmentsByBlock.get(titleBlock.id) ?? [] as segment (segment.id)} + {@render renderSegment(titleBlock, segment)} + {/each} + {:else} + {book.title} + {/if} +

+

+ {Math.max(1, Math.round(player.totalDuration / 60))} min read · {book.segments.length} + passages{#if documentPageCount} · {documentPageCount} pages{/if} +

+
+ +
+ {#each rootBlocks as block (block.id)} + {@const markerPage = pageStartsById.get(block.id)} + + {#if markerPage !== undefined && block.id !== rootBlocks[0]?.id} + + {/if} + {#if block.kind === 'list-item'} +
    {@render renderBlock(block)}
+ {:else} + {@render renderBlock(block)} + {/if} + {/each} +
-
- {#each rootBlocks as block (block.id)} - {@const markerPage = pageStartsById.get(block.id)} - - {#if markerPage !== undefined && block.id !== rootBlocks[0]?.id} - - {/if} - {#if block.kind === 'list-item'} -
    {@render renderBlock(block)}
- {:else} - {@render renderBlock(block)} + {#if narrationStartAction} + {@const selectedSegment = book.segments.find( + (segment) => segment.id === narrationStartAction?.segmentId + )} + {#if selectedSegment} +
+ + + + + + + +
{/if} + {/if} + + {#each annotationMarkers as marker (marker.id)} + {/each} -
- {#if narrationStartAction} - {@const selectedSegment = book.segments.find( - (segment) => segment.id === narrationStartAction?.segmentId - )} - {#if selectedSegment} + {#if annotationEditor} + {@const editor = annotationEditor}
- - - - - - - -
- {/if} - {/if} - - {#each annotationMarkers as marker (marker.id)} - - {/each} - - {#if annotationEditor} - {@const editor = annotationEditor} - - {/if} - - {#if explainBox} - - {/if} -
+ {/if} +
+ {/if}

{narrationAnnouncement}

{#if !player.autoFollow} From 058726cf8cdf1b0b23fb4de9b25cb428de1ae316 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:18:32 -0500 Subject: [PATCH 03/15] feat: carry reader ink and assistant focus onto the original pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Highlights, margin notes, and the passage the assistant is pointing at were only visible in the reading view; on the page view they simply went missing, which reads as lost work. They paint as their own rectangles here, ordered back to front — persistent ink under live emphasis — since there is no cascade to settle it for us. Also covers the layout service, which is how the window-join bug surfaced: a caller waiting on a read someone else started was handed that caller's page instead of its own. Co-Authored-By: Claude Opus 5 --- .gitignore | 3 + src/lib/components/PdfPageView.svelte | 72 +++++++++-- src/lib/domain/pdf-layout.spec.ts | 9 +- src/lib/services/pdf-layout.spec.ts | 167 ++++++++++++++++++++++++++ src/lib/services/pdf-layout.ts | 142 +++++++++++++--------- src/routes/read/+page.svelte | 3 + tests/reader.e2e.ts | 72 +++++++++++ 7 files changed, 395 insertions(+), 73 deletions(-) create mode 100644 src/lib/services/pdf-layout.spec.ts diff --git a/.gitignore b/.gitignore index e503f4a..4387109 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ skills-lock.json # Local test fixtures (user-provided, not committed) test_documents/ .probe/ + +# Locally staged preview fixtures (never committed) +static/attention.pdf diff --git a/src/lib/components/PdfPageView.svelte b/src/lib/components/PdfPageView.svelte index c6ba8e8..51d0c8a 100644 --- a/src/lib/components/PdfPageView.svelte +++ b/src/lib/components/PdfPageView.svelte @@ -13,6 +13,12 @@ segments: SpeechSegment[]; activeSegmentId?: string; activeWordIndex?: number; + /** Passages carrying a reader highlight or margin note. */ + annotatedSegmentIds?: ReadonlySet; + /** The stretch the assistant is discussing, and the one sentence inside + * it that it is pointing at right now. */ + assistantSegmentIds?: ReadonlySet; + assistantPointId?: string; /** Whether playback should pull the page along with it. */ follow?: boolean; onPlaySegment: (segmentId: string) => void; @@ -24,6 +30,9 @@ segments, activeSegmentId, activeWordIndex, + annotatedSegmentIds, + assistantSegmentIds, + assistantPointId, follow = true, onPlaySegment }: Props = $props(); @@ -188,6 +197,38 @@ return `left:${rect.x * scale}px;top:${rect.y * scale}px;width:${rect.width * scale}px;height:${rect.height * scale}px`; } + type MarkKind = 'annotated' | 'assistant' | 'point' | 'passage' | 'hover'; + + /** + * Everything to paint over one page, back to front. The reading view puts + * these on the same passage as competing backgrounds and lets specificity + * decide; here they are separate rectangles, so the order they are emitted + * in is what decides — live emphasis last, over persistent ink. + */ + function marksFor(page: number, placed?: Map) { + const marks: Array<{ key: string; kind: MarkKind; rect: PageRect }> = []; + if (!placed) return marks; + const add = (kind: MarkKind, segmentId: string) => { + const rects = placed.get(segmentId)?.rects ?? []; + rects.forEach((rect, index) => + marks.push({ key: `${kind}:${segmentId}:${index}`, kind, rect }) + ); + }; + for (const segmentId of annotatedSegmentIds ?? []) { + if (segmentId !== activeSegmentId) add('annotated', segmentId); + } + for (const segmentId of assistantSegmentIds ?? []) { + if (segmentId !== activeSegmentId && segmentId !== assistantPointId) { + add('assistant', segmentId); + } + } + if (assistantPointId && assistantPointId !== activeSegmentId) add('point', assistantPointId); + if (hovered?.page === page && hovered.segmentId !== activeSegmentId) + add('hover', hovered.segmentId); + if (activeSegmentId) add('passage', activeSegmentId); + return marks; + } + /** The passage under a point on a page, in PDF points. Rectangles are * tested with a little vertical slack: line boxes stop at the type's * bounds, and the gap between two lines belongs to one of them. */ @@ -287,18 +328,11 @@ > {#if live.has(size.page)} {/if} @@ -380,6 +414,20 @@ background: color-mix(in srgb, var(--primary) 12%, transparent); } + /* Persistent reader ink: the same bookmark gold the reading view paints + under an annotated passage. */ + .mark.annotated { + background: color-mix(in srgb, var(--bookmark) 34%, transparent); + } + + .mark.assistant { + background: color-mix(in srgb, var(--primary) 14%, transparent); + } + + .mark.point { + background: color-mix(in srgb, var(--primary) 30%, transparent); + } + .mark.word { background: var(--active-word-bg, rgba(112, 176, 143, 0.34)); box-shadow: 0 0 0 0.1em var(--active-word-bg, rgba(112, 176, 143, 0.34)); diff --git a/src/lib/domain/pdf-layout.spec.ts b/src/lib/domain/pdf-layout.spec.ts index cd69004..ff6ffaa 100644 --- a/src/lib/domain/pdf-layout.spec.ts +++ b/src/lib/domain/pdf-layout.spec.ts @@ -24,8 +24,9 @@ function line(text: string, y: number, startX = 0): PageWordBox[] { return boxes; } -function words(text: string): Array<{ start: number; end: number }> { +function words(text: string): Array<{ text: string; start: number; end: number }> { return [...text.matchAll(/\S+/g)].map((match) => ({ + text: match[0], start: match.index ?? 0, end: (match.index ?? 0) + match[0].length })); @@ -173,9 +174,9 @@ describe('mergeWordRects', () => { it('breaks at a column jump rather than sweeping across the gutter', () => { const merged = mergeWordRects([ - { text: 'left', x: 40, y: 100, width: 40, height: 12 }, - { text: 'right', x: 300, y: 100, width: 40, height: 12 }, - { text: 'wrapped', x: 40, y: 100, width: 40, height: 12 } + { x: 40, y: 100, width: 40, height: 12 }, + { x: 300, y: 100, width: 40, height: 12 }, + { x: 40, y: 100, width: 40, height: 12 } ]); expect(merged).toHaveLength(2); }); diff --git a/src/lib/services/pdf-layout.spec.ts b/src/lib/services/pdf-layout.spec.ts new file mode 100644 index 0000000..dbc3d9a --- /dev/null +++ b/src/lib/services/pdf-layout.spec.ts @@ -0,0 +1,167 @@ +import { describe, expect, it, vi } from 'vitest'; +import { DocumentLayout, type PageLayout, type PageWindowReader } from './pdf-layout'; +import type { SpeechSegment } from '../domain/types'; + +const BYTES = new Uint8Array([1, 2, 3]); + +/** A page of three words on one line, positioned so placement has something + * to bite on. */ +function page(number: number, words: string[]): PageLayout { + let x = 72; + return { + page: number, + width: 612, + height: 792, + boxes: words.map((text) => { + const box = { text, x, y: 100, width: text.length * 6, height: 12 }; + x += text.length * 6 + 4; + return box; + }) + }; +} + +function reader(pages: Record): { + read: PageWindowReader; + calls: Array<[number, number]>; +} { + const calls: Array<[number, number]> = []; + const read: PageWindowReader = async (_data, from, to) => { + calls.push([from, to]); + const out: PageLayout[] = []; + for (let number = from; number <= to; number += 1) { + if (pages[number]) out.push(page(number, pages[number])); + } + return out; + }; + return { read, calls }; +} + +function layoutFor(pages: Record, source: Uint8Array | null = BYTES) { + const { read, calls } = reader(pages); + return { layout: new DocumentLayout('doc', async () => source, read), calls }; +} + +function segment(id: string, text: string, pageNumber: number): SpeechSegment { + return { + id, + blockId: 'b1', + text, + normalizedText: text, + start: 0, + end: text.length, + words: [...text.matchAll(/\S+/g)].map((match) => ({ + text: match[0], + start: match.index ?? 0, + end: (match.index ?? 0) + match[0].length + })), + estimatedDuration: 1, + anchor: { page: pageNumber } + }; +} + +describe('DocumentLayout', () => { + it('reads a window around the page asked for, leaning ahead', async () => { + const { layout, calls } = layoutFor({ 3: ['alpha'] }); + await layout.pageLayout(3, 20); + expect(calls).toEqual([[2, 5]]); + }); + + it('does not read past the end of the document', async () => { + const { layout, calls } = layoutFor({ 5: ['alpha'] }); + await layout.pageLayout(5, 5); + expect(calls).toEqual([[4, 5]]); + }); + + it('serves later pages of a window without reading again', async () => { + const { layout, calls } = layoutFor({ 2: ['alpha'], 3: ['beta'], 4: ['gamma'] }); + await layout.pageLayout(2, 10); + const ahead = await layout.pageLayout(4, 10); + expect(ahead?.boxes[0].text).toBe('gamma'); + expect(calls).toHaveLength(1); + }); + + it('joins a read already in flight rather than starting a second', async () => { + const { layout, calls } = layoutFor({ 1: ['alpha'], 2: ['beta'] }); + const [first, second] = await Promise.all([layout.pageLayout(1, 10), layout.pageLayout(2, 10)]); + expect(first?.page).toBe(1); + expect(second?.page).toBe(2); + expect(calls).toHaveLength(1); + }); + + it('remembers a page the read had nothing for, and stops asking', async () => { + const { layout, calls } = layoutFor({ 1: ['alpha'] }); + expect(await layout.pageLayout(2, 10)).toBeNull(); + expect(await layout.pageLayout(2, 10)).toBeNull(); + expect(calls).toHaveLength(1); + }); + + it('has no geometry when the file is gone from this device', async () => { + const { layout, calls } = layoutFor({ 1: ['alpha'] }, null); + expect(await layout.pageLayout(1, 10)).toBeNull(); + expect(calls).toHaveLength(0); + }); + + it('survives a read that throws', async () => { + const failing: PageWindowReader = () => Promise.reject(new Error('wasm said no')); + const layout = new DocumentLayout('doc', async () => BYTES, failing); + expect(await layout.pageLayout(1, 10)).toBeNull(); + }); + + it('places the passages anchored to a page', async () => { + const { layout } = layoutFor({ 1: ['the', 'harbour', 'master'] }); + const placed = await layout.placements( + 1, + 1, + [segment('s1', 'The harbour master', 1)], + new Map() + ); + expect(placed.get('s1')?.rects[0].y).toBe(100); + }); + + it('places each page once and keeps the answer', async () => { + const { layout, calls } = layoutFor({ 1: ['the', 'harbour', 'master'] }); + const segments = [segment('s1', 'The harbour master', 1)]; + const first = await layout.placements(1, 1, segments, new Map()); + const again = await layout.placements(1, 1, segments, new Map()); + expect(again).toBe(first); + expect(calls).toHaveLength(1); + }); + + it('throws its placements away when the passages are rebound', async () => { + const { layout, calls } = layoutFor({ 1: ['the', 'harbour', 'master'] }); + const first = await layout.placements( + 1, + 1, + [segment('s1', 'The harbour master', 1)], + new Map() + ); + const rebound = await layout.placements( + 1, + 1, + [segment('s1:n0', 'The harbour master', 1)], + new Map() + ); + expect(rebound).not.toBe(first); + expect(rebound.has('s1:n0')).toBe(true); + // The geometry is still good, though — only the placement was stale. + expect(calls).toHaveLength(1); + }); + + it('does not cache a placement computed against passages already replaced', async () => { + const gate = vi.fn(); + let release: (() => void) | undefined; + const read: PageWindowReader = async () => + new Promise((resolve) => { + release = () => resolve([page(1, ['the', 'harbour', 'master'])]); + gate(); + }); + const layout = new DocumentLayout('doc', async () => BYTES, read); + const stale = layout.placements(1, 1, [segment('old', 'The harbour master', 1)], new Map()); + await vi.waitFor(() => expect(gate).toHaveBeenCalled()); + const fresh = layout.placements(1, 1, [segment('new', 'The harbour master', 1)], new Map()); + release?.(); + await stale; + const settled = await fresh; + expect(settled.has('new')).toBe(true); + }); +}); diff --git a/src/lib/services/pdf-layout.ts b/src/lib/services/pdf-layout.ts index 0b4ee33..ff2fa85 100644 --- a/src/lib/services/pdf-layout.ts +++ b/src/lib/services/pdf-layout.ts @@ -50,26 +50,39 @@ async function loadLiteparse(): Promise Promise; + /** * One document's page geometry, parsed on demand and kept for the session. * Requests for the same page while a parse is in flight join it rather than * starting a second one — scrolling asks for the same page many times. */ -class DocumentLayout { +export class DocumentLayout { readonly documentId: string; #source: Promise; + #read: PageWindowReader; #pages = new Map(); - #pending = new Map>(); + /** The in-flight window read covering each page, shared by every caller + * waiting on it. */ + #pending = new Map>(); #placements = new Map>(); #segments: SpeechSegment[] = []; - constructor(document: NormalizedDocument) { - this.documentId = document.id; - this.#source = (async () => { - const blob = await getSource(document); - if (!blob) return null; - return new Uint8Array(await blob.arrayBuffer()); - })().catch(() => null); + constructor( + documentId: string, + loadSource: () => Promise, + read: PageWindowReader + ) { + this.documentId = documentId; + this.#read = read; + this.#source = loadSource().catch(() => null); } /** Word boxes for one page, or null when the source is gone (OPFS @@ -77,63 +90,40 @@ class DocumentLayout { pageLayout(page: number, pageCount: number): Promise { const cached = this.#pages.get(page); if (cached !== undefined) return Promise.resolve(cached); + // What a waiting caller wants is its OWN page out of the finished read, + // not the page whoever started the read had asked for. + const settle = (work: Promise) => + work.then( + () => this.#pages.get(page) ?? null, + () => null + ); const inFlight = this.#pending.get(page); - if (inFlight) return inFlight; + if (inFlight) return settle(inFlight); const from = Math.max(1, page - WINDOW_BEHIND); const to = Math.max(from, Math.min(pageCount || page + WINDOW_AHEAD, page + WINDOW_AHEAD)); - const work = this.#parseWindow(from, to).then( - () => this.#pages.get(page) ?? null, - () => null - ); + const work = this.#parseWindow(from, to); for (let target = from; target <= to; target += 1) { if (!this.#pages.has(target)) this.#pending.set(target, work); } - void work.finally(() => { - for (let target = from; target <= to; target += 1) { - if (this.#pending.get(target) === work) this.#pending.delete(target); - } - }); - return work; + void work + .catch(() => undefined) + .finally(() => { + for (let target = from; target <= to; target += 1) { + if (this.#pending.get(target) === work) this.#pending.delete(target); + } + }); + return settle(work); } async #parseWindow(from: number, to: number): Promise { const data = await this.#source; - if (!data) { - for (let page = from; page <= to; page += 1) this.#pages.set(page, null); - return; - } - const glue = await loadLiteparse(); - const parser = new glue.LiteParse({ - ocrEnabled: false, - outputFormat: 'markdown', - // Only the geometry is wanted here; images and links are the import's - // business and decoding them again would dominate the parse. - imageMode: 'off', - extractLinks: false, - skipDiagonalText: true, - emitWordBoxes: true, - quiet: true, - targetPages: from === to ? String(from) : `${from}-${to}` - }); - let parsed: Awaited>; - try { - // LiteParse transfers the buffer to wasm; hand it a copy so the next - // window still has bytes to read. - parsed = await parser.parse(data.slice()); - } finally { - parser.free(); - } + const parsed = data ? await this.#read(data, from, to) : []; const seen = new Set(); - for (const page of parsed.pages) { - seen.add(page.pageNum); - this.#pages.set(page.pageNum, { - page: page.pageNum, - width: page.width, - height: page.height, - boxes: pageWordBoxes(page.textItems ?? []) - }); + for (const layout of parsed) { + seen.add(layout.page); + this.#pages.set(layout.page, layout); } - // A page the parse skipped has no geometry and never will; recording the + // A page the read skipped has no geometry and never will; recording the // miss stops every scroll from asking again. for (let page = from; page <= to; page += 1) if (!seen.has(page)) this.#pages.set(page, null); } @@ -172,12 +162,52 @@ class DocumentLayout { } } +/** The real reader: LiteParse over a page range, asked for geometry only. */ +const readWithLiteparse: PageWindowReader = async (data, from, to) => { + const glue = await loadLiteparse(); + const parser = new glue.LiteParse({ + ocrEnabled: false, + outputFormat: 'markdown', + // Only the geometry is wanted here; images and links are the import's + // business and decoding them again would dominate the parse. + imageMode: 'off', + extractLinks: false, + skipDiagonalText: true, + emitWordBoxes: true, + quiet: true, + targetPages: from === to ? String(from) : `${from}-${to}` + }); + let parsed: Awaited>; + try { + // LiteParse transfers the buffer to wasm; hand it a copy so the next + // window still has bytes to read. + parsed = await parser.parse(data.slice()); + } finally { + parser.free(); + } + return parsed.pages.map((page) => ({ + page: page.pageNum, + width: page.width, + height: page.height, + boxes: pageWordBoxes(page.textItems ?? []) + })); +}; + let open: DocumentLayout | undefined; /** The layout source for a document, kept warm one at a time — opening * another document releases the previous one, matching the page renderer. */ export function openPdfLayout(document: NormalizedDocument): DocumentLayout { - if (open?.documentId !== document.id) open = new DocumentLayout(document); + if (open?.documentId !== document.id) { + open = new DocumentLayout( + document.id, + async () => { + const blob = await getSource(document); + return blob ? new Uint8Array(await blob.arrayBuffer()) : null; + }, + readWithLiteparse + ); + } return open; } @@ -187,5 +217,3 @@ export function releasePdfLayout(documentId?: string): void { if (documentId && open.documentId !== documentId) return; open = undefined; } - -export type { DocumentLayout }; diff --git a/src/routes/read/+page.svelte b/src/routes/read/+page.svelte index 13766da..982f1d6 100644 --- a/src/routes/read/+page.svelte +++ b/src/routes/read/+page.svelte @@ -2090,6 +2090,9 @@ segments={book.segments} {activeSegmentId} activeWordIndex={player.currentWordIndex} + annotatedSegmentIds={annotationPaint.segmentIds} + {assistantSegmentIds} + {assistantPointId} follow={player.autoFollow} onPlaySegment={playPlacedSegment} /> diff --git a/tests/reader.e2e.ts b/tests/reader.e2e.ts index ec32946..96b69e9 100644 --- a/tests/reader.e2e.ts +++ b/tests/reader.e2e.ts @@ -1785,6 +1785,78 @@ test('imports a PDF with page markers, page navigation, and the original-page vi await expect(page.getByRole('dialog')).toHaveCount(0); }); +test('reads a PDF as its own pages, with the spoken passage painted on', async ({ page }) => { + await openReadyLibrary(page); + const pdf = await PDFDocument.create(); + const font = await pdf.embedFont(StandardFonts.Helvetica); + const sourcePages = [ + [ + 'Ledgers of the Coast', + 'The harbour master kept a ledger of every tide that entered the bay.', + 'Each entry recorded the hour, the depth, and the weather over the water.' + ], + [ + 'The Second Winter', + 'By the second winter the ledger had outgrown its binding entirely.', + 'A cooper in the town sewed the loose pages into a heavier cover.' + ] + ]; + for (const [heading, first, second] of sourcePages) { + const sheet = pdf.addPage([612, 792]); + sheet.drawText(heading, { x: 72, y: 720, size: 22, font }); + sheet.drawText(first, { x: 72, y: 660, size: 12, font }); + sheet.drawText(second, { x: 72, y: 640, size: 12, font }); + } + await page.locator('#document-upload').setInputFiles({ + name: 'ledgers.pdf', + mimeType: 'application/pdf', + buffer: Buffer.from(await pdf.save()) + }); + await expect(page.getByRole('heading', { name: 'Ledgers of the Coast' })).toBeVisible(); + + // The view control only appears for a document that still has its file. + const viewSwitch = page.getByRole('button', { name: /Switch view/ }); + await expect(viewSwitch).toHaveAccessibleName(/Reading view/); + await viewSwitch.click(); + await expect(page.locator('.reading-canvas')).toHaveCount(0); + await expect(viewSwitch).toHaveAccessibleName(/Original pages/); + + // Every page is laid out up front, and the near ones actually draw. + await expect(page.locator('.page-slot')).toHaveCount(2); + const firstPage = page.locator('.page-slot[data-page="1"]'); + await expect + .poll( + async () => firstPage.locator('canvas').evaluate((node: HTMLCanvasElement) => node.width), + { timeout: 30_000 } + ) + .toBeGreaterThan(400); + + // Double-clicking a sentence on the paper plays it, and it lights up where + // it was printed rather than anywhere the reflowed text would have put it. + const sheet = firstPage.locator('.page-sheet'); + const sheetBox = await sheet.boundingBox(); + expect(sheetBox).not.toBeNull(); + // The first body line's baseline sits at 660pt from the foot of a 792pt + // page, so its type occupies roughly 123–135pt from the head. + const scale = sheetBox!.width / 612; + await sheet.dblclick({ position: { x: 200 * scale, y: 129 * scale } }); + const marks = firstPage.locator('.mark.passage'); + await expect.poll(async () => marks.count(), { timeout: 30_000 }).toBeGreaterThan(0); + const painted = await marks.first().boundingBox(); + expect(painted).not.toBeNull(); + // Painted over the line that was clicked, not over the heading above it. + expect(painted!.y - sheetBox!.y).toBeGreaterThan(100 * scale); + expect(painted!.y - sheetBox!.y).toBeLessThan(160 * scale); + expect(painted!.x - sheetBox!.x).toBeGreaterThan(60 * scale); + + // The preference survives leaving and reopening the document. + await page.getByRole('link', { name: 'Library' }).first().click(); + await page.getByRole('link', { name: 'Ledgers of the Coast' }).first().click(); + await expect(page.locator('.page-slot')).toHaveCount(2); + await page.getByRole('button', { name: /Switch view/ }).click(); + await expect(page.locator('.reading-canvas')).toBeVisible(); +}); + test('recognizes a scanned PDF with on-device text recognition', async ({ page }) => { // Import covers a ~7 MB engine download (local assets) plus recognition. test.setTimeout(180_000); From cb45ae0a6691748bbef2ebe31df67bf0e13bd17d Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:22:41 -0500 Subject: [PATCH 04/15] fix: fit a page to the pane, and show which sentence a click will play MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At 100% a page now sizes to the room it has, however narrow the window is — a page you must scroll sideways to read is not a readable page. Above 100% it overflows on purpose (that is what zooming into a figure is for), and the stack centres safely so the left edge stays reachable. The hover wash was mixed for the reader's own background and all but vanished on white paper, leaving nothing to say the paper was clickable. Co-Authored-By: Claude Opus 5 --- src/lib/components/PdfPageView.svelte | 46 +++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/lib/components/PdfPageView.svelte b/src/lib/components/PdfPageView.svelte index 51d0c8a..ae6eaa9 100644 --- a/src/lib/components/PdfPageView.svelte +++ b/src/lib/components/PdfPageView.svelte @@ -67,9 +67,35 @@ }); }); - /** The width a page is drawn at, in CSS pixels: the reader's canvas width, - * capped so a page never overflows the column it sits in. */ - let renderWidth = $derived(readerChrome.documentCanvasWidth); + /** The room the stack has for a page, in CSS pixels. */ + let available = $state(0); + + /** + * The width a page is drawn at. At 100% a page fits the pane, however + * narrow it is — a page you have to scroll sideways to read is not a + * readable page. Above 100% it is meant to overflow: that is what zooming + * into a figure is for, and the stack scrolls both ways to follow. + */ + let renderWidth = $derived.by(() => { + // The reading view's column width before zoom — the same measure, so the + // two views feel like one document at one size. + const column = readerChrome.documentCanvasWidth / readerChrome.documentZoom; + const fitted = available > 0 ? Math.min(column, available) : column; + return Math.max(280, fitted) * readerChrome.documentZoom; + }); + + function trackStack(node: HTMLElement) { + scroller = node; + const observer = new ResizeObserver(() => { + const style = getComputedStyle(node); + available = node.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight); + }); + observer.observe(node); + return () => { + observer.disconnect(); + if (scroller === node) scroller = undefined; + }; + } let blockText = $derived(new Map(book.blocks.map((block) => [block.id, block.text]))); @@ -306,7 +332,7 @@ }); -
+
{#each sizes as size (size.page)} {@const scale = pageScale(size.page)} {@const placed = placements.get(size.page)} @@ -318,6 +344,8 @@
handleActivate(event, size.page)} @@ -346,7 +374,9 @@ display: flex; flex: 1; flex-direction: column; - align-items: center; + /* `safe` matters once a zoomed page is wider than the pane: plain + centring would put its left edge out of reach of the scrollbar. */ + align-items: safe center; gap: 26px; overflow: auto; padding: calc(var(--app-header-height) + 22px) 20px calc(var(--player-height) + 32px); @@ -379,6 +409,10 @@ box-shadow: 0 6px 20px rgba(0, 0, 0, 0.18); } + .page-sheet.over-passage { + cursor: pointer; + } + .page-sheet canvas { display: block; width: 100%; @@ -411,7 +445,7 @@ } .mark.hover { - background: color-mix(in srgb, var(--primary) 12%, transparent); + background: color-mix(in srgb, var(--primary) 17%, transparent); } /* Persistent reader ink: the same bookmark gold the reading view paints From d78855256e97ef5a423d74baca264514d650a9b0 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:24:06 -0500 Subject: [PATCH 05/15] fix: let the contents panel and the assistant move the page view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Outline navigation looked up the heading's element and gave up when it found none — which, in the page view, is always: the document there is a picture of paper. It now addresses the heading's page instead, so clicking a section actually goes somewhere. The page also follows the assistant's fingertip while nothing is playing, which is exactly when "show me this passage" is asked. Co-Authored-By: Claude Opus 5 --- src/lib/components/PdfPageView.svelte | 10 ++++++---- src/routes/read/+page.svelte | 12 +++++++++--- tests/reader.e2e.ts | 10 ++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/lib/components/PdfPageView.svelte b/src/lib/components/PdfPageView.svelte index ae6eaa9..0147536 100644 --- a/src/lib/components/PdfPageView.svelte +++ b/src/lib/components/PdfPageView.svelte @@ -305,12 +305,14 @@ // Follow playback. A passage already comfortably on screen stays put: // re-centring every sentence would drag the page under the reader's eyes. $effect(() => { - if (!follow || !activeSegmentId || !scroller) return; - const page = pageBySegment.get(activeSegmentId); - const placement = activePlacement; + // Playback leads; when it is not running, the assistant's fingertip does. + const followed = activeSegmentId ?? assistantPointId; + if (!follow || !followed || !scroller) return; + const page = pageBySegment.get(followed); + const placement = activePlacement ?? placements.get(page ?? -1)?.get(followed); const target = page === undefined ? undefined : scroller.querySelector(`[data-page="${page}"]`); if (page === undefined || !(target instanceof HTMLElement)) { - const anchored = segments.find((segment) => segment.id === activeSegmentId)?.anchor.page; + const anchored = segments.find((segment) => segment.id === followed)?.anchor.page; if (anchored) goToPage(anchored); return; } diff --git a/src/routes/read/+page.svelte b/src/routes/read/+page.svelte index 982f1d6..c2848ef 100644 --- a/src/routes/read/+page.svelte +++ b/src/routes/read/+page.svelte @@ -449,6 +449,7 @@ book?.sourceKind === 'pdf' && Boolean(book.sourcePath || book.sourceBlob) ); let pagePeek = $state<{ page: number }>(); + let pageView = $state>(); // The original-page view needs both the file and a page count to lay out; // a document missing either reads as markdown whatever the preference says. let pageViewActive = $derived( @@ -872,14 +873,18 @@ } function navigateToOutlineBlock(block: DocumentBlock): void { - const element = elementInReader(block.id); - if (!element) return; + // In the page view there is no element for a heading — the document is a + // picture of paper. Its page number is the address instead, and the + // passage that follows re-centres once it has been placed. + const element = pageViewActive ? undefined : elementInReader(block.id); + if (!element && !pageViewActive) return; const compactOutline = window.matchMedia('(max-width: 820px)').matches; outlineNavigationBlockId = block.id; activeOutlineBlockId = block.id; outlineAnnouncement = `Moved to ${block.text}`; - scrollReaderTo(element, compactOutline); + if (element) scrollReaderTo(element, compactOutline); + else if (block.anchor.page) pageView?.goToPage(block.anchor.page); if (compactOutline) readerChrome.outlineOpen = false; const index = firstSegmentIndex(block); @@ -2085,6 +2090,7 @@ {#if pageViewActive && documentPageCount} page.locator('.page-stack').evaluate((node) => node.scrollTop), { + timeout: 10_000 + }) + .toBeGreaterThan(sheetBox!.height / 2); + // The preference survives leaving and reopening the document. await page.getByRole('link', { name: 'Library' }).first().click(); await page.getByRole('link', { name: 'Ledgers of the Coast' }).first().click(); From 48a7832ca5febf8e8ce05252ebe00225ea645cd3 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:26:21 -0500 Subject: [PATCH 06/15] fix: read a two-column page in reading order before placing anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LiteParse hands back word boxes in raster order — every line of a two-column page interleaved with the line beside it — while its markdown reads the columns properly. A monotonic alignment between two streams that disagree about order finds almost nothing: 57% of passages placed on a two-column paper, against 90% on a single-column one. A recursive XY-cut recovers the order, splitting at vertical gutters before horizontal ones so a page breaks into columns before it breaks into paragraphs. Two-column placement goes to 92%; the single-column paper is unchanged at 91%. Co-Authored-By: Claude Opus 5 --- src/lib/domain/pdf-layout.spec.ts | 59 ++++++++++++++++ src/lib/domain/pdf-layout.ts | 109 ++++++++++++++++++++++++++++++ src/lib/services/pdf-layout.ts | 4 +- 3 files changed, 171 insertions(+), 1 deletion(-) diff --git a/src/lib/domain/pdf-layout.spec.ts b/src/lib/domain/pdf-layout.spec.ts index ff6ffaa..29fac74 100644 --- a/src/lib/domain/pdf-layout.spec.ts +++ b/src/lib/domain/pdf-layout.spec.ts @@ -6,6 +6,7 @@ import { mergeWordRects, pageWordBoxes, placeSegments, + readingOrder, segmentsForPage, type PageWordBox, type PlaceableSegment @@ -86,6 +87,64 @@ describe('pageWordBoxes', () => { }); }); +describe('readingOrder', () => { + /** A two-column page: a full-width title, then lines that alternate + * between the columns exactly as a raster scan hands them over. */ + function twoColumnPage(): PageWordBox[] { + const boxes = [...line('A Title Across The Page', 40, 150)]; + for (let index = 0; index < 6; index += 1) { + const y = 100 + index * 14; + boxes.push(...line(`left${index}a left${index}b`, y, 50)); + boxes.push(...line(`right${index}a right${index}b`, y + 2, 320)); + } + return boxes; + } + + it('reads each column through before starting the next', () => { + const ordered = readingOrder(twoColumnPage(), 612).map((box) => box.text); + const leftEnd = ordered.indexOf('left5b'); + const rightStart = ordered.indexOf('right0a'); + expect(ordered[0]).toBe('A'); + expect(leftEnd).toBeLessThan(rightStart); + }); + + it('keeps a full-width heading ahead of the columns beneath it', () => { + const ordered = readingOrder(twoColumnPage(), 612).map((box) => box.text); + expect(ordered.slice(0, 5)).toEqual(['A', 'Title', 'Across', 'The', 'Page']); + }); + + it('leaves a single column in the order it arrived', () => { + const single = [ + ...line('the first line of the paragraph', 100, 50), + ...line('the second line of the paragraph', 114, 50), + ...line('and a third', 128, 50) + ]; + expect(readingOrder(single, 612)).toEqual(single); + }); + + it('reads a line left to right even when its words arrive scrambled', () => { + const words = line('alpha beta gamma', 100, 50); + const scrambled = [words[2], words[0], words[1]]; + expect(readingOrder(scrambled, 612).map((box) => box.text)).toEqual(['alpha', 'beta', 'gamma']); + }); + + it('does not read a running head as two columns', () => { + // One line with something at each margin is a header, not a page of + // columns — splitting it would put the folio before the title. + const header = [ + { text: 'Chapter', x: 50, y: 40, width: 40, height: 10 }, + { text: '12', x: 540, y: 40, width: 12, height: 10 } + ]; + expect(readingOrder(header, 612).map((box) => box.text)).toEqual(['Chapter', '12']); + }); + + it('has nothing to reorder on an empty or single-word page', () => { + expect(readingOrder([], 612)).toEqual([]); + const one = line('alone', 10, 10); + expect(readingOrder(one, 612)).toEqual(one); + }); +}); + describe('chunkText', () => { it('keys whitespace-separated chunks and remembers where they came from', () => { expect(chunkText('The Transformer, again')).toEqual([ diff --git a/src/lib/domain/pdf-layout.ts b/src/lib/domain/pdf-layout.ts index 81c638a..ce3a663 100644 --- a/src/lib/domain/pdf-layout.ts +++ b/src/lib/domain/pdf-layout.ts @@ -115,6 +115,115 @@ export function pageWordBoxes(items: PageTextItem[]): PageWordBox[] { return boxes; } +/** + * Reading order for a page's words. + * + * LiteParse hands back word boxes in raster order — every line of a + * two-column page interleaved with the line beside it. Its *markdown* reads + * the columns properly, so the passages arrive in true reading order and the + * boxes do not, and a monotonic alignment between the two finds almost + * nothing. (Measured on a two-column paper: 57% of passages placed, against + * 90% on a single-column one.) + * + * A recursive XY-cut recovers the order. At each step the region is split at + * whitespace that runs all the way across it — vertical gutters first, + * because a page whose body is two columns must break into columns before it + * breaks into paragraphs, or the halves interleave again. A single-column + * page finds no gutter, splits into paragraphs, and comes out in the order it + * already had. + */ +export function readingOrder(boxes: PageWordBox[], pageWidth: number): PageWordBox[] { + if (boxes.length < 2) return boxes; + const line = medianHeight(boxes); + const columnGap = Math.max(10, pageWidth * 0.025); + const rowGap = Math.max(2, line * 0.8); + return cut(boxes, columnGap, rowGap, line, 0); +} + +/** Typical line height on the page, used to size the gaps that count as + * whitespace. The median shrugs off headings and subscripts. */ +function medianHeight(boxes: PageWordBox[]): number { + const heights = boxes.map((box) => box.height).sort((left, right) => left - right); + return heights[heights.length >> 1] || 10; +} + +/** Regions this deep are paragraphs; splitting further only costs time. */ +const MAX_CUT_DEPTH = 8; +/** A vertical split needs a region tall enough to be a column. Without this, + * a single line with something at each margin — a running head, a page + * number — reads as two columns. */ +const MIN_COLUMN_LINES = 4; + +function cut( + boxes: PageWordBox[], + columnGap: number, + rowGap: number, + line: number, + depth: number +): PageWordBox[] { + if (boxes.length < 2 || depth >= MAX_CUT_DEPTH) return byLine(boxes); + const height = extent(boxes, 'y'); + if (height >= line * MIN_COLUMN_LINES) { + const columns = split(boxes, 'x', columnGap); + if (columns.length > 1) { + return columns.flatMap((column) => cut(column, columnGap, rowGap, line, depth + 1)); + } + } + const rows = split(boxes, 'y', rowGap); + if (rows.length > 1) { + return rows.flatMap((row) => cut(row, columnGap, rowGap, line, depth + 1)); + } + return byLine(boxes); +} + +function extent(boxes: PageWordBox[], axis: 'x' | 'y'): number { + const size = axis === 'x' ? 'width' : 'height'; + let low = Infinity; + let high = -Infinity; + for (const box of boxes) { + low = Math.min(low, box[axis]); + high = Math.max(high, box[axis] + box[size]); + } + return high - low; +} + +/** Split a region wherever whitespace runs all the way across it on one axis, + * keeping the pieces in ascending order. */ +function split(boxes: PageWordBox[], axis: 'x' | 'y', minGap: number): PageWordBox[][] { + const size = axis === 'x' ? 'width' : 'height'; + const ordered = [...boxes].sort((left, right) => left[axis] - right[axis]); + const groups: PageWordBox[][] = []; + let group: PageWordBox[] = []; + let reach = -Infinity; + for (const box of ordered) { + if (group.length && box[axis] - reach > minGap) { + groups.push(group); + group = []; + } + group.push(box); + reach = Math.max(reach, box[axis] + box[size]); + } + if (group.length) groups.push(group); + return groups; +} + +/** The last word: rows of type, each read left to right. */ +function byLine(boxes: PageWordBox[]): PageWordBox[] { + const ordered = [...boxes].sort((left, right) => left.y - right.y || left.x - right.x); + const lines: PageWordBox[][] = []; + for (const box of ordered) { + const current = lines[lines.length - 1]; + const previous = current?.[0]; + const sameLine = + previous && + Math.abs(box.y + box.height / 2 - (previous.y + previous.height / 2)) < + Math.max(box.height, previous.height) / 2; + if (sameLine) current.push(box); + else lines.push([box]); + } + return lines.flatMap((entries) => entries.sort((left, right) => left.x - right.x)); +} + interface Chunk { key: string; start: number; diff --git a/src/lib/services/pdf-layout.ts b/src/lib/services/pdf-layout.ts index ff2fa85..3fa9029 100644 --- a/src/lib/services/pdf-layout.ts +++ b/src/lib/services/pdf-layout.ts @@ -2,6 +2,7 @@ import type { NormalizedDocument, SpeechSegment } from '../domain/types'; import { pageWordBoxes, placeSegments, + readingOrder, segmentsForPage, type PageWordBox, type SegmentPlacement @@ -189,7 +190,8 @@ const readWithLiteparse: PageWindowReader = async (data, from, to) => { page: page.pageNum, width: page.width, height: page.height, - boxes: pageWordBoxes(page.textItems ?? []) + // Raster order as read; reading order as written. + boxes: readingOrder(pageWordBoxes(page.textItems ?? []), page.width) })); }; From 2083d6bd78fb8a708f6df9347e3db3e4af603a8e Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 23:39:48 -0500 Subject: [PATCH 07/15] fix: stop the page view snapping back to the playhead while scrolling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following was a plain effect over state that changes for reasons having nothing to do with the playhead: every page that finishes placing as you scroll updates `placements`, and each of those updates re-ran the scroll. Reading ahead was impossible — the document pulled itself back within a second or two, over and over. Following is now edge-triggered, the way the reflowed canvas has always done it: one passage becoming current earns at most one jump to its page and one settle onto its line. And a wheel or touch scroll stops following outright, again matching the reflowed canvas — that listener was simply missing here, so "Follow narration" never appeared and there was no way to opt out. Co-Authored-By: Claude Opus 5 --- src/lib/components/PdfPageView.svelte | 62 ++++++++++++++++++++++++--- src/routes/read/+page.svelte | 1 + tests/reader.e2e.ts | 27 ++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/src/lib/components/PdfPageView.svelte b/src/lib/components/PdfPageView.svelte index 0147536..c8145b2 100644 --- a/src/lib/components/PdfPageView.svelte +++ b/src/lib/components/PdfPageView.svelte @@ -1,4 +1,5 @@ -
+
{#each sizes as size (size.page)} {@const scale = pageScale(size.page)} {@const placed = placements.get(size.page)} @@ -585,6 +588,21 @@ box-shadow: 0 6px 20px rgba(0, 0, 0, 0.18); } + /* Toning is applied to the whole sheet, marks included, so the highlights + are still worked out against white paper the way a highlighter behaves + and only then follow the page into the dark. */ + [data-tone='dim'] .page-sheet { + filter: brightness(0.72) contrast(1.02); + } + + /* Inverting and rotating the hue back keeps the figures' colors roughly + themselves while the paper turns dark and the ink light. Not quite to + black: pure black paper under white type is its own kind of glare. */ + [data-tone='night'] .page-sheet { + box-shadow: none; + filter: invert(0.92) hue-rotate(180deg); + } + .page-sheet.over-passage { cursor: pointer; } diff --git a/src/lib/icons.ts b/src/lib/icons.ts index 5ea7160..e0f6b76 100644 --- a/src/lib/icons.ts +++ b/src/lib/icons.ts @@ -28,6 +28,7 @@ import { CloudRainIcon, CoffeeIcon, CollapseIcon, + ContrastIcon, CopyIcon, CpuIcon, DatabaseIcon, @@ -119,6 +120,7 @@ export const Clock3 = Clock03Icon; export const Cloud = CloudIcon; export const CloudRain = CloudRainIcon; export const Coffee = CoffeeIcon; +export const Contrast = ContrastIcon; export const Copy = CopyIcon; export const Cpu = CpuIcon; export const Database = DatabaseIcon; diff --git a/src/lib/services/pdf-pages.ts b/src/lib/services/pdf-pages.ts index 10d9dc8..6e3db2d 100644 --- a/src/lib/services/pdf-pages.ts +++ b/src/lib/services/pdf-pages.ts @@ -7,8 +7,13 @@ const MAX_PIXELS = 16 * 1024 * 1024; export interface PageRasterizer { readonly pageCount: number; - /** Draws a page (1-based) into `canvas` sized for `cssWidth` CSS pixels at - * the device pixel ratio. Sets the canvas buffer and style sizes. + /** Draws a page (1-based) into `canvas` at the device pixel ratio, for a + * display width of `cssWidth`, and reports the CSS size the result wants to + * be shown at. It does NOT set that size on the canvas: a caller whose + * canvas is sized by its own layout (the page view stretches one to fill its + * sheet) must be free to resize the drawing between draws, and pinning it in + * pixels here left the picture at its old size while everything around it + * had already grown. * * The page is painted off-screen and handed over whole, so `canvas` keeps * whatever it was showing until the new picture is finished. Painting into @@ -25,7 +30,7 @@ export interface PageRasterizer { canvas: HTMLCanvasElement, cssWidth: number, signal?: AbortSignal - ): Promise; + ): Promise<{ width: number; height: number }>; /** Renders a page (1-based) to an OffscreenCanvas at `scale`× the page's * natural point size — the OCR path's rasterizer. */ rasterize(page: number, scale: number): Promise; @@ -114,8 +119,10 @@ export async function createPageRasterizer(data: Uint8Array): Promise serialize(async () => { diff --git a/src/lib/state/reader-chrome.svelte.ts b/src/lib/state/reader-chrome.svelte.ts index 7734a4d..ba3f73f 100644 --- a/src/lib/state/reader-chrome.svelte.ts +++ b/src/lib/state/reader-chrome.svelte.ts @@ -11,6 +11,21 @@ export type ReaderView = 'reading' | 'page'; export const READER_VIEWS: ReaderView[] = ['reading', 'page']; +/** + * How bright the original pages are allowed to be. Paper is the page as + * printed; dimmed takes the glare off white paper without touching what the + * figures look like; night inverts it, so the paper is dark and the ink + * light. Unset follows the theme — a dark theme should not open a document + * by shining a white page at the reader. + */ +export type PageTone = 'paper' | 'dim' | 'night'; + +export const PAGE_TONES: PageTone[] = ['paper', 'dim', 'night']; + +function isPageTone(value: unknown): value is PageTone { + return PAGE_TONES.includes(value as PageTone); +} + function isReaderView(value: unknown): value is ReaderView { return READER_VIEWS.includes(value as ReaderView); } @@ -23,6 +38,8 @@ class ReaderChromeState { /** The preferred view, remembered across documents. A document with no * original pages falls back to reading without changing this. */ readerView = $state('reading'); + /** Undefined until the reader chooses: see `pageToneFor`. */ + pageTone = $state(); /** The listening mode new imports start in. Per-document overrides live on * the document itself and take precedence in the reader. */ defaultListeningMode = $state(DEFAULT_LISTENING_MODE); @@ -49,10 +66,29 @@ class ReaderChromeState { if (isListeningMode(mode)) this.defaultListeningMode = mode; const view = window.localStorage.getItem('voicebook:reader-view'); if (isReaderView(view)) this.readerView = view; + const tone = window.localStorage.getItem('voicebook:page-tone'); + if (isPageTone(tone)) this.pageTone = tone; this.assistantCaptions = window.localStorage.getItem('voicebook:assistant-captions') !== '0'; this.spokenChatReplies = window.localStorage.getItem('voicebook:spoken-chat-replies') !== '0'; } + /** The tone in force, given whether the current theme is a dark one. */ + pageToneFor(darkTheme: boolean): PageTone { + return this.pageTone ?? (darkTheme ? 'dim' : 'paper'); + } + + cyclePageTone(darkTheme: boolean): void { + const current = this.pageToneFor(darkTheme); + this.setPageTone(PAGE_TONES[(PAGE_TONES.indexOf(current) + 1) % PAGE_TONES.length]); + } + + setPageTone(tone: PageTone): void { + this.pageTone = tone; + if (typeof window !== 'undefined') { + window.localStorage.setItem('voicebook:page-tone', tone); + } + } + /** Steps to the next view. A cycle rather than a flip, so a third view can * join without the control changing shape. */ cycleReaderView(): void { diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 955ff4d..7fac31a 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -11,6 +11,7 @@ PanelLeftOpen, BookOpenText, BrainCircuit, + Contrast, CircleHelp, Fullscreen, RefreshCw, @@ -21,6 +22,8 @@ FileUp, Search, Shrink, + Moon, + Sun, Link2, List, Menu, @@ -79,6 +82,8 @@ readerBook?.sourceKind === 'pdf' && Boolean(readerBook.sourcePath || readerBook.sourceBlob) ); const viewLabels = { reading: 'Reading view', page: 'Original pages' } as const; + const toneLabels = { paper: 'Paper', dim: 'Dimmed paper', night: 'Night paper' } as const; + let pageTone = $derived(readerChrome.pageToneFor(appearanceState.themeSpec.dark)); let tourContext = $derived( isReader @@ -303,6 +308,24 @@ {/if} {/if} + {#if originalPagesAvailable && readerChrome.readerView === 'page'} + + {/if}
{/if} - {#if originalPagesAvailable && readerChrome.readerView === 'page'} - - {/if}