diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6ba9788 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# Shell scripts must keep LF endings even when checked out on Windows: a CRLF +# shebang line makes the kernel look for an interpreter named "bash\r", and the +# error it produces ("bad interpreter: No such file or directory") points at +# the interpreter rather than at the line ending. scripts/bootstrap.sh is +# authored on Windows here and run on Linux, so this is not hypothetical. +*.sh text eol=lf + +# Same reasoning for anything a Linux runner executes or parses strictly. +*.yml text eol=lf +*.yaml text eol=lf diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 99bf168..9ca18d2 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -7,13 +7,28 @@ # everyone can reproduce -- so CI is the number of record and local runs are # indicative. # -# Two lanes (testkit/runall.py lanes): refine OFF is the uncontaminated -# converter number; refine ON is the product default. refine() tunes against -# the same renderer the gate measures with, so only the pair is meaningful. +# Two lanes (testkit/runall.py): `raw` is the uncontaminated converter number, +# `product` is exactdoc.options.PRODUCT -- the profile the API, the CLI and every +# published number all share. refine() tunes against the same renderer the gate +# measures with, so only the pair is meaningful, and **both** lanes gate the exit +# code. Gating on the refined lane alone meant the control lane, whose entire +# purpose is to be untainted, was the one nobody had to answer for. # -# The purity test is a hard failure. The gate lanes are reporting-only until -# the first Linux run establishes the baseline (thresholds so far were -# calibrated on Windows fonts); once pinned, drop the continue-on-error. +# Every step here is fail-closed. That is the whole design: a green check must +# mean that all 16 manifest documents existed, the renderer answered, every +# required metric was computed, nothing regressed past its recorded number, and +# the backend policy still describes reality. It previously could mean none of +# those things -- see the docstrings in testkit/gate.py for the list, each entry +# of which is now a test in tests/test_gate_mutations.py. +# +# Provisioning is scripts/bootstrap.sh, the same command a contributor runs, so +# CI cannot drift away from the documented setup without going red. --strict +# makes a missing oracle a failure. +# +# The dependency versions come from uv.lock (--frozen). The goldens are pinned +# to the PyMuPDF version -- measured: 1.26 and 1.24 both put 02_research_paper +# p2 at 4 blocks where 1.28 puts 7 -- so an unpinned resolve would fail the +# golden step for a reason that has nothing to do with this repository's code. name: gate on: @@ -24,7 +39,7 @@ on: jobs: gate: runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -32,42 +47,60 @@ jobs: with: python-version: "3.12" - - name: Install LibreOffice + fonts (the pinned oracle) - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends libreoffice-writer fonts-liberation - soffice --version + - name: Install uv (uv.lock is the pinned truth) + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true - - name: Install package + test deps - run: | - python -m pip install -q -e .[test] - python -m pip install -q pytest + - name: Provision the oracles (LibreOffice, Chromium, fonts) + deps + run: bash scripts/bootstrap.sh --strict - - name: Generate corpora (ReportLab + Chromium/WeasyPrint-class dialects) + # --strict here too: this printed "SKIPPED 8 document(s)" and "gate numbers + # are NOT comparable", then exited 0, and the next step scored the 8 that + # existed against a 16-document baseline. Prose the next step ignores is + # not a safeguard. + - name: Generate the corpus (16 documents, or fail) run: | - python corpus/make_corpus.py - python testkit/gen_corpus.py testkit/adv + uv run python testkit/gen_corpus.py testkit/adv --strict + uv run python corpus/make_corpus.py - - name: Purity - writing the same layout twice must be byte-identical - run: python tests/test_purity.py + - name: Corpus manifest - is this the corpus the baseline describes? + run: uv run python testkit/corpus_manifest.py verify + + - name: Unit tests (write purity, corpus degradation, gate mutations) + run: | + uv run python tests/test_purity.py + uv run python tests/test_corpus_degradation.py + uv run python tests/test_gate_mutations.py - name: Golden IR - the parser's output must not drift - # Also the acceptance test for any future permissive-parser backend: - # a replacement is correct when it reproduces these. - run: python testkit/golden_ir.py verify + run: uv run python testkit/golden_ir.py verify + + - name: Fidelity gate, both lanes, fail closed + run: uv run python testkit/runall.py - - name: Fidelity gate, both lanes - continue-on-error: true # reporting until the Linux baseline is pinned - env: - REFINE: lanes - run: python testkit/runall.py testkit/adv corpus/pdfs + # No longer continue-on-error. It was reporting-only "until the swap + # lands", which made the number it exists to drive the one number nothing + # depended on. The policy the two accepted shortfalls were ratified under + # now lives in testkit/parity_policy.json with numeric floors, so the + # executable rule and the ratified rule are the same rule and the step can + # be required. + - name: Backend parity - the licence-swap verdict + run: uv run python testkit/backend_parity.py + + - name: Evidence - one artifact every published number traces to + if: always() + run: uv run python testkit/evidence.py --out testkit/batch/evidence.json - - name: Upload lane results + - name: Upload lane results and evidence if: always() uses: actions/upload-artifact@v4 with: name: gate-results path: | - testkit/batch/lane_norefine/results.json - testkit/batch/lane_refine/results.json - if-no-files-found: warn + testkit/batch/evidence.json + testkit/batch/lane_raw/results.json + testkit/batch/lane_raw/verdict.json + testkit/batch/lane_product/results.json + testkit/batch/lane_product/verdict.json + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 08d8fb6..70766fd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,11 @@ dist/ .vscode/ .idea/ +# Written by scripts/bootstrap.sh: discovered oracle paths, and any tool it had +# to fetch because the distribution does not package it. +scripts/env.sh +.tools/ + # Generated test corpora and conversion outputs (all reproducible) testkit/adv/ testkit/real/ @@ -44,5 +49,15 @@ client_secret* # personal test inputs -- never publish my_samples/ -# private working notes -- not for publication +# Private working notes -- not for publication. The planning and review +# documents all share the form "exactdoc — .md" and live in the repo +# root beside the tracked docs, which is one `git add -A` away from publishing +# them. The tracked docs (README, STATUS, THEORY, FINDINGS) do not carry the +# prefix, so this pattern cannot catch them by accident. Executor Advisory.md +exactdoc — *.md +*Execution Plan*.md +# The em-dash pattern above missed `exactdoc-production-readiness-plan.md`, which +# is the same kind of document under a hyphen. Match the suffix as well: no +# tracked doc ends in -plan.md. +*-plan.md diff --git a/FINDINGS.md b/FINDINGS.md index 4728e27..c5e7da1 100644 --- a/FINDINGS.md +++ b/FINDINGS.md @@ -7,6 +7,12 @@ > in §1 are historical. Two claims here were later falsified by measurement and > are marked inline. > +> "v1.1" is a pre-release internal label from before this repository had +> versioned releases. It corresponds to no tag and no published artifact; the +> version line starts at `0.1.0a1` (see the README's Versions table). The +> 18-document corpus measured here is also not the current one — today's gate +> corpus is 16 generated documents. +> > For current state: **[STATUS.md](STATUS.md)**. For the design: **[THEORY.md](THEORY.md)**. 18 documents, 4 producer engines, measured with `testkit/` (shares no code with diff --git a/README.md b/README.md index 919dcda..7194014 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ **PDF → DOCX that keeps the design, survives Google Docs, and measures whether it worked.** +> **Status: alpha (0.1.0a1). Nothing has been released yet.** It works well on the +> documents it was developed against and it fails on pagination for PDFs it has +> never seen. Both numbers are below, in the same table, on purpose. +> +> **Next: the permissive relicence.** The AGPL is inherited from PyMuPDF, and the +> replacement parser now passes the parity gate with zero regressions under a +> ratified, executable policy. The flip to Apache-2.0 is the next milestone — +> [ROADMAP.md](ROADMAP.md) has the sequence and the distance. + Most PDF-to-Word converters either redesign your page (Word's reflow), turn every line into a floating frame that Google Docs then mangles (LibreOffice import), or throw the design away entirely and emit Markdown (Docling, Marker, MinerU). @@ -13,24 +22,45 @@ columns and rules as real editable Word constructs, restricted to the subset Google Docs imports faithfully — no text boxes, no VML, no embedded fonts. Then it checks its own work. Every claim below is a number produced by -[`testkit/`](testkit/README.md), which shares no code with the converter: +[`testkit/`](testkit/README.md), which shares no code with the converter, and +every one of them traces to a single machine-readable artifact — +`testkit/batch/evidence.json`, keyed to the commit, the dependency versions and +the LibreOffice build that produced it. [STATUS.md](STATUS.md) is the authority +on what they mean: -| | | -|---|---| -| gate passed | 15/18 documents | -| page count 1:1 | 17/18 | -| live (editable) text recovered | 96.9% | -| words within 2pt of source | 40.4% | -| median per-word vertical drift | 1.02pt | +| | 16-document corpus | 4 wild PDFs (holdout) | +|---|---|---| +| gate passed | 13/16 | **0/4** | +| page count 1:1 | 15/16 | fails | +| live (editable) text recovered | 96.5% | 94–97% | +| words within 2pt of source | 51.2% | — | +| median per-word vertical drift | 0.62pt | — | + +The corpus has been developed against; the [holdout](testkit/fetch_holdout.py) +never has. The gap between those two columns is the honest measure of how far +along this is: **the text survives, the pagination does not.** + +The corpus column is the `product` lane — the profile a bare `exactdoc file.pdf` +or `convert(file)` actually runs, which is now the same profile the numbers are +measured on. It was not: the API ran 0 refine rounds, the CLI ran 2, and these +figures came from a CI lane that ran 3, so "reproduce it with `convert()`" +produced the raw number with nothing anywhere to say why. There is one profile +now ([`exactdoc/options.py`](exactdoc/options.py)), and the uncontaminated +zero-refine `raw` lane is reported beside it always, because the refine loop +tunes against the same renderer the gate measures with. ## Install +Not on PyPI yet — the first published release will be the Apache-2.0 one (see +[Versions](#versions)). Until then: + ```bash pip install git+https://github.com/ebt55/exactdoc.git ``` -Optional extras: `[test]` for the measurement harness, `[gdocs]` for the Google -Docs oracle. Neither is needed for a plain conversion. +Optional extras: `[test]` for the measurement harness, `[pdfium]` for the +experimental permissive parser, `[gdocs]` for the Google Docs oracle. None is +needed for a plain conversion. `--verify` and `--refine` additionally need LibreOffice on PATH; without it, conversion still works and simply skips the feedback loop. @@ -50,8 +80,8 @@ exactdoc *.pdf --dpi 300 --verify # batch, high-res figures, with a repor ``` ```python -from exactdoc.convert import convert -convert("whitepaper.pdf", "whitepaper.docx", target="gdocs", refine_rounds=2) +from exactdoc import convert +convert("whitepaper.pdf", "whitepaper.docx", target="gdocs") ``` ## Why a "target" matters @@ -63,18 +93,8 @@ LibreOffice places 99% of words within 2pt of the source, **Google Docs places paragraph boundary, and it accumulates down the page. Most converters are tuned against one renderer and silently assume it -generalises. It does not. exactdoc makes the target an explicit choice and -optimises for it. - -### Pick a target renderer - -There is no single "correct" DOCX. Word, LibreOffice and Google Docs lay the -same file out differently, and the gap is not small: on a document where -LibreOffice places 99% of words within 2pt of the source, Google Docs places -1% — Docs adds a one-off gap after the first heading plus ~3pt at every -paragraph boundary, which accumulates down the page. - -So `--target` chooses which program the output should look right in, and the +generalises. It does not. exactdoc makes the target an explicit choice, so +`--target` chooses which program the output should look right in, and the closed-loop pass (below) optimises for that renderer: | `--target` | Oracle | Notes | @@ -89,15 +109,20 @@ Measured, opening the result in Google Docs: tuning for `gdocs` instead of ### Closed-loop correction -`--refine N` (default 2) writes the DOCX, renders it back through the chosen -target, measures page overflow and per-page offsets, corrects the layout and -rewrites — keeping the best round. Without an oracle available it degrades -silently to a single ordinary write, so conversion never depends on it. +`--refine N` writes the DOCX, renders it back through the chosen target, measures +page overflow and per-page offsets, corrects the layout and rewrites — keeping the +best round. Without an oracle available it degrades to a single ordinary write, so +conversion never depends on it. -Python API: +The default is 3, and it is 3 everywhere: the CLI, the Python API and the lane +every published number is measured on all read it from one place +([`exactdoc/options.py`](exactdoc/options.py)). `--refine 0` is the deliberate +open-loop control. + +Python API — same profile, no arguments needed: ```python -from exactdoc.convert import convert +from exactdoc import convert convert("whitepaper.pdf", "whitepaper.docx") ``` @@ -163,21 +188,51 @@ convert("whitepaper.pdf", "whitepaper.docx") ## Verified corpus results -18 documents across five producer dialects (Chromium/Skia, WeasyPrint, -ReportLab, fpdf2, LibreOffice), measured by `testkit/`, which shares no code -with this package: +16 generated documents across four producer dialects — Chromium/Skia (8), +ReportLab (6), fpdf2 (1), LibreOffice/Word-native (1) — measured by `testkit/`, +which shares no code with this package. WeasyPrint and LaTeX/pdfTeX are covered +by real documents outside the gate corpus; LaTeX is the worst case and the +largest open defect (see below). -| | | -|---|---| -| gate passed | 15/18 | -| page count 1:1 | 17/18 | -| live (editable) text | 96.9% mean | -| words within 2pt of source | 40.4% mean | -| median per-word vertical drift | 1.02pt | -| SSIM (LibreOffice render-back) | 0.809 mean | +Both lanes, because only the pair is meaningful: + +| | `raw` (0 refine rounds) | `product` (shipped) | +|---|---|---| +| gate passed | 12/16 | 13/16 | +| page count 1:1 | 13/16 | 15/16 | +| live (editable) text | 96.5% | 96.5% | +| words within 2pt of source | 34.9% | **51.2%** | +| median per-word vertical drift | 2.20pt | **0.62pt** | + +Every figure comes from `testkit/gate_baseline.json`, which records the numeric +value of every gated metric for every document in both lanes, together with the +environment that produced it — Linux, LibreOffice 24.2.7.2, the Liberation metric +fonts, and the exact dependency versions. Three environments (CI Linux, a local +`ubuntu:24.04` container, Windows) agree on every structural number and differ in +the third decimal of `within2pt`; the gate's tolerances are sized from that +spread. + +`refine()` optimises against the same renderer the gate scores with, so a +refined-only number can improve because the loop memorised the oracle rather +than because the converter got better. Reporting one lane would hide that — and +until recently the exit code did exactly that, gating on the refined lane while +the control lane could regress freely. + +Run it yourself (needs the `[test]` extra, LibreOffice for the render-back, and +Chrome to generate the Chromium half of the corpus): -Run it yourself: `python testkit/runall.py testkit/adv my_samples`. It exits -non-zero on regression, so it doubles as CI. +```bash +python testkit/gen_corpus.py testkit/adv --strict && python corpus/make_corpus.py +``` + +```bash +python testkit/corpus_manifest.py verify && python testkit/runall.py +``` + +Both lanes gate the exit code, so it doubles as CI. Add `--absolute` for the +release-qualification gate, which **fails today** — D3 and D10 sit below +threshold, and the point of a separate absolute gate is that it says so instead +of being folded into "nothing got worse". **Do not use SSIM as the headline number.** It is dominated by whitespace and it *rewards* a rasterised page: a resume converted into two flat images scored @@ -226,23 +281,50 @@ For text-flow documents — whitepapers, papers, reports, resumes — near-perfe 5. **Gradients, rounded corners and rotated text** have no paragraph-flow equivalent and must rasterise. +## Versions + +Nothing has been published, so the version numbering is being reset once, now, +while it is free to do so: + +| Version | What it means | +|---|---| +| `0.1.0a1` | today — alpha, AGPL (inherited from PyMuPDF), git install only | +| `0.2.0a1` | the first *published* release, Apache-2.0, after the permissive parser reaches zero parity regressions | +| `0.x` betas | gated on the holdout number improving, not on the corpus number | +| `1.0` | not before wild PDFs stop failing on pagination | + +No AGPL wheel will ever be published: the licence swap lands before the first +release, not after it. + ## Documentation +- [ROADMAP.md](ROADMAP.md) — what is done, what is left, and how far. Start here + if you want to know where this is going +- [STATUS.md](STATUS.md) — the authority on every number, the defect register, + and the measurement mistakes that produced confident wrong answers +- [SESSIONS.md](SESSIONS.md) — the working log: what each session expected to + happen before it ran - [THEORY.md](THEORY.md) — the fidelity model, what worked, what didn't, and why -- [FINDINGS.md](FINDINGS.md) — an independent audit of v1.1 with reproductions +- [FINDINGS.md](FINDINGS.md) — a frozen independent audit with reproductions. + Its "v1.1" is a pre-release internal label from before this repo had versioned + releases; it does not correspond to any tag or published artifact. - [testkit/README.md](testkit/README.md) — the measurement harness and its metrics ## Contributing The fastest way to help is a PDF that breaks it. Producer dialects differ far more than content does, and the corpus is thin on LaTeX, Typst, InDesign and -Quartz. Run `python testkit/runall.py testkit/adv` — it exits non-zero on -regression, so it doubles as CI. +Quartz. Run `python testkit/runall.py` — both lanes gate the exit code, so it +doubles as CI, and `python tests/test_gate_mutations.py` checks the gate itself +in about a second without needing a corpus or an oracle. ## License -[AGPL-3.0-or-later](LICENSE). exactdoc links PyMuPDF, which is AGPL-3.0; the -copyleft is inherited, not chosen. +[AGPL-3.0-or-later](LICENSE) **today, Apache-2.0 next.** exactdoc links PyMuPDF, +which is AGPL-3.0; the copyleft is inherited, not chosen — and the permissive +replacement parser is now measured good enough to take over. The flip is the +next milestone, and no AGPL wheel will ever be published: see +[ROADMAP.md](ROADMAP.md). Relicensing means replacing the parser, and the obstacle is not the API — it is that every threshold downstream was tuned against the *shape* of PyMuPDF's @@ -261,11 +343,45 @@ the vector paths on arXiv papers. pypdfium2 (Apache-2.0) extracts text and paths but provides no line/block grouping, so that clustering has to be written here. -A pypdfium2 backend is written and selectable (`EXACTDOC_BACKEND=pdfium`), but -it is **not** the default and the licence has **not** changed. Measured against -PyMuPDF over the corpus it stands at **9 regressions** on fine placement — -`within2pt` 0.510 → 0.291, median word drift 0.69pt → 2.02pt. Extraction is at -parity (text character-identical, paths exact); positional precision is not. +A pypdfium2 backend is written and selectable (`--backend pdfium`, or +`EXACTDOC_BACKEND=pdfium`; requires the `[pdfium]` extra). It is not the default +*yet* — but it is no longer the blocker it was. + +Measured against PyMuPDF over the corpus, under the acceptance policy in +[`testkit/parity_policy.json`](testkit/parity_policy.json): + +| verdict | count | which | +|---|---|---| +| regression | **0** | — | +| same | 11 | | +| better | 1 | `05_memo`, 0.64 → 0.88 within-2pt | +| expected divergence | 2 | `c4_i18n`, `c5_graphics` — pdfium is the *correct* one, verified by rendering | +| accepted shortfall | 2 | `01_whitepaper_market`, `02_research_paper` — D2, bounded by recorded numeric floors | + +Down from 9 regressions. Those last four documents used to be prose: the code +exited on `regressions == 0` while the docs said two of them were formally +accepted, so CI marked the step `continue-on-error` to keep the build usable — +which retired the only gate the whole relicensing effort was aimed at. The policy +is now data the test executes, the two acceptances carry numeric floors that fail +when crossed, and an acceptance that stops describing reality fails as stale. The +step is required. + +The remaining two are attributed, and the attribution is why they are being +accepted rather than chased: `infer()` derives the page's vertical origin from +line-box *tops*, which is the one vertical quantity two correct parsers +legitimately disagree about, because each reads it from font-metric tables the +other does not have. PyMuPDF puts Helvetica's box 1.075× the type size above the +baseline; pdfium says 0.905×. On Symbol, where both fall back to the *embedded* +font's metrics, they agree to three decimals — which is how we know it is the +tables and not the code. pdfium exposes exactly one vertical font metric and the +parser already uses it, so matching PyMuPDF would mean vendoring MuPDF's own +base-14 table into a permissive tree. That is not something this project will +do. See [STATUS.md](STATUS.md) D2 and [ROADMAP.md](ROADMAP.md) §4. + +Everything else that separated the two parsers has been closed: extraction was +always at parity (text character-identical, baselines identical on 4,734 of +4,734 lines, paths exact), and grouping, path geometry, span segmentation and +whitespace now match the incumbent exactly on every document where they can. Two documents diverge on purpose, both verified by rendering, and on both the new backend is the *correct* one: RTL text (PyMuPDF returns visual order, so diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..860ac99 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,226 @@ +# exactdoc — roadmap: what is done, what is left, how far + +**Updated 2026-07-30.** STATUS.md is the authority on measured numbers; this +file is the authority on *sequence and distance*. If they disagree, STATUS wins +on numbers and this file gets corrected. + +--- + +## The short answer + +**The licence swap is no longer blocked.** It was the one thing standing between +this project and being usable by anyone who cannot accept AGPL, and as of +2026-07-30 the permissive parser is measured **not worse than the incumbent on +14 of 16 corpus documents**, with the remaining two attributed to a cause that +is proven unreachable from a permissive parser and formally accepted as a +documented divergence. + +| question | answer | +|---|---| +| How far to `pip install exactdoc` under Apache-2.0? | **three working sessions** — see §3 | +| How far to "fully working on any PDF you throw at it"? | **much further, and it is a different project** — see §5 | +| Is anything still blocking the swap? | **No.** The one item that was queued ahead of it (superscript) turned out to need no code at all — §3.1 | + +**Before the flip, the gate had to become worth trusting.** The licence swap is a +change to which parser produces every number, and it was about to be judged by a +gate that could pass while the renderer failed on every document, while a +required metric was missing, while 8 of the 16 corpus documents did not exist, +and while a known shortfall slid arbitrarily far. That work is done and is +described in [STATUS.md §1](STATUS.md#1-where-the-converter-stands): a numeric +per-document baseline, an exact corpus manifest, both lanes gating the exit code, +the parity policy as executable data rather than prose, one evidence artifact, +and a mutation test for every false-green path the old gate had. + +The distinction in the last two rows is the important one. Shipping a permissive, +honest, well-measured alpha is close. Making the converter *good on documents it +has never seen* is the open-ended part, and it is deliberately not on the +critical path to a release. + +--- + +## 1. Where the converter stands + +| | value | +|---|---| +| Parity gate (pdfium vs PyMuPDF) | **2 regressions, 13 same, 1 better** | +| Started at | 9 regressions, then 8 when first measured on the canonical environment | +| Mean within-2pt, pdfium | **0.461** against the incumbent's 0.511 | +| Documents at or above the incumbent | **14 of 16** — four exactly equal, two better | +| Gate lanes (default backend) | 13/16 page match raw, 15/16 product; both lanes gate the exit code | +| Golden IR | 7/7 | +| CI | green, and fail-closed — see the three questions in [STATUS §1](STATUS.md#1-where-the-converter-stands) | +| Release-qualification gate (`--absolute`) | **fails, on the record.** D3 and D10 are below threshold and say so | +| Holdout (4 wild PDFs) | **0/4** — unchanged, and the honest generalisation number | + +--- + +## 2. Done, with the evidence + +| | what it means now | +|---|---| +| **M0 — identity reset** | Version `0.1.0a1`, Alpha classifier, every published claim reconciled to measurement. Tagged. | +| **M1 — reproducible measurement** | One command provisions a bare Linux box; the corpus generates or says exactly what it skipped; goldens carry an environment manifest; the gate is a *regression* gate with a recorded baseline, and CI runs it for real. Verified in a clean `ubuntu:24.04` container and on GitHub Actions. | +| **M2.a — instruments** | `backend_spans`, `backend_paths`, `block_gaps`, `residual`, `margin_probe`, `--only`. Six measuring tools that did not exist, each built because a question could not otherwise be answered. | +| **M2.b — page-space geometry** | Path points transformed by the object matrix; bboxes are the geometric path, not the ink envelope. 578 of 612 corpus paths carry a non-identity matrix. | +| **M2.c — block grouping** | Body-pitch reference per type size instead of a page-wide median. Block boundaries now match the incumbent exactly on the documents that were failing. | +| **M2.d — the metric box** | The single largest find: x was read from the ink box while y came from the metric box. Fixing it took three documents out of the regression set at once. | +| **M2.e — span and space fidelity** | Spans end where styles end; generated spaces have real widths; invented end-of-line spaces dropped; justified text no longer doubles its spaces. Superscript, the last item, needed no code: measured, the writer never sees the parser's flag (§3.1). | +| **The line-box escalation** | Granted, built, measured, **reverted** — and the cause closed out as unreachable. §4. | +| **M2.g — the gate made worth trusting** | One product profile shared by API, CLI, CI and docs, replacing three that disagreed. An exact corpus manifest. A numeric per-document baseline for every gated metric, replacing a list of metric *names*. Both lanes gating. The parity policy as data with numeric floors and stale detection, so `continue-on-error` could come off the step. One `evidence.json`. A mutation test for every false-green path the old gate had. | + +--- + +## 3. Left to do, in order + +### 3.1 — Superscript (`M2.e` remainder) · **CLOSED by measurement, no code** + +`parse_pdfium.py` hardcodes `superscript=False`, and the plan was to implement +it. It does not need implementing, and the way to find that out was to measure +the level that matters rather than the level that looked wrong. + +`testkit/backend_superscript.py` compares both levels across the corpus: + +| level | what it compares | result | +|---|---|---| +| parse | spans the *parser* flags | `c2_paper2col` 3 (PyMuPDF) vs 0 (PDFium); every other document 0 vs 0 | +| **layout** | runs the *writer* receives, after `normalize()` + `infer()` | **3 vs 3 on `c2_paper2col`, identical text; 16 of 16 documents agree** | + +`dialect._merge_row_lines` and `infer` both promote a small fragment sitting +above its host line's baseline, measured from geometry inside the em box, and +neither looks at the backend. So the parser flag is not load-bearing: it never +reaches a DOCX. One corpus document in sixteen even has a superscript, and its +flags survive the swap. + +**Done. Verdict recorded, no backend change made:** + +```bash +python testkit/backend_superscript.py +``` + +This is the cheaper half of a habit worth keeping — before implementing a +missing feature in a component, measure whether anything downstream consumes it. + +### 3.2 — The flip and the relicence (`M2.f`) · *one session, mechanical* + +This is the milestone the whole project has been driving at. + +1. `pypdfium2` becomes the main dependency; `pymupdf` moves to a `[mupdf]` + extra. **`parse.py` is not deleted** — it becomes the extra's backend. +2. Re-freeze the goldens *from the pdfium backend*, with manifest; archive the + MuPDF goldens for diffing. Its own commit, per law 14. +3. `LICENSE` → Apache-2.0, add `NOTICE`, classifier swap, README licence + section rewritten — including the plain statement that installing the + `[mupdf]` extra makes the *combination* AGPL-governed for distribution. +4. Version → `0.2.0a1`. +5. Re-verify: clean venv *without pymupdf installed* converts a PDF; + `import exactdoc.convert` pulls in no `fitz`. + +**Done when:** the parity table from the canonical environment is in the PR +description, `git grep -il affero` returns only historical narrative and the +extra's documentation, and the gate is green. + +**Acceptance (amended, and this is why it is now reachable):** 0 regressions, +except `01_whitepaper_market` and `02_research_paper`, attributed in STATUS D2 +to a font-metric convention no permissive parser can reproduce. + +### 3.3 — Ship the alpha (`M3`) · *one session* + +1. **D8**: encrypted and truncated PDFs raise a clean `UnsupportedInputError` + with a non-zero exit, not a backend traceback. Both degenerate files go into + CI. +2. Package: name check, wheel + sdist, TestPyPI dry run, then PyPI. +3. Release notes that lead with what it is *measured not to do* as well as what + it does — corpus numbers and the 0/4 holdout in the same breath — plus one + side-by-side screenshot. +4. Invite the contribution the project actually needs: *a PDF that breaks it*. + +**Done when:** `pip install exactdoc` on a machine with no dev setup converts a +browser-printed PDF, and the PyPI page reads Apache-2.0, `3 - Alpha`, 0.2.0a1. + +--- + +## 4. Closed, and why they will not be reopened + +Recording these so nobody spends another session rediscovering them. + +- **The last two parity regressions.** `infer()` derives the page's vertical + origin from line-box tops — the one vertical quantity two correct parsers + legitimately disagree about, because each reads it from font-metric tables + the other does not have. Proven unreachable: pdfium exposes exactly one + vertical font metric and the parser already uses it; matching PyMuPDF means + vendoring MuPDF's base-14 table, which the licence plan forbids and which is + measurably version-dependent. A baseline-anchored origin was granted, built + and reverted — it reached 0.000pt backend agreement on 14 of 16 documents and + still cost the *incumbent* `c6_long` 0.76 → 0.45, because the spacing chain + downstream is itself calibrated on box tops. Fixing it properly means moving + the origin, `_para_box` and the `space_before` chain together: a real project, + not a patch. Instrument kept: `testkit/margin_probe.py`. +- **Structural convergence of the IR.** Finished, and *demonstrated* finished: + three separate structurally-confirmed changes scored exactly flat, and the + third explained the other two — justified text lets the renderer redistribute + inter-word space, so text- and span-level differences cannot reach + `within2pt` at all. +- **Matching PyMuPDF bit-for-bit** as a target. It refuses to reproduce three + PyMuPDF behaviours because they are bugs, and PyMuPDF's grouping is not stable + across its own point releases. The parity gate is the contract; the golden IR + is a microscope. + +--- + +## 5. After the release — the part that is genuinely open + +None of this blocks shipping. All of it is what stands between "a good alpha" +and "trust it with any PDF". + +| | state | +|---|---| +| **D1 — LaTeX/pdfTeX pagination** | The largest open defect and the reason the holdout is 0/4. Page counts inflate 25–90%; text survives 94–97%. Three attribution attempts each produced a partly-wrong answer. Needs writer-side instrumentation — per-element emitted-vs-source height accounting inside `docxout` — not another hypothesis. | +| **The holdout** | 0/4, and it is the honest generalisation number. It moves when D1 moves. | +| **D3 nested tables** | Inner borders misplaced, cell content merges. | +| **D4 rounded-corner cards** | `border-radius` makes the card a curve; the detector requires a rect. | +| **D5 letter-spaced headings** | Tracking-spaced text loses its spaces. | +| **D6 mixed page geometry** | Page size and orientation taken from page 1 for the whole document. | +| **D7 Google Docs** | The actual target renderer, and the least measured. The full-bleed cover band has *never* been checked in Docs — the single most likely silent failure of the stated product goal. | +| **D9 `w:shd` spam** | File-size smell, not a correctness bug. | +| **The vertical model** | §4's baseline-consistent rewrite. Would close the last two parity regressions and probably help everywhere. | + +**Honest distance to "fully working":** unknown, and anyone who gives you a +number for it is guessing. D1 alone has resisted three attributed attempts. What +*is* known is that it does not block a release, because the release does not +claim to have solved it — the README leads with the 0/4. + +--- + +## 6. How to tell if this is on track + +The project's own discipline, in four checks anyone can run: + +```bash +bash scripts/bootstrap.sh --strict && python testkit/gen_corpus.py testkit/adv --strict && python corpus/make_corpus.py +``` + +```bash +python testkit/corpus_manifest.py verify +``` + +```bash +python testkit/runall.py +``` + +```bash +python testkit/backend_parity.py +``` + +The first must produce all 16 documents — `--strict` makes a skip a failure, +because the un-strict version printed "the corpus is incomplete", exited 0, and +the gate then scored 8 documents against a 16-document baseline. The second +proves the corpus is the one the baseline describes. The third must print +`gate PASS` for **both** lanes. The fourth is the swap's verdict, and it is now a +required check rather than a report. + +Every number in STATUS.md traces to `testkit/batch/evidence.json`, which those +commands write and CI attaches to the run: commit, dependency versions, oracle +versions, the profile measured, the corpus manifest, both lanes and the parity +verdict, in one file. Prose in three documents drifted apart once — one said +`12 same / 2 better` where another said `13 same / 1 better` — and prose cannot +be diffed. diff --git a/SESSIONS.md b/SESSIONS.md new file mode 100644 index 0000000..d647da4 --- /dev/null +++ b/SESSIONS.md @@ -0,0 +1,1606 @@ +# Sessions + +One entry per working session, written **before** the work starts (protocol §12.1 +of the execution plan): goal, gate-before numbers, hypothesis → experiment → +expected movement, and the files the session intends to touch. Results are +appended to the same entry afterwards, including the ones that failed. + +The point is not bookkeeping. This project has twice produced a confident wrong +answer that survived because nobody had written down what they expected to see +before they saw it. + +--- + +## 2026-07-29 · M0 — identity and truth reset + +**Goal.** Make the repository's claims match its own measurements, and reset the +version to something that does not promise more than the evidence supports. + +**Gate before.** Not applicable — this session changes no code that any gate +measures. Recorded instead: the environment this session established, so later +sessions can tell whether a number moved because of a change or because of the +machine. + +| | | +|---|---| +| Platform | Windows 11, Python 3.13.12, uv 0.6.0 | +| Backend deps | pymupdf 1.28.0, pypdfium2 5.12.1 | +| Oracles | LibreOffice (`C:\Program Files\LibreOffice`), Chrome (system) | +| Corpus generated | 15/16 — `l1_word_native` failed (see below) | + +**Hypothesis → experiment → expected movement.** None. This is a documentation +and metadata session; the expected movement of every measured number is *zero*. +If any gate number moves, something was edited that should not have been. + +**Files intended.** `pyproject.toml`, `README.md`, `STATUS.md`, `FINDINGS.md` +(banner only), `SESSIONS.md` (new). + +**Outside the M0 allowlist, with justification (protocol §12.7):** `.gitignore` — +the execution plan and advisory notes sit untracked in the repo root and are +private working documents; one ignore line prevents a `git add -A` from +publishing them, which is the same class of accident the credentials patterns +already guard against. + +**Result.** + +- `pyproject.toml`: `0.2.0` → `0.1.0a1`, `Development Status :: 4 - Beta` → + `3 - Alpha`. Licence fields unchanged (AGPL is still true today). +- `README.md`: rewritten against STATUS.md. Every number in the claims ledger + (plan §16) resolved — see the table in this entry. +- `FINDINGS.md`: one banner line clarifying that "v1.1" is a pre-release + internal label, no other change (the file is frozen). +- `STATUS.md`: two inline notes where a figure predates the 16-document corpus + and was being read as current (D7's LibreOffice column, D9's denominator). + +Claims resolved: + +| Claim (before) | After | Source of truth | +|---|---|---| +| gate 15/18 | 13/16 refine lane, 12/16 no-refine | STATUS §1 | +| page count 17/18 | 15/16 refine, 13/16 no-refine | STATUS §1 | +| within2pt 40.4% | 51.0% refine, 36.1% no-refine | STATUS §1 | +| median drift 1.02pt | 0.69pt refine, 2.79pt no-refine | STATUS §1 | +| live text 96.9% | 96.5% | STATUS §1 | +| SSIM 0.809 mean | removed — an 18-document figure, and never the headline | STATUS §4.5 | +| 18 documents / five dialects | 16 documents / four dialects in the corpus | corpus contents, verified | +| pdfium "9 regressions" | 7 | STATUS D2 | +| `runall.py testkit/adv my_samples` | `runall.py testkit/adv corpus/pdfs` | `my_samples` is not in the repo | +| holdout | stated in the same table as the corpus numbers | STATUS §1 | + +**Acceptance (plan §7), each box with the evidence beside it.** + +- [x] `pyproject.toml` shows only the new values — `version = "0.1.0a1"`, + `Development Status :: 3 - Alpha`; no `0.2.0`, no `Beta`. + `uv run python -c "importlib.metadata.version('exactdoc')"` → `0.1.0a1`. +- [x] No numeric claim in README contradicts STATUS §1–§2 — the ten ledger rows + above are each resolved; a grep for the retired figures + (`15/18|17/18|18 documents|9 regressions|0.809|40.4|1.02pt|my_samples`) + returns nothing. +- [x] README shows corpus AND holdout numbers in the same top table. +- [x] Tag `v0.1.0a1` exists on a docs/metadata-only commit (`95fcb9d`). +- [x] Every command quoted in the README runs. Verified: + `uv run exactdoc ` → wrote a 37KB DOCX; corpus generators and + `golden_ir.py verify` → 7/7 both run here; the two harness commands carry + their prerequisites (`[test]` extra, LibreOffice, Chrome) in the sentence + that introduces them. + +**Defect found while setting up (deferred to M1, not fixed here).** +`gen_corpus.py`'s LibreOffice document (`l1_word_native`) is generated with +`-env:UserInstallation=file:///` + a **relative** path, which LibreOffice +resolves against the filesystem root. It exits 1 and writes nothing, and the +function returns `None` without printing anything — so the corpus silently +comes back 15 documents instead of 16 and every downstream number is computed +over a different corpus than the one recorded. `harness.py` does not have this +bug (it builds the profile path from `tempfile.gettempdir()`, absolute). +This is exactly the M1 failure mode: a gate that quietly measures something +else. Fixed in the M1 entry. + +--- + +## 2026-07-29 · M1 — make the measurement machinery survive a fresh clone + +**Goal.** A fresh clone on a clean machine can run the gate, and a missing +oracle degrades into a printed skip list instead of a traceback or a silently +smaller corpus. + +**Gate before** (this machine, Windows, PyMuPDF default — the environment the +goldens were frozen on): + +| Measurement | Result | +|---|---| +| `golden_ir.py verify` | **7/7** | +| Corpus generated | **15/16** — `l1_word_native` missing, silently | +| `runall.py` lanes | not yet run here | +| `backend_parity.py --refine 3` | not yet run here | + +**Hypotheses → experiments → expected movement.** + +1. *The l1 gap is the relative `-env:UserInstallation` URL, not a broken + LibreOffice.* Experiment: run the same soffice command with an absolute + profile URL. Expected: `_l1.pdf` appears. **Already confirmed** during M0 + setup — the plain invocation wrote a 66KB PDF; the relative one exits 1. + Expected movement after the fix: corpus 15 → 16 documents. No fidelity + number should move for the other 15. +2. *The goldens are environment-pinned in practice but not in name.* Experiment: + freeze on this Windows machine (already done historically → 7/7 here) and + verify inside an `ubuntu:24.04` container provisioned exactly like + `gate.yml`. Expected: fewer than 7/7 on Linux, on documents whose *producer* + is deterministic — i.e. drift attributable to the environment, not the + parser. If Linux reproduces 7/7, the plan's §3 finding does not hold here and + the manifest is a precaution rather than a fix; either way the manifest gets + written, and which it is gets recorded. +3. *`corpus/make_corpus.py` still carries the `•` mojibake (FINDINGS §2.7).* + Experiment: read the bytes. **Falsified** — the bullets are `E2 80 A2`, + correct UTF-8. No change needed, and no golden churn incurred for one. + +**Verification environment.** Docker is available on this machine, so the +"clean container" acceptance criterion is executed literally, in +`ubuntu:24.04` — the same image family `gate.yml` runs on — rather than +deferred to a CI run nobody can see yet. That container is also the canonical +environment for anything the plan says must be frozen or baselined on Linux. + +**Files intended.** `testkit/gen_corpus.py`, `testkit/golden_ir.py`, +`testkit/README.md`, `testkit/golden/*` (only in a separate, justified commit), +`scripts/bootstrap.sh` (new), `.github/workflows/gate.yml`, `README.md`, +`STATUS.md`. **Forbidden this milestone:** anything under `exactdoc/`. + +### Mid-milestone addition: `harness.py` (a metric, not a threshold) + +Running the two lanes on Linux produced **12/16** where Windows records 13/16. +The extra failure is `c4_i18n`, on `doc_recall` 0.8298 / `word_recall` 0.8298. +Chasing it produced one falsified hypothesis and one measured cause. + +*Hypothesis A (falsified).* The container lacks CJK fonts, so the corpus +document or the oracle's render loses glyphs. Experiment: count characters per +script in the source and in the render-back; then install `fonts-noto-cjk` and +re-run. Result: **the source PDFs are character-identical between Windows and +Linux** (82 ideographs / 37 Hangul / 33 Kana / 88 Arabic / 71 Hebrew on both), +**the render-back carries every one of them**, and installing the fonts moved +`doc_recall` by exactly 0.0000. Not fonts. + +*Hypothesis B (measured, confirmed).* The metric cannot see the text it is +counting. `page_words` tokenises with PyMuPDF's `get_text("words")`, which +splits on whitespace — and Chinese, Japanese and Korean do not use any. A +"word" is therefore an entire rendered line, up to 32 characters; LibreOffice +re-wraps that line one character differently and the token no longer matches, +though every character is present. Evidence: of 94 source tokens, 16 go +unmatched, and **all 16 are Hangul (11), CJK (4) or Kana (1)** — zero Latin, +zero Arabic, zero Hebrew. Mean unmatched token length 9.3 characters against +4.4 for matched ones. + +So the gate fails a correct conversion, for a reason that depends on which +font the renderer wrapped with — which is why Windows passes and Linux does +not. That is not a threshold to tune; it is a measurement that is wrong. + +*Change.* Tokenise scriptio-continua runs (CJK ideographs, Kana, Hangul) per +character in `harness.page_words`, with the bbox divided across them. Scripts +that do use spaces are untouched. + +*Expected movement, written before running.* `c4_i18n` `doc_recall` and +`word_recall` 0.83 → ≈1.00, and the document passes the gate: refine lane +12/16 → 13/16 on Linux, matching Windows. `within2pt` for `c4_i18n` will also +move, because the matched population changes. **No other corpus document may +move at all** — none of the other 15 contains a CJK, Kana or Hangul character. +A change anywhere else means this fix is wrong, not that it is generous. + +*Result.* Held exactly. Re-scoring the existing renders — same DOCX, same +render-back on disk, so only the metric changed — moved `c4_i18n` `doc_recall` +0.8298 → 1.0000 and `within2pt` 0.3333 → 0.4160, and left all 15 other +documents identical to four decimal places on every metric. `1 document(s) +moved`. The refine lane went 12/16 → 13/16. + +### Second mid-milestone finding: the gate could never pass + +Dropping `continue-on-error` was blocked by something other than calibration. +`runall.py` exits 1 if *any* document misses a threshold, and three never have +(`c3_tables` D3, `c5_graphics`, `04_exec_brief` live-text 0.941 vs 0.95). So the +gate had returned non-zero on every run ever made, on both platforms — the CI +flag was not hiding an uncalibrated threshold, it was hiding a check that could +not pass. Fixed by gating on the delta against a recorded per-lane baseline +(`testkit/gate_baseline.json`), including treating a *stale* record — a document +that passes while the record says it fails — as a failure, since a record that +over-permits silently re-admits the regression it exists to catch. + +**Acceptance (plan §8), each box with the evidence beside it.** + +- [x] Clean container: provision → generate → exit 0. Executed literally, in + `ubuntu:24.04` populated from `git archive HEAD` (a fresh clone with no + Python at all): `bash scripts/bootstrap.sh` → 6/6 capabilities OK; + `gen_corpus.py` → **11 PDFs**, `make_corpus.py` → 5, i.e. the full + 16-document corpus, page counts identical to Windows. +- [x] Without the oracles it yields the subset plus a printed skip list and + never a traceback — `tests/test_corpus_degradation.py`, both directions + (bare machine exits 0 having still produced the two pure-Python + documents; a `CHROME` pointing at nothing exits 1). +- [x] `golden_ir.py verify` → **7/7** on the CI environment, against re-frozen + manifest-carrying goldens. Also 7/7 on Windows, with the platform + difference named rather than presented as parser drift. +- [x] CI runs golden verify + two-lane runall + parity, thresholds pinned, no + `continue-on-error` on the lanes. **Confirmed on a real runner** — + [run 30455217670](https://github.com/ebt55/exactdoc/actions/runs/30455217670), + green in 3m21s, every step passing: + + Golden IR 7/7 documents match + lane norefine 12/16 pass · 12 known, 0 new, 0 stale + lane refine 13/16 pass · 9 known, 0 new, 0 stale + lane comparison norefine 13/16 0.366 0.9652 2.20 + refine 15/16 0.529 0.9652 0.68 + backend parity 8 regressions, 7 same, 1 better + + The baseline recorded in a local container transferred to GitHub's runner + with **0 new and 0 stale in both lanes**, and parity reproduced its count + exactly. That is the regression gate proving portable across two + independent Linux environments, which is the property it needed to have. +- [x] Numbers recorded in STATUS.md §1 as the Linux/CI baseline, beside the + Windows column. +- [x] A deliberately broken environment still produces a passing + corpus-generation run with an explicit skip report. +- [x] `pypdfium2` declared in `testkit/README.md`'s quick start; `SOFFICE` / + `CHROME` documented in a table. +- [x] Mojibake (plan §8.6): **not present** — bullets are `E2 80 A2`. No golden + churn spent on a defect that had already been fixed. + +**Closed later the same day.** This box was first recorded as unverified, +because "CI is green" is a claim only a CI run can make and `gh` was +unauthenticated at the time. The owner authenticated it; the branch was pushed, +PR #1 opened, and the run came back green. The box above carries its output. + +--- + +## 2026-07-29 · M2 — finish the licence swap + +**Goal.** `backend_parity.py --refine 3` reports **0 regressions**, so the +default parser can become pypdfium2 and the licence can become Apache-2.0. + +**Gate before.** Measured on the canonical Linux container, this session: + +| Measurement | Result | +|---|---| +| `backend_parity.py --refine 3` | *(running — recorded below)* | +| `golden_ir.py verify` | 7/7 | +| Gate lanes | 12/16 no-refine, 13/16 refine; 0 new, 0 stale | + +**The reframe this milestone starts from.** The parity gate is the contract; +the golden IR is a microscope. The two definitions of "correct" have already +diverged — the backend deliberately refuses to reproduce three PyMuPDF +behaviours because they are bugs (RTL visual order, dropped gradients, Calibri's +serif flag), and M1 measured a fourth reason: **MuPDF's grouping changes between +its own point releases** (1.26 puts `02_research_paper` p2 in 4 blocks, 1.28 in +7). Converging bit-for-bit on a target that moves with the dependency version is +not a finish line. Not worse on the rendered output is. + +**Working loops.** Inner: golden digest diff + `backend_geom.py`, seconds, no +oracle. Middle: `backend_parity.py --only `, one document in ~10s — to be +added first, since without it every hypothesis costs a full 16-document run. +Outer: the full parity run, which is the only thing that decides anything. + +**Order.** (1) `--only`, (2) the 9.B instrument `backend_spans.py` and the +diagnosis of the code-heavy pair, because it is the part the plan marks +*unattributed* and guessing at it is how this project has been wrong before, +(3) 9.A grouping convergence document by document, worst first, (4) 9.C +superscript, (5) 9.D flip and relicense. + +**Files intended.** `exactdoc/parse_pdfium.py`, `exactdoc/backend.py`, +`testkit/*`, and at 9.D `pyproject.toml`, `LICENSE`/`NOTICE`, docs. +**Forbidden:** `exactdoc/parse.py`, `infer.py`, `docxout.py`, `dialect.py` — if a +parity failure traces into the shared pipeline, stop and escalate rather than +tune the shared code to flatter one backend. + +**Gate before (measured).** `backend_parity.py --refine 3` on the container: +**8 regressions, 7 same, 1 better**. One more than the plan's 7 — +`02_research_paper` (w 0.76 → 0.57) is a regression here and was not in the +audit's list. The set: `01_whitepaper_market`, `02_research_paper`, +`03_tech_report_code`, `05_memo`, `c1_whitepaper`, `c6_long`, `c7_code`, +`c8_toc_links`. `04_exec_brief` is *better* under pdfium (0.22 → 0.34). + +### 9.B — the code-heavy pair, attributed + +Built `testkit/backend_spans.py` (new): pairs lines across backends by baseline +and x, then diffs span structure, text, injected space runs, mono flags and +style keys. Run on the two failing documents with two passing ones as controls. + +**All four of the plan's candidate hypotheses are wrong.** Measured: + +| | c7_code | 03_tech | f1 (control) | r1 (control) | +|---|---|---|---|---| +| space-run diff | **0%** | **0%** | 0% | 0% | +| text diff | **0%** | 36% | 40% | 42% | +| lines unmatched | **16/26** | 9/73 | 0/20 | 0/36 | + +Multi-space synthesis (§9.B.1) is not it — space runs agree on every line of +every document. Different text (§9.B.1's consequence) is not it either: the two +*passing* controls have 40% and 42% text differences, more than the failing +`c7_code`, which has none. `LINE_SPLIT_EM` (§9.B.2) is not splitting anything, +and `superscript` (§9.B.4) is unrelated. + +What it is: **PDFium does not report leading indentation, and PyMuPDF +synthesises it.** Verified against the raw character stream rather than by +reading grouping code — for ` def __init__(...)`, PDFium's first character is +`d` at x=93.17 with no space anywhere before it, while PyMuPDF reports the same +line beginning at x=72.25 with four leading spaces. PDFium *does* synthesise +spaces between characters (there is a gap to measure); at the start of a line +there is nothing to the left of the first glyph, so the indent is simply +absent. The line box then starts at first ink, the paragraph is written at the +wrong x, and every glyph on the line is displaced by the indent width. + +*Fixes made, each measured:* + +1. `_reconstruct_indents` — rebuild leading indentation for monospace runs + against the leftmost line of the run. +2. Excluding lines that share a baseline from those runs. Necessary: a + configuration table whose cells are monospace puts three on one baseline at + x=61/153/223, and read as a listing they were "indented" by 18 and 32 spaces. + Measured cost of the bug: `03_tech` 0.23 → 0.03. +3. `local_pitch` — the block splitter's gap threshold was multiplying a + **page-wide** median pitch. Page 1 of `03_tech` has fourteen distinct + pitches and a median of 22.0pt (the table's rows outnumber everything), so + the threshold was 35.2pt and the 23.0pt blank lines inside a code listing — + an unmistakable double of the listing's own 11.5pt — were swallowed, fusing + three PyMuPDF blocks into one. `BLOCK_GAP_FACTOR` is untouched: the factor + was never wrong, the statistic it multiplied was. + +*Result, isolated by an A/B on the same corpus and renders:* + +| document | indent OFF | indent ON | pymupdf | +|---|---|---|---| +| `c7_code` | 0.16 | **0.59** | 0.91 | +| `03_tech_report_code` | **0.23** | 0.02 | 0.46 | + +So the indent reconstruction is worth +0.43 on one document and −0.21 on the +other, and `local_pitch` alone is score-neutral on both (it reproduces the +0.16/0.23 baseline exactly). Both documents remain regressions either way, so +the *verdict count* is unmoved at this point. + +### `local_pitch` reverted — the subset run was hiding its real cost + +A two-document run cannot decide a page-wide change, so both variants were then +run over the whole corpus. That is what caught it: + +| document | baseline | local pitch | local pitch + indent | +|---|---|---|---| +| `02_research_paper` | 0.57 | **0.02** | **0.02** | +| `c5_graphics` (expected-div) | 0.24 | 0.66 | 0.66 | +| `03_tech_report_code` | 0.23 | 0.23 | 0.02 | +| `c7_code` | 0.16 | 0.16 | **0.59** | +| the other twelve | — | unchanged | unchanged | + +`local_pitch` costs `02_research_paper` **0.55** of within-2pt and buys nothing +on the count: 8 regressions before, 8 after, 8 with both. A local window inside +a dense two-column body finds a pitch small enough to cut paragraphs in half — +the mirror image of the problem it fixed on the code listing. **Reverted**, with +the reasoning left in the code where the next person will look for it. + +That is the second failed attempt at moving `03_tech_report_code`, so by §12.5 +this stops here rather than trying a third estimator. What is *kept* is the +attribution and the instrument: `backend_spans.py`, `--only`, and a named, +measured cause for a defect the register had carried as unexplained. + +### Owner decision: land the indent fix, and trace 03 downstream + +Escalated per §12.8 and the owner chose to land it and to lift the +forbidden-file rule for the trace. The trace was read-only in the end — it never +needed to edit the shared pipeline, because it found the cause in the parser. + +**Where 03 loses it.** Rendered x of the same source line, against a source x of +84.40: + +| | `policy=Policy...` | `return downstream...` | +|---|---|---| +| PyMuPDF render | 90.25 (+5.9) | 90.25 (+5.9) | +| PDFium render | 199.70 (+115) | 281.30 (+197) | + +Two lines with the *same* source x landing 80pt apart is not a drift, it is a +structural failure: the code listing is being laid out as flowing prose. Reading +the layout confirmed it — PyMuPDF builds that region as + + role=code rows=1 col_widths=[504.0] + para leading=11.50 line_breaks=True vis_lines=10 runs=15 + +and PDFium built it as `role=table`, two columns of 3.0pt and 501.0pt, three +paragraphs with `line_breaks=False`, so `detector = Detector(` and the four +lines beneath it were concatenated into one line of prose. + +**Why: the two backends disagree about what a stroked path's bbox means.** +`FPDFPageObj_GetBounds` returns the *ink envelope* — a stroked path inflated by +its line width in every direction — while PyMuPDF returns the geometric path: + +| path | PyMuPDF | PDFium | +|---|---|---| +| box border, 0.75pt | `x=54.00..54.00` (w 0.00) | `x=53.25..54.75` (w **1.50**) | +| callout accent, 3pt | `x=57.00..57.00` (w 0.00) | `x=54.00..60.00` (w **6.00**) | + +`infer.py`'s table detector reads that 1.5pt bar as a column boundary, which is +where the phantom 3pt first column came from. `_classify` already worked around +this for *orientation* by reading the path points instead of the bounds; taking +the bbox from the same place makes the workaround whole, and it is confined to +`parse_pdfium.py` — the shared pipeline needed no change at all. + +With the bbox taken from the points, PDFium builds the region **identically** to +PyMuPDF: `role=code`, one 504.0pt column, `line_breaks=True`, 10 visual lines, +15 runs. + +**And the rendered score still did not follow.** `03_tech` 0.02 → 0.03, +`c7_code` **0.59 → 0.30** with `word_recall` slipping 1.00 → 0.95. On the full +corpus — because a two-document subset had already misled this session once — +it is worse still: + + 8 regressions -> 9 + +with `01_whitepaper_market` pages 3/3 → 3/4 (w 0.31 → 0.01), `02_research_paper` +2/2 → 2/3, `c5_graphics` 1/1 → 1/2, and `r1_reportlab_report` newly a regression +at 0.57 → 0.38. **Reverted.** + +The reason is worth keeping. A stroked box's *ink envelope* contains its text, +while the geometric path is the centreline — so making the bbox faithful makes +containment tests fail at the edges and box detection starts losing boxes +(`c7_code` drops from two `TableEl`s to one). The convention is not +independently right or wrong; it has to match whatever the containment tests +were tuned against, and they were tuned against PyMuPDF's. + +That is the third time this session that a demonstrably more faithful IR scored +*worse*, which is a finding about the pipeline rather than about the parser, and +it is the strongest evidence yet for the plan's RC1: the downstream is tuned to +PyMuPDF's *shape*, including the parts of that shape that are arbitrary. Two +consequences for whoever picks this up: + +1. **The remaining regressions are unlikely to fall one parser fix at a time.** + Three separate faithfulness improvements each cost more than they paid. + `exp_regroup.py` already showed grouping fully recovers `c6_long` (0.23 → + 0.73) and `c8_toc_links` (0.63 → 1.00) — those two are the honest next + targets, because there the evidence says the downstream *agrees* with the + more faithful answer. +2. **A containment/tolerance audit of `infer.py` is the real unlock**, and it is + a shared-pipeline change that must be measured on *both* backends. The bar: + pymupdf's numbers may not move at all. + +**Gate after this session:** `backend_parity.py --refine 3` → **8 regressions, +7 same, 1 better** — unchanged in count from the session's start, with +`c7_code` 0.16 → 0.59 and `03_tech_report_code` 0.23 → 0.02 inside it. + +--- + +## 2026-07-29 · M2.b — page-space path geometry (plan v2) + +**Correction I am acting on.** The stroke-bbox experiment above was confounded +and my conclusion from it was wrong. I derived path bboxes from +`FPDFPath_GetPathSegment` points, which PDFium reports in **object space**, +without applying `FPDFPageObj_GetMatrix`. So the change did not give the corpus +PyMuPDF's convention; it gave every transformed path scrambled coordinates. +That, not "faithfulness is punished", is why pagination broke on exactly the +documents it broke on — and the inference I drew from it (that `infer.py`'s +tolerances are the blocker) has **no valid evidence behind it** and is +withdrawn. `infer.py` stays closed; M2.d is the only way in. + +The tell I missed: the two example paths in my own probe table were +identity-matrix paths. A microscope aimed at two objects cannot see a systematic +transform. Law 15 exists now because of this, and law 16 because the same shape +of error nearly landed twice. + +**Goal.** Put path geometry in page space throughout `parse_pdfium.py`: matrix +first, then centreline bboxes from the transformed points, then `_classify`, +`_rect_pts` and the frame-edge decomposition all reading the same space. + +**Gate before** (canonical environment, unchanged from the last session): + +| Measurement | Value | +|---|---| +| `backend_parity.py --refine 3` | **8 regressions, 7 same, 1 better** | +| `golden_ir.py verify` | 7/7 | +| Gate lanes | 12/16 no-refine, 13/16 refine; 0 new, 0 stale | +| `03_tech_report_code` | w 0.02 (pymupdf 0.46) | +| `c7_code` | w 0.59 (pymupdf 0.91) | + +**Hypotheses → experiments → expected movement.** Written before running, and +deliberately more falsifiable than "may move": + +- **H1 (the probe, and the gate on everything else).** On Chromium/Skia + documents ≥90% of path objects carry a non-identity matrix, and + matrix-transformed points reproduce `GetBounds` on ~100% of paths once the + stroke envelope is accounted for, while raw points reproduce it only on the + identity ones. There is a strong prior: `_page_chars` already compensates for + Chromium's 0.75 text matrix, so the same 0.75 should appear on paths. + **If H1 fails, the scrutiny's finding is wrong and I stop and report rather + than "fix" anything.** +- **H2 (structural, 03_tech).** With page-space centreline bboxes the 0.75pt + box border reports width ≤ 0.1pt, the phantom 3.0pt table column disappears, + and the region classifies `role=code` with `rows=1`, one ~504pt column, + `line_breaks=True` — matching PyMuPDF's layout dump. +- **H3 (03_tech score).** ≥ 0.23, i.e. it recovers at least the value it had + before the indent fix, because the phantom column was its attributed cause. + I expect better than that — 0.30–0.46 — since the indent fix is still in and + the two were fighting each other. +- **H4 (c7_code).** Holds at ≥ 0.55. Its box is Chromium-produced, so all four + of its paths currently run mixed-space classification; I do not predict a + direction for it beyond "does not regress". +- **H5 (the requirement).** Count ≤ 8 and **no new regression documents**. + Chromium documents c1/c6/c8 run 100% mixed-space classification today, so + they may move either way; movement in either direction is informative, a new + regression is a failure. +- **H6 (invariance).** Golden IR stays 7/7 and the pymupdf column of the parity + table is unchanged — trivially, since no shared code is touched, but checked + rather than assumed. + +**Files intended.** `testkit/backend_paths.py` (new probe, committed first and +alone, per M2.b's build guideline and law 15), then +`exactdoc/parse_pdfium.py`. **Forbidden:** `exactdoc/parse.py`, `infer.py`, +`docxout.py`, `dialect.py`. Baselines (`gate_baseline.json`, `testkit/golden/*`) +are not expected to change at all; if one must, it is its own commit (law 14). + +### H1 — confirmed, and reproduced independently + +`testkit/backend_paths.py` compares, per path object, the raw-points bbox, the +matrix-transformed-points bbox and `FPDFPageObj_GetBounds`. A geometric bbox +"reconstructs" GetBounds when it lands within 0.6pt after the stroke envelope is +added back. + +| document | paths | non-identity matrix | raw reconstruct | worst raw miss | matrix reconstruct | +|---|---|---|---|---|---| +| `01_whitepaper_market` | 41 | 34 | 7 | 588.00pt | **41** | +| `02_research_paper` | 13 | 13 | 0 | 420.00pt | **13** | +| `03_tech_report_code` | 46 | 44 | 2 | 692.00pt | **46** | +| `04_exec_brief` | 18 | 16 | 2 | 590.00pt | 17 (0.60pt) | +| `05_memo` | 1 | 1 | 0 | 623.20pt | **1** | +| `c1_whitepaper` | 43 | **43** | 0 | 758.16pt | **43** | +| `c2_paper2col` | 5 | 5 | 0 | 578.00pt | **5** | +| `c3_tables` | 342 | **342** | 0 | 1680.00pt | **342** | +| `c5_graphics` | 9 | **9** | 0 | 646.50pt | **9** | +| `c6_long` | 50 | **50** | 0 | **5438.00pt** | **50** | +| `c7_code` | 4 | **4** | 0 | 382.00pt | **4** | +| `c8_toc_links` | 3 | **3** | 0 | 226.25pt | **3** | +| `f1_fpdf_brief` | 12 | 0 | **12** | 0.00pt | **12** | +| `l1_word_native` | 11 | 0 | **11** | 0.00pt | **11** | +| `r1_reportlab_report` | 14 | 14 | 0 | 377.40pt | **14** | +| **corpus** | **612** | **578** | **34** | — | **611** | + +The correspondence is exact: raw points reconstruct GetBounds on **34** paths, +and there are **34** identity-matrix paths in the corpus. Every Chromium +document is 100% non-identity. Worst raw miss is 5438pt on a Letter page. + +These numbers reproduce plan v2 §4's table cell for cell on every column it +reports (`01_whitepaper` 34/41 and 7; `03_tech` 44/46 and 2; `c1` 43/43 and 0; +`c5` 9/9 and 0; `c7` 4/4 and 0) from an independently written probe. The +scrutiny's finding is confirmed, not taken on trust — **H1 holds and the work +proceeds.** + +One path, in `04_exec_brief`, misses by 0.60pt after transformation; the other +611 land within 0.12pt and most at 0.00. Bezier control points hull wider than +the drawn curve, which is why the change below keeps `GetBounds` for curves. + +**A probe bug found and fixed before the table was trusted.** The first run +reported a worst miss of exactly 1.00pt on hundreds of paths — a suspiciously +round constant. Cause: PDFium reports a stroke *width* of 1.0 on fill-only +objects, and the probe was inflating every filled rectangle by it. The envelope +is now added only when the object is genuinely stroked (draw mode + non-zero +stroke alpha), with the width scaled by the matrix like everything else. Worth +recording because it is the same class of error as the one being corrected: +a number read from this API means nothing until you know what it is measured +in and when it applies. + +### The change, and its DrawCmd-level diff + +`_page_paths` now applies `FPDFPageObj_GetMatrix` to every segment point before +the y-flip, derives the path bbox from those transformed points (keeping +`GetBounds` for curves, whose control points hull wider than the drawn curve, +and for paths with no points), and scales the stroke width by the matrix too. +`_classify`, `_rect_pts` and the frame-edge decomposition consume the same +points, so the mixed-space logic is gone rather than worked around. + +Before/after taken from a **git worktree of the pre-change commit**, so both +sides are real code reading byte-identical PDFs: + +| document | draws | shape changes | bboxes moved | worst | stroke there | +|---|---|---|---|---|---| +| `01_whitepaper_market` | 41→41 | none | 25 | 3.00pt | 3.00 | +| `02_research_paper` | 13→13 | none | 9 | 0.90pt | 0.90 | +| `03_tech_report_code` | 46→46 | none | 35 | 3.00pt | 3.00 | +| `04_exec_brief` | 18→18 | none | 5 | 3.50pt | 3.50 | +| `05_memo` | 1→1 | none | 1 | 0.80pt | 0.80 | +| `c1_whitepaper` | 43→43 | none | 0 | — | — | +| `c2_paper2col` | 5→5 | none | 0 | — | — | +| `c3_tables` | 342→342 | none | 0 | — | — | +| `c5_graphics` | 9→9 | none | 2 | 0.75pt | 1.00 | +| `c6_long` | 50→50 | none | 0 | — | — | +| `c7_code` | 4→4 | none | 2 | 0.76pt | 1.00 | +| `c8_toc_links` | 3→3 | none | 0 | — | — | +| `f1_fpdf_brief` | 12→12 | none | 12 | 0.57pt | 0.57 | +| `l1_word_native` | 10→10 | none | 10 | 1.00pt | 1.00 | +| `r1_reportlab_report` | 14→14 | none | 10 | 0.40pt | 0.40 | +| **total** | **611→611** | **none** | **111** | | | + +**No path changed shape or disappeared**, and every moved bbox shrank by at most +its own stroke width — which is exactly what removing an ink envelope should +look like. One exception, checked rather than waved through: a `complex` path in +`04_exec_brief` moved 2.60pt against a 2.00pt stroke. It is the line chart's +series polyline, and its new bbox (`x=100.0..470.0`) lands precisely on the +data-marker centres (markers at 97.4..102.6 → centre 100.0; 467.4..472.6 → +centre 470.0). The excess over half the stroke width is the **miter join** at +the sharp vertices, which extends further than the stroke itself. Explained. + +The Chromium documents move zero bboxes because Chromium draws its borders as +*filled* rectangles, where the envelope and the path coincide. Their paths were +still being classified in the wrong space, which is what the change fixes for +them. + +### H2 — confirmed, exactly + +`03_tech_report_code`'s border geometry is now identical to PyMuPDF's, coordinate +for coordinate: + +``` +PyMuPDF vline x=54.00..54.00 y=354.70..485.70 w=0.00 lw=0.75 +PDFium vline x=54.00..54.00 y=354.70..485.70 w=0.00 lw=0.75 +PyMuPDF vline x=57.00..57.00 y=493.70..538.70 w=0.00 lw=3.00 +PDFium vline x=57.00..57.00 y=493.70..538.70 w=0.00 lw=3.00 +``` + +and the code box classifies the same way, with the phantom 3.0pt column gone: + +``` +PyMuPDF role=code rows=1 col_widths=[504.0] leading=11.50 line_breaks=True vis_lines=10 runs=15 +PDFium role=code rows=1 col_widths=[504.0] leading=11.50 line_breaks=True vis_lines=10 runs=15 +``` + +**H6 — confirmed.** Golden IR 7/7, purity 16/16; no shared code was touched. + +### The full gate, and the hypothesis I got wrong + +`backend_parity.py --refine 3`, canonical environment: **8 regressions, 8 same, +0 better** — count held, no new regression documents. + +| document | before | after | Δ | +|---|---|---|---| +| `c7_code` | 0.59 | **0.76** | **+0.17** | +| `03_tech_report_code` | 0.02 | 0.05 | +0.03 | +| `04_exec_brief` | 0.34 *(better)* | 0.20 *(same)* | **−0.14** | +| `01_whitepaper_market` | 0.31 | 0.31 | — | +| `02_research_paper` | 0.57 | 0.57 | — | +| `05_memo` | 0.49 | 0.49 | — | +| `c1_whitepaper` | 0.00 | 0.00 | — | +| `c2_paper2col` | 0.21 | 0.21 | — | +| `c3_tables` | 0.00 | 0.00 | — | +| `c5_graphics` | 0.24 | 0.24 | — | +| `c6_long` | 0.21 | 0.21 | — | +| `c8_toc_links` | 0.54 | 0.54 | — | +| `f1_fpdf_brief` | 0.60 | 0.60 | — | +| `l1_word_native` | 0.03 | 0.03 | — | +| `r1_reportlab_report` | 0.57 | 0.57 | — | + +**Scorecard against what I wrote before running:** + +| | prediction | outcome | +|---|---|---| +| H1 | ≥90% non-identity on Chromium; matrix reconstructs, raw does not | ✅ 100% on every Chromium doc; 611/612 vs 34/612 | +| H2 | border w ≤ 0.1pt, phantom column gone, `role=code` | ✅ exact coordinate match with PyMuPDF | +| H3 | `03_tech` ≥ 0.23, expect 0.30–0.46 | ❌ **0.05** | +| H4 | `c7_code` holds ≥ 0.55 | ✅ 0.76 | +| H5 | count ≤ 8, no new regressions | ✅ 8, none | +| H6 | golden 7/7, pymupdf lane unmoved | ✅ | + +**H3 is the one that matters, and it failed.** The phantom column was fixed — +H2 proves it structurally and perfectly, the layout dump is now +indistinguishable from PyMuPDF's — and `03_tech` moved 0.02 → 0.05. So the +phantom column was *a* cause of that document's drop but not the dominant one. +The attribution in the previous session was incomplete, and I should not have +predicted a full recovery from a structural match: **matching the structure of +one region does not bound the error of the page.** Whatever else is wrong with +`03_tech` is still unnamed, and naming it is M2.d's job, not a guess here. + +**The trade, stated explicitly (law 17).** `04_exec_brief` loses 0.14 and its +*better* verdict, landing level with PyMuPDF (0.20 against 0.22) instead of +ahead of it (0.34). That document is the line chart, and its series polyline is +exactly the path whose bbox shrank by the miter join. The old number came from +an inflated chart bbox — a figure region larger than the drawing — so the +document was scoring *better* off a geometric error. I do not think a score +earned that way is worth keeping, and it is not a regression either way. Net +across the corpus: +0.17 and +0.03 on two regression documents, −0.14 on a +non-regression one, thirteen documents pinned exactly, and a whole class of +coordinate-space confusion removed from the parser. + +--- + +## 2026-07-29 · M2.c — grouping convergence: `c6_long`, `c8_toc_links` + +**Goal.** Both documents leave the regression set. The evidence for picking +these two is `exp_regroup.py`: grafting PyMuPDF's block boundaries onto pdfium's +geometry recovers `c6_long` 0.23→0.73 and `c8_toc_links` 0.63→1.00, so on these +the downstream *agrees* with the more faithful answer — unlike `03_tech`, where +a perfect structural match moved the score by 0.03. + +**Gate before** (canonical environment, after M2.b): + +| | pdfium | pymupdf | gap | +|---|---|---|---| +| `c6_long` | 0.21 | 0.76 | **0.55** | +| `c8_toc_links` | 0.54 | 1.00 | 0.46 | +| whole gate | 8 regressions, 8 same, 0 better | | | + +Worst first, so `c6_long` leads. + +**Method, per §9.A — and no hypothesis yet, deliberately.** The rule change is +not allowed to precede the pattern. Step 1 is a per-page block-boundary diff +(which lines each backend starts a block at), step 2 is *naming* what the +disagreement is, step 3 is the rule and its predicted effect on the other 15 +documents, written before the gate runs. §9.A also bans tuning a threshold +before plotting the two distributions it separates, and the `local_pitch` dead +end from the previous session is a standing reminder that a block-split +estimator must be robust in both directions. + +**Files intended.** A block-diff instrument in `testkit/`, then +`exactdoc/parse_pdfium.py`. **Forbidden:** `parse.py`, `infer.py`, `docxout.py`, +`dialect.py`. Baselines only in their own commit (law 14); full corpus decides +(law 16). + +### Step 1 — the pattern, named + +A block-boundary diff (pair lines across backends, compare which of them each +backend starts a block at) gives one pattern and only one: + +| document | disagreements | direction | +|---|---|---| +| `c6_long` | 72 of 201 lines | **pdfium MERGES where PyMuPDF splits** — 72, and 0 the other way | +| `c8_toc_links` | 3 of 17 lines | same, 3 and 0 | + +With the context, the pattern names itself: pdfium fuses *consecutive +paragraphs and list items*. + +``` +pdfium MERGES p1 gap=23.2 prev |measure only average relevance will not see it.| + this |The mitigation is unglamorous. Chunk boundaries…| +pdfium MERGES p1 gap=19.5 prev |Point one for section 1.| + this |Point two for section 1, somewhat longer so that…| +pdfium MERGES p1 gap=18.0 prev |1. Motivation| + this |2. Architecture| +``` + +Body pitch on these pages is ~15pt, the boundaries are 18.0–23.2pt, and the +shipped rule allows anything up to `median_pitch × 1.6` ≈ 24pt into the same +block. So the paragraph boundary falls inside the tolerance. + +### Step 2 — the distributions, before any threshold is touched (§12.6) + +`testkit/block_gaps.py` (new) labels every consecutive pdfium line pair with +PyMuPDF's answer — same block or not — and plots `gap / reference` for three +candidate references. + +**Per document, every reference separates perfectly. Corpus-wide, none does**, +because each document's clean split sits at a *different* ratio: + +| document | separable on `p20`? | its own cut | +|---|---|---| +| `c6_long` | yes, 0 of 194 wrong | 1.05 | +| `c8_toc_links` | yes, 0 of 16 | 1.05 | +| `f1_fpdf_brief` | yes, 0 of 11 | 1.00 | +| `r1_reportlab_report` | yes, 0 of 23 | 1.00 | +| `l1_word_native` | yes, 0 of 16 | **1.24** | +| **corpus (685 pairs)** | **no** | best fixed 1.11, still 135 wrong | + +That is the §12.6 answer in full: the decision is well-posed *locally* and the +global constant is what is wrong. Scoring candidate rules against PyMuPDF's +labels: + +| rule | wrong | +|---|---| +| shipped: `gap ≤ median × 1.60` | **355/685 (52%)** | +| `gap ≤ p20 × 1.60` | 278 (41%) | +| `gap ≤ p20 × 1.30` | 178 (26%) | +| **`gap ≤ p20 × 1.15`** | **140 (20%)** | +| `gap ≤ p20 × 1.05` | 154 (22%) | +| adaptive: per-page Otsu cut on the page's own gaps | 322 (47%) | + +Two things worth stating plainly. The shipped rule is **wrong more often than +right** on this labelled set. And the *adaptive* estimator — the clever option, +the one I would have reached for after `local_pitch` — is worse than a fixed +factor on a better reference. Measuring it cost minutes; implementing it would +have cost a session. + +*(Caveat, stated because the number is startling: this scores the `else`-branch +condition applied uniformly to every consecutive pair, while the shipped +`_build_blocks_one` has other branches in front of it — the same-baseline case, +the size-change split. So 52% overstates the shipped parser's real error rate. +It is a proxy for ranking rules, not a measurement of the parser. The gate is +the measurement.)* + +**Why `p20` rather than the median, as an argument and not a fit.** The +reference is supposed to stand for the *intra-paragraph* line pitch. The median +of all gaps includes the boundary gaps themselves, plus table-row pitches, so it +is biased upward by exactly the quantity it is trying to exclude. The 20th +percentile approximates the tightest recurring pitch on the page, which is what +body text sets. The factor 1.15 sits mid-range of the per-document optima +(1.00–1.24) observed above. + +### Step 3 — the change, and what I expect of it (written before running) + +`_build_blocks_one`: reference becomes the 20th-percentile gap instead of the +median, and `BLOCK_GAP_FACTOR` 1.6 → 1.15. + +- **`c6_long`** recovers substantially; `exp_regroup` put full grouping recovery + at 0.73, so I predict **≥ 0.50** (from 0.21). +- **`c8_toc_links`** predict **≥ 0.80** (from 0.54). +- **`l1_word_native`** is the document I expect to suffer: its own optimum is + 1.24, above the 1.15 being adopted, so it may over-split. It sits at 0.03 + against PyMuPDF's 0.01 and is *same*, so there is room, but if anything turns + into a new regression I expect it here. +- **Everything else** should hold. This is a global change to every document's + blocking, so "should" is doing real work in that sentence — the full gate + decides, and the requirement is unchanged: **count ≤ 8, no new regressions**. + +### Result — 8 regressions → 6, and both predictions narrowly missed + +Block boundaries first: on both target documents pdfium now agrees with PyMuPDF +on **every single boundary** — `c6_long` 201 of 201 lines, `c8_toc_links` 17 of +17, from 72 and 3 disagreements. Grouping on these two is finished. + +`backend_parity.py --refine 3`, canonical environment: **6 regressions, 10 same, +0 better.** + +| document | before | after | Δ | verdict | +|---|---|---|---|---| +| `c6_long` | 0.21 | **0.46** | **+0.25** | still regression (pymupdf 0.76) | +| `c8_toc_links` | 0.54 | **0.78** | **+0.24** | still regression (pymupdf 1.00) | +| `01_whitepaper_market` | 0.31 | **0.48** | **+0.17** | still regression | +| `05_memo` | 0.49 | **0.64** | **+0.15** | **left the set** — equals pymupdf exactly | +| `c1_whitepaper` | 0.00 | **0.12** | **+0.12** | **left the set** (pymupdf 0.18) | +| `c7_code` | 0.76 | 0.72 | −0.04 | still regression | +| `c2_paper2col` | 0.21 | 0.20 | −0.01 | same | +| the other 8 | | | — | unchanged | + +**Scorecard.** + +| | prediction | outcome | +|---|---|---| +| `c6_long` ≥ 0.50 | | ❌ **0.46** — right direction, missed the number | +| `c8_toc_links` ≥ 0.80 | | ❌ **0.78** — same | +| `l1_word_native` is where a new regression would appear | | ❌ unchanged at 0.03 | +| no new regressions, count ≤ 8 | | ✅ **6**, none | +| pymupdf lane unmoved | | ✅ golden 7/7, purity 16/16 | + +Three of five predictions wrong, and the milestone still moved further than any +change so far. Worth being precise about what that means: I predicted the two +documents I was *aiming* at and missed both by 0.02–0.04, while the change's +biggest effects landed on `01_whitepaper_market` and the two documents that +actually left the set — **neither of which I predicted at all.** A global +change to blocking does not respect the document you had in mind. + +**The trade (law 17).** `c7_code` −0.04 and `c2_paper2col` −0.01. Neither +changes a verdict, and both are inside the parity comparator's 0.08 tolerance +band. `c7_code` remains far above where this session found it (0.16). + +**M2.c's own acceptance is not fully met, and I am not going to claim it is.** +It asks that both target documents *leave the regression set*; they did not, +they improved by ~0.25 each and stayed in. But their block boundaries now match +PyMuPDF's exactly, which means **grouping is exhausted as an explanation for +them** — whatever residual `c6_long` and `c8_toc_links` carry is a different +cause, and naming it is M2.d's re-attribution, not a second grouping attempt. +That is also why I am not iterating further here: §12.5 stops a second attempt +on the same metric, and in this case the instrument says there is nothing left +to converge. + +--- + +## 2026-07-29 · M2.d — re-attribution, and the cause behind three failed predictions + +**Gate before.** 6 regressions, 10 same, 0 better. Survivors: +`01_whitepaper_market` .48, `02_research_paper` .57, `03_tech_report_code` .05, +`c6_long` .46, `c7_code` .72, `c8_toc_links` .78. + +### The measurement that broke the case open + +`testkit/residual.py` (new) splits each document's placement error into the part +a second pass could remove — a per-page affine trend in y, a per-page constant +in x — and the part that survives it, then reports the **ceiling**: the +within-2pt a perfect anchoring fix could reach. + +| document | backend | median dx raw→resid | median dy raw→resid | within2 → ceiling | +|---|---|---|---|---| +| `c6_long` | pymupdf | 0.11 → 0.15 | 0.65 → 0.21 | 0.758 → 0.935 | +| `c6_long` | **pdfium** | **0.56 → 0.73** | 1.35 → 0.27 | 0.462 → 0.682 | +| `c8_toc_links` | pymupdf | 0.14 → 0.21 | 0.10 → 0.34 | 1.000 → 1.000 | +| `c8_toc_links` | **pdfium** | **0.61 → 0.76** | 0.10 → 0.32 | 0.784 → 0.763 | +| `c7_code` | pymupdf | 0.26 → 0.04 | 0.70 → 0.35 | 0.915 → 0.989 | +| `c7_code` | **pdfium** | **0.55 → 0.26** | 0.70 → 1.01 | 0.722 → 0.477 | + +The vertical axis is fine, and on `c6_long` it is *excellent* (1.35 → 0.27, more +systematic than PyMuPDF's own). **Every document's horizontal error is 2–5× +PyMuPDF's, and it does not shrink when a per-page constant is removed.** That is +diffuse sub-point horizontal error — precisely the shape that no structural fix +can touch, and precisely why three of them didn't. + +### The structural instruments say there is nothing left to fix + +| document | lines | span count diff | text diff | space-run diff | style diff | +|---|---|---|---|---|---| +| `c6_long` | 201/201 | **0%** | **0%** | **0%** | **0%** | +| `c8_toc_links` | 17/17 | **0%** | 12% | 12% | **0%** | +| `c7_code` | 26/26 | 73% | 0% | 0% | 73% | + +`c6_long`'s IR is identical to PyMuPDF's on every axis these instruments can +see — lines, spans, text, spaces, styles, and block boundaries (201/201) — and +it scores 0.46 against 0.76. The difference therefore lives *inside the +instruments' tolerances*: sub-point, per-line, horizontal. + +### Cause, measured + +`_page_chars` takes **y** from `FPDFText_GetLooseCharBox` — the font-metric box +— with a comment in the code explaining that the tight ink box made every line +start below the true ascent. It still takes **x** from `FPDFText_GetCharBox`, +the tight ink box. The bug was half-fixed. + +PyMuPDF reports every line of `c6_long` starting at exactly x=61.500, the pen +origin. pdfium reports the ink left edge, which moves with whichever glyph +happens to start the line: + +| first char | PyMuPDF x0 | pdfium x0 | delta | +|---|---|---|---| +| `L` | 61.500 | 63.194 | **+1.694** | +| `1` | 61.500 | 62.606 | +1.106 | +| `R` | 61.500 | 62.004 | +0.504 | +| `m` | 61.500 | 61.794 | +0.294 | +| `T` | 61.500 | 61.574 | +0.074 | +| `w` | 61.500 | 61.489 | −0.011 | + +That is the left side bearing, and it is a *different* number for every line. +Probing the API directly (law 15) settles which box is which: `loose.left` +equals `GetCharOrigin`'s x to **±0.000** on every character sampled, and equals +PyMuPDF's line x0 exactly, while the tight box is off by +0.074 to +1.694. + +**This one defect explains all three failed predictions on this branch.** It is +per-character, so no per-page correction removes it; it is present on every line +of every document, so structural convergence cannot reach it; and it is exactly +2–5× the horizontal error PyMuPDF carries, which is what the residual table +measures. + +**Hypothesis → change → expected movement.** Take x from the loose box, as y +already is. Expected: median |dx| falls to PyMuPDF's order (~0.1–0.3pt) on every +document, and within-2pt rises across the board — most on the documents whose +structure is already exact (`c6_long`, `c8_toc_links`). Risk to name in advance: +the space-synthesis thresholds (`SPAN_GAP_EM`, `SPACE_GAP_EM`, `LINE_SPLIT_EM`) +were calibrated against *ink* gaps, and advance boxes tile, so gaps shrink and +fewer spaces may be synthesised. The structural instruments will show that as a +text/space-run diff before the gate sees it — if they do, this needs splitting +into two coordinate systems rather than one. Requirement unchanged: **count ≤ 6, +no new regressions.** + +### Result — 6 regressions → 3 + +Line x0 agreement first: median delta **+0.368 → +0.000** on `c6_long` and +**+0.399 → +0.000** on `c8_toc_links`. The named risk did not materialise — +`c8_toc_links`'s text diff went 12% → **0%**, `c6_long` stayed at 0%, and +`c7_code`'s span fragmentation *improved* 73% → 65%. Advance boxes tile, so the +gap heuristics saw cleaner input rather than degraded input. + +`backend_parity.py --refine 3`: **3 regressions, 13 same, 0 better.** + +| document | before | after | Δ | | +|---|---|---|---|---| +| `03_tech_report_code` | 0.05 | **0.48** | **+0.43** | **left the set** — *above* pymupdf's 0.46 | +| `c6_long` | 0.46 | **0.76** | **+0.30** | **left the set** — equals pymupdf exactly | +| `c8_toc_links` | 0.78 | **1.00** | **+0.22** | **left the set** — equals pymupdf exactly | +| `c7_code` | 0.72 | **0.82** | +0.10 | still regression (pymupdf 0.91) | +| `c4_i18n` | 0.49 | 0.57 | +0.08 | expected-divergence | +| `01_whitepaper_market` | 0.48 | 0.53 | +0.05 | still regression (0.72) | +| `c1_whitepaper` | 0.12 | 0.15 | +0.03 | | +| `02_research_paper` | 0.57 | 0.57 | — | still regression (0.76) | +| the rest | | | ±0.01 | | + +**Scorecard.** I predicted median |dx| would fall to PyMuPDF's order and that +the documents with already-exact structure would gain most. Both held: +`c6_long` and `c8_toc_links` were the two structurally-exact documents and they +gained 0.30 and 0.22, landing on PyMuPDF's number *to the second decimal*. The +one I did not predict is `03_tech_report_code` at +0.43 — the document that had +resisted three previous fixes — which now scores **above** the incumbent. + +**Aggregate.** Mean within-2pt over all 16 documents: **0.384 → 0.461**, against +PyMuPDF's 0.511. The branch has now closed **75%** of the gap it started with +(0.312 → 0.461 against a 0.511 target). + +**What this says about the three failed predictions earlier on this branch.** +They were not failures of the fixes; they were masked by a per-character error +underneath them. The indent reconstruction, the page-space geometry and the +grouping convergence were all correct and all necessary — `c6_long` could only +land exactly on 0.76 because its blocks were already exactly right, and +`03_tech` could only reach 0.48 because its code box was already classified +correctly. Each looked disappointing in isolation and paid in combination. That +is worth remembering the next time a correct change measures flat. + +**Invariance:** golden IR 7/7, purity 16/16, pymupdf column unchanged. + +**Remaining three**, with the 0.08 comparator band: + +| document | pdfium | pymupdf | needs | +|---|---|---|---| +| `c7_code` | 0.82 | 0.91 | **0.02** | +| `01_whitepaper_market` | 0.53 | 0.72 | 0.11 | +| `02_research_paper` | 0.57 | 0.76 | 0.11 | + +--- + +## 2026-07-29 · M2.e — the last three + +**Gate before.** 3 regressions, 13 same, 0 better. + +**Horizontal is finished.** The residual table on the current renders: + +| document | median dx pymupdf | median dx pdfium | +|---|---|---| +| `01_whitepaper_market` | 0.30 → 0.31 | **0.26 → 0.29** | +| `02_research_paper` | 0.10 → 0.11 | **0.10 → 0.12** | +| `c7_code` | 0.26 → 0.04 | **0.27 → 0.04** | + +pdfium now matches or beats PyMuPDF horizontally on all three. Everything left +is vertical, or structural. + +**Three separate causes, each named before any rule is written.** + +1. **`c7_code` — span fragmentation on identical styles.** 26 lines, and PyMuPDF + emits 26 spans (1.00 per line) against pdfium's 105 (4.04). Of pdfium's 79 + intra-line span boundaries, **79 are between spans whose style keys are + identical**, every one at a gap of exactly 3.401pt — one space at that size. + `_build_lines` tests `gap > SPAN_GAP_EM` in the *same* condition as the style + change, and that test runs *before* the space-insertion branch, so a + space-sized gap ends the span instead of becoming a space. The line's text is + still right (text diff 0%), but it reaches the writer as four runs instead of + one, and LibreOffice lays fragmented runs out slightly differently. +2. **`01_whitepaper_market` — trailing spaces.** 25% of lines differ in text, + with space-run diff 0%: pdfium appends one trailing space PyMuPDF does not + (`|•|` vs `|•·|`, `|Tier|` vs `|Tier·|`, `|…Confidential|` vs + `|…Confidential·|`). PDFium's end-of-line generated space is being kept. +3. **`02_research_paper` — 4 missing lines** (93 against 89) plus the largest + vertical error left anywhere: median |dy| 1.29 against PyMuPDF's 0.04, and it + barely improves under a per-page affine fit (1.15). That is a different + problem from the other two and the hardest of the three. + +**Order and hypotheses.** Take them cheapest-first, one commit each. + +- **H1 (span splits).** A span should end where the *style* ends. A gap with + identical style on both sides should insert spaces and continue, which is what + PyMuPDF does. Note `LINE_SPLIT_EM` already ends the *line* at 1.10em, so any + gap still under consideration is small enough for spaces to bridge. + Expected: pdfium spans-per-line on `c7_code` 4.04 → ~1.0; `c7_code` within2pt + 0.82 → ≥ 0.88; text unchanged (0% diff must stay 0%). +- **H2 (trailing space).** Strip a single trailing generated space from a line. + Expected: `01_whitepaper_market` text diff 25% → near 0%; within2pt improves + by an unknown amount — I will not pretend to predict it, since a trailing + space affects placement only via wrap and alignment. +- **H3 (`02_research_paper`).** Unnamed as yet; diagnose after the first two, + since both change line construction and may move it. + +Requirement throughout: **count ≤ 3, no new regressions**, pymupdf lane +untouched. + +### H1 result — structurally exact, and score-neutral. The prediction failed. + +Two changes, interdependent and therefore one commit. Splitting spans on style +alone was *wrong on its own*: the gap it stopped consuming then reached the +space-synthesis branch and produced `def··rerank`, doubling every space +(c7_code text diff 0% → 65%). Chasing that exposed the real defect underneath — +PDFium reports a generated space's box as degenerate (`x..x` at one coordinate), +so the branch that inherits the previous character's end gives it 1.70pt where +its true advance is 5.10pt, and the remaining 3.401pt surfaces as a phantom gap +indistinguishable from positioned text. + +Giving the space one space-advance of width (capped at the next character) fixed +both. The cap matters: running it to the next character also closes *table cell* +gaps, which `LINE_SPLIT_EM` splits rows on — measured, that fused cells and cost +`01_whitepaper_market` 130 lines → 105 and `03_tech_report_code` 73 → 53. + +Structural convergence afterwards, on every document measured: + +| document | span count diff | text diff | space-run diff | +|---|---|---|---| +| `c7_code` | 65% → **0%** | 0% → **0%** | 0% → **0%** | +| `01_whitepaper_market` | 5% → **0%** | | | +| `02_research_paper` | 20% → **0%** | | | +| `03_tech_report_code` | 13% → **0%** | | | +| `c6_long`, `c8_toc_links` | **0%** | **0%** | **0%** | + +`c7_code` now emits 26 spans for 26 lines — exactly PyMuPDF's 1.00 per line, +down from 105. + +**And the gate did not move.** 3 regressions, 13 same. `c7_code` 0.82 before and +0.82 after; every other document unchanged except `r1_reportlab_report` +0.58 → 0.55 and `l1_word_native` 0.03 → 0.01 (which now equals PyMuPDF exactly). + +**H1 predicted `c7_code` ≥ 0.88. It scored 0.82. The prediction failed, and the +hypothesis behind it — that span fragmentation was costing placement — is +falsified.** Fragmented runs and merged runs lay out identically here; the +writer's output was already equivalent. + +**The trade (law 17), and why this is kept anyway.** It costs +`r1_reportlab_report` 0.03, changes no verdict, and buys no measured score. What +it buys is correctness that is not visible in this metric: without the +generated-space fix the parser emits *doubled spaces* in its text — content that +is simply wrong, and that `live_text_cov` cannot see because it strips +whitespace. It also removes 79 spurious runs from one document's DOCX. And this +branch has twice now seen structurally-correct changes measure flat and then pay +in combination (M2.b and M2.c both looked disappointing until the metric-box fix +landed). That is an argument from precedent, not proof, and it is labelled as +such. + +### H2 result — text converged, score flat, and the r1 loss came back + +PDFium synthesises a space at the end of a line where the producer merely +stopped drawing. PyMuPDF does not report it. Dropping it (after RTL reordering, +so "trailing" means the end of the logical text): + +| document | text diff before → after | +|---|---| +| `03_tech_report_code` | 33% → **0%** | +| `01_whitepaper_market` | 29% → **12%** | +| `02_research_paper` | 36% → **22%** | + +Gate: **3 regressions, 13 same** — unchanged again, with +`r1_reportlab_report` 0.55 → **0.58**, recovering exactly the 0.03 the previous +commit cost it. So the two commits together are score-neutral and leave four of +the six documents I have been working with byte-identical to PyMuPDF on lines, +spans, text, space runs, styles and block boundaries. + +### Where M2.e stops + +**Gate: 3 regressions, 13 same, 0 better.** Down from 8 at the session's start. + +| document | pdfium | pymupdf | structural diff remaining | +|---|---|---|---| +| `c7_code` | 0.82 | 0.91 | **none — 0% on every measure** | +| `01_whitepaper_market` | 0.53 | 0.72 | text 12% | +| `02_research_paper` | 0.57 | 0.76 | text 22%, 4 lines short (89 vs 93) | + +**The plateau is real and worth naming.** `c7_code` is now identical to PyMuPDF +on every axis every instrument in this repository can measure — 26 lines, 26 +spans, 0% text, 0% space runs, 0% style keys, block boundaries matching — and it +scores 0.82 against 0.91. Its residual decomposition says the horizontal error is +already PyMuPDF's (0.27 vs 0.26 raw, 0.04 vs 0.04 after fit) and its *vertical* +residual is better than PyMuPDF's (0.20 vs 0.35). It is within 0.02 of the +comparator's tolerance band and I cannot find a structural difference left to +close. + +That is the honest boundary of this approach: **structural convergence on the IR +is finished, and three documents remain.** What is left is vertical placement +under `--refine`, which is a writer/refiner interaction rather than a parser +one — `02_research_paper` carries median |dy| 1.29 against PyMuPDF's 0.04 and +barely improves under a per-page affine fit, which is the signature of a +different mechanism entirely, and it is also the document that is 4 lines short. + +Per §12.5 this is where I stop rather than try a third parser-side idea: two +hypotheses (H1 span fragmentation, H2 trailing spaces) were both structurally +confirmed and both scored flat. The next move needs new attribution, and on the +evidence it points outside `parse_pdfium.py` — which makes it an M2.d escalation +packet question, not another parser change. + +--- + +## 2026-07-29 · Decision-memo session 1 — `02_research_paper`, then the text diffs + +Following the memo's §5 sequencing and §6 kickoff. Target-selection rule +accepted: while any document in the regression set shows a structural diff, the +next target is the largest structural diff on the worst-gapped document. That +resolves the "third self-picked target" worry — the rule picks, not me. + +**Gate before.** 3 regressions, 13 same, 0 better. + +### The four missing lines — named, and both leading hypotheses falsified + +Evidence ask answered. They are all on **one baseline**: + +| page | y | x0..x1 | size | text | +|---|---|---|---|---| +| 1 | 585.30 | 378.55..415.75 | 9.5 | `decoding ` | +| 1 | 585.30 | 423.59..449.20 | 9.5 | `builds ` | +| 1 | 585.30 | 457.04..468.92 | 9.5 | `on ` | +| 1 | 585.30 | 476.76..548.00 | 9.5 | `rejection-sampling` | + +- **Memo hypothesis 1 (whitespace-only lines dropped by construction): + falsified.** None of them is whitespace, and the PyMuPDF IR for this document + contains **zero** whitespace-only lines. +- **Memo hypothesis 2 (superscript fragments): falsified.** All four are body + text at 9.5pt, the document's body size, and none is a marker. +- **What it actually is:** pdfium is not *missing* lines. **PyMuPDF is + fragmenting one.** These four are consecutive word-groups of a single + justified line, split at its stretched word gaps (7.84pt each, a constant + 0.83em). pdfium emits the whole line, `378.55..548.00`, as one Line — and + `LINE_SPLIT_EM` at 1.10em (10.45pt here) correctly declines to split at 7.84pt. + Every pdfium line matched a PyMuPDF line; there are **zero** lines in pdfium + that PyMuPDF lacks. + +So the "4 missing lines" is a **line-count difference in which pdfium is the +more faithful side**, not a defect. Nothing to fix, and I am not going to +reproduce a fragmentation to flatter a count. + +### The text diffs — one mechanism, quoted + +Same justified text, and this one *is* a defect: + +``` +mupdf |Speculative·decoding·accelerates·autoregressive·generation| +pdfium |Speculative··decoding··accelerates··autoregressive··generation| + +mupdf |with·a·large·one.·Fixed·draft·models,·however,·leave| +pdfium |with··a··large··one.··Fixed··draft··models,··however,··leave| + +mupdf |Priya·Raman···Diego·Álvarez···Hannah·Cole| runs=[3, 3] +pdfium |Priya·Raman··Diego·Álvarez··Hannah·Cole| runs=[2, 2] +``` + +Justified text stretches its word gaps. A space *character* is already present; +the stretched remainder still exceeds `SPACE_GAP_EM`, so the synthesis adds +another on top. The existing guard caps the addition at one for proportional +text (`n_sp = min(n_sp, 1)`) — which is exactly how every gap comes out as two +spaces instead of one. + +**Hypothesis → change → expected movement.** In proportional text a gap that is +already occupied by a space character should contribute **no** additional space; +MuPDF emits one space however far the gap is stretched. Monospace keeps the +existing behaviour, because there the count is load-bearing (code indentation) +and the earlier measurement stands. Expected: `02_research_paper` text diff +22% → near 0, `01_whitepaper_market` 12% → near 0; both are justified-text +documents and this is the whole of their remaining structural diff. Score +prediction, written before running and deliberately modest given the last two +flat results: **02 improves, because doubled spaces displace every word after +them on a justified line — unlike the fragmentation and trailing-space fixes, +this one moves ink.** Requirement unchanged: count ≤ 3, no new regressions. + +*(Counter-example noted and not swept under: `1··Introduction` → `1·Introduction` +runs the other way — PyMuPDF emits two spaces at a wide heading gap where pdfium +emits one. That is a second, rarer pattern with the opposite sign; it is left +alone this session rather than fitted, and recorded here so it is not lost.)* + +### Result — text converged again, score identical again. Prediction failed. + +| document | text diff | space-run diff | +|---|---|---| +| `02_research_paper` | 22% → **8%** | 22% → **7%** | +| `01_whitepaper_market` | 12% → **8%** | 5% → **1%** | + +**Gate: 3 regressions, 13 same — every single number identical to the previous +run.** `02_research_paper` 0.57 before and after; `01_whitepaper_market` 0.53 +before and after. + +I predicted this one would move, and said why: *"doubled spaces displace every +word after them on a justified line — unlike the fragmentation and +trailing-space fixes, this one moves ink."* **It does not, and now I know why:** +in justified text the renderer redistributes inter-word space to fill the +measure, so the *number* of spaces in the source has no effect on where the +words land. LibreOffice re-justifies to the same width whether the source says +one space or two. The doubled spaces were wrong content, and positionally inert. + +**That is three structurally-confirmed, score-flat hypotheses in a row** — H1 +span fragmentation, H2 trailing spaces, H3 justified spacing — and the third one +retroactively explains the first two. Text- and span-level differences in this +corpus do not reach `within2pt` at all, because the renderer normalises exactly +those degrees of freedom. §12.5 stops this line of work, and this time the stop +is principled rather than merely procedural: **the class of defect has been +shown not to matter to the metric.** + +**The trade (law 17):** no score movement, no regression, no verdict change. Kept +on the same grounds as the trailing-space fix — one space is the correct content +and two is not, `live_text_cov` strips whitespace so it cannot see the +difference, and a user opening the DOCX would. Structural fidelity is worth +having on its own terms; it is simply not what the last three documents are +losing on. + +**Structural convergence is now finished and demonstrated finished.** Every +document in the regression set is at or near 0% on every structural instrument, +and the remaining gaps are entirely vertical placement — which is memo §5 item 4, +gated behind the `c7_code` noise floor. + +### `c7_code` noise floor (memo §4) — step 3, not step 2 + +| refine | pymupdf | pdfium | gap | +|---|---|---|---| +| 0 | 0.56 | 0.38 | −0.180 | +| 1 | 0.91 | 0.82 | −0.090 | +| 2 | 0.91 | 0.82 | −0.090 | +| 3 | 0.91 | 0.82 | −0.090 | +| 3 (repeat) | 0.91 | 0.82 | −0.090 | + +The raw spread across configurations is 0.090, which touches the memo's ≥0.09 +step-2 trigger — but reading it that way would be wrong. Refine 0 is a different +*configuration* (no correction loop at all), not a noisy repeat of the same one. +At refine 1, 2 and 3 the numbers are **identical**, the repeat is bit-identical, +and the gap never changes sign. The harness is not noisy here; it is exact. +So: **step 3 — something systematic survives below the structural floor.** + +### And it was mine + +Per-word attribution (memo §4 step 3's suggested tool). The entire gap is **17 +words on exactly two source lines**, and pdfium's horizontal error accumulates +linearly along each: + +``` +word src y pymupdf dx,dy pdfium dx,dy +quality 103.9 (+0.03, -1.95) ( -2.62, -1.40) +degrades 103.9 (-0.03, -1.95) ( -5.31, -1.40) +non-linearly 103.9 (-0.08, -1.95) ( -7.99, -1.40) +... ... +embedding 103.9 (-0.43, -1.95) (-34.72, -1.40) +``` + +−2.67pt per word gap, perfectly linear — one space advance at that size. Words +clearing 2pt under PyMuPDF but not pdfium: **17. The other way round: 0.** + +The block diff named the cause: `pdfium SPLITS where PyMuPDF merges`, three +times, every one at exactly gap=15.0 with overlap 489.7 — the body-text pitch. +**This was my own M2.c change.** `c7_code` sets its code listings at an 11.25pt +pitch, which drags the page-wide 20th percentile *below* the body text's 15.0pt, +so body paragraphs split into one block per line, each became its own justified +paragraph, and a one-line justified paragraph is not stretched to the measure. +Exactly the mirror of the median's failure: dragged *up* by tables then, *down* +by code now. + +**Fix: compute the body pitch per type size.** Text of one size shares one +leading, so the reference lives with the text rather than with the page. It is +not the reverted sliding window — a window has no idea what it is averaging +over; a size bucket is a property of the text itself. Falls back to the page +percentile when a size has fewer than three samples. + +**Result: 3 regressions → 2, 13 same, 1 better.** + +| document | before | after | | +|---|---|---|---| +| `c7_code` | 0.82 | **0.91** | **left the set — equals PyMuPDF exactly** | +| `05_memo` | 0.64 | **0.88** | **BETTER than PyMuPDF's 0.64** | +| everything else | | | unchanged | + +Block boundaries: `c7_code` 23/26 → **26/26**, `c6_long` and `c8_toc_links` hold +at 201/201 and 17/17. + +**The memo's §4 owner decision is now moot.** `c7_code` needed 0.02 and gained +0.09; it sits *on* PyMuPDF's number. No `ACCEPTED_SHORTFALL` entry is required, +and I have not created the mechanism. + +**Remaining: 2.** `01_whitepaper_market` 0.53 vs 0.72, `02_research_paper` 0.57 +vs 0.76. Both are the vertical-placement question — memo §5 item 4. + +--- + +## 2026-07-29 · Decision-memo session 2 — the vertical question (read-only first) + +**Gate before.** 2 regressions, 13 same, 1 better. Only `01_whitepaper_market` +(0.53 vs 0.72) and `02_research_paper` (0.57 vs 0.76) remain, and for the first +time there is a single open line of attack rather than several. + +**Read-only, per memo §5 item 4 and §3.** No parser change is planned before the +histogram says what the error is shaped like. + +**Note on the memo's dissolution route.** §3 offered: *if the dy histogram is +bimodal with a mode near one leading, the four missing lines are the cause and +the dy question dissolves into Q3.* That route is closed — the four lines turned +out to be PyMuPDF fragmenting one justified line, with pdfium the more faithful +side, so there is nothing to close. The histogram is still the right first +measurement; it just cannot dissolve into that answer. + +**Prediction, written before running.** `02_research_paper`'s median |dy| is +1.29pt and its post-affine residual is 1.15pt — a per-page affine fit removes +almost none of it. A missing-line or wrap difference would show as a mode near +one leading (≈13pt at this document's 9.5pt type). 1.29pt is two orders below +that. So I predict: + +- **unimodal, not bimodal**, centred near 1–1.5pt, with no mass near 13pt; +- therefore **not** a line-count or wrap problem, but a small per-paragraph + anchoring offset — the `para_top = baseline − (leading − 0.21·size)` model or + `space_before`, quantised; +- and because it survives a per-page affine fit, it must vary *between* + paragraphs rather than accumulate down the page. + +If instead there is a mode near one leading, I am wrong and the cause is +structural after all. + +### Result — prediction confirmed, and the cause located + +The histogram is **unimodal with no mass near one leading**, exactly as +predicted. But the control is what makes it decisive — the two backends produce +*the same distribution, displaced*: + +| bucket | PyMuPDF | | bucket | pdfium | +|---|---|---|---|---| +| **+0.0** | **252** | → | **+1.5** | **225** | +| +1.0 | 35 | → | +2.5 | 47 | +| **+3.0** | **55** | → | **+4.5** | **55** | + +Every cluster displaced by exactly **+1.5pt**, and the 55-word cluster appears +with *identical count* on both sides. That is a constant offset, not scatter. +It is also why a per-page affine fit removes so little: a least-squares line +through a multi-modal distribution sits between the modes. + +**Where it enters.** Baselines are identical on every line (`dbase = +0.00`). +The line *boxes* are not: pdfium's y0 sits 0.57–2.60pt lower, scaling with type +size. `margin_t` is derived from the topmost line's box top, and comes out +**63.30 (PyMuPDF) against 64.90 (pdfium)** — a 1.6pt page-wide shift, which is +the +1.5 mode. + +**Why the boxes differ — and why this one cannot simply be "converged".** The +box is font-dependent in both, from *different metric sources*: + +| font | PyMuPDF up/size, down/size | pdfium up/size, down/size | +|---|---|---| +| Helvetica | 1.075, 0.299 | 0.905, 0.211 | +| Helvetica-Bold | 1.070, 0.307 | 0.905, 0.211 | +| Times-Roman | 1.053, 0.281 | 0.891, 0.215 | +| Times-Bold | 1.044, 0.341 | 0.891, 0.215 | +| Symbol | 1.010, 0.293 | **1.010, 0.293** | + +pdfium *is* reading font metrics (Helvetica and Times differ), just not the same +ones — and on Symbol, where both fall back to the embedded metrics, they agree +exactly. PyMuPDF's numbers are its own built-in base-14 table. Reproducing them +means vendoring MuPDF's private font metrics, which §13 forbids outright and +which this branch has already proved is version-dependent. + +**Causality tested, not assumed.** A labelled temporary experiment scaled the +box toward PyMuPDF's ratios (1.188 above the baseline, 1.417 below): + +| document | shipped | scaled | pymupdf | +|---|---|---|---| +| `02_research_paper` | 0.57 | **0.64** | 0.76 | +| `01_whitepaper_market` | 0.53 | 0.54 | 0.72 | + +So the box convention **is** a real cause, worth +0.07 on the document with the +worst vertical error — and it is **not the whole gap**: 0.64 is still 0.12 short, +and `01_whitepaper_market` barely moves, so it has a different problem again. +The experiment was reverted; a fitted pair of constants that does not even close +the gap is not something to ship. + +**Where this leaves M2.** The remaining two documents are not blocked on +anything structural in the parser — they are blocked on a design question the +parser cannot answer alone: + +> `margin_t` (and paragraph anchoring) is derived from line-box *tops*, a +> quantity on which two correct parsers legitimately disagree because it comes +> from font-metric tables they do not share. Baselines, which both report +> identically to 4,734 of 4,734, carry the same information without the +> disagreement. + +That is a question about `infer.py`'s derivation, with a valid experiment and a +measured magnitude behind it — which is the first time on this branch that the +escalation-packet bar in plan v2 §5.M2.d has actually been met on evidence +rather than on frustration. + +--- + +## 2026-07-29 · Line-box escalation, granted — executing under ruling law 18 + +Option (a) granted. `infer.py` open for the vertical-origin derivation only, one +formula, no backend conditionals, built from baselines/leadings/sizes and +exactdoc's own published constants. + +**The change.** `margin_t`'s `tops` collection currently takes each text line's +box *top* (`l.bbox[1]`). It will instead take +`baseline − (leading − 0.21·size)` — the writer's own paragraph-top formula, so +the page origin is computed the same way as the paragraphs placed against it. +Leading comes from `infer`'s existing rule (median baseline delta in the block; +`max(size × 1.16, 4.0)` for a single-line block), so no new constant is +introduced. Drawings contribute their box tops unchanged in both derivations — +they have no baseline, and M2.b already made path geometry agree exactly. +`margin_b`/`ye` is **not** touched: the measured disagreement is at the top, and +scope stays where the evidence is. + +### Gate 3 (before any render): the margin_t probe + +`testkit/margin_probe.py` (new), unrounded, both backends: + +| document | shipped mu/pf | Δ shipped | anchored mu/pf | Δ anchored | +|---|---|---|---|---| +| `03_tech_report_code` | 24.3 / 25.9 | 1.53 | 25.5 / 25.5 | **0.000** | +| `05_memo` | 77.0 / 79.3 | 2.31 | 78.7 / 78.7 | **0.000** | +| `f1_fpdf_brief` | 26.5 / 29.5 | 2.97 | 28.7 / 28.7 | **0.000** | +| `r1_reportlab_report` | 62.3 / 65.3 | 2.97 | 64.5 / 64.5 | **0.000** | +| `c4_i18n`, `c7_code`, `c8_toc_links` | 67.8 / 67.8 | 0.01 | 66.9 / 66.8 | 0.002 | +| `c2_paper2col` | 67.1 / 67.1 | 0.00 | 66.4 / 66.4 | 0.005 | +| `01_whitepaper`, `04_exec`, `c1`, `c3`, `c6`, `l1` | — | 0.00 | — | **0.000** | +| **`02_research_paper`** | 63.3 / 64.9 | 1.67 | 64.4 / 63.8 | **0.640** | +| **`c5_graphics`** | 87.0 / 64.5 | 22.49 | 86.4 / 64.5 | **21.900** | + +**14 of 16 land at ≤0.005pt, from disagreements of up to 2.97pt.** The two that +do not are attributed, and neither is the convention: + +- **`c5_graphics`** — PyMuPDF's minimum is set by `text |Gradient Band|`, + pdfium's by `draw rect`. That is the *documented* gradient divergence: + PyMuPDF does not report the band at all, so its topmost element is different + content. Already in `EXPECTED_DIVERGENCE` with rendered evidence. +- **`02_research_paper`** — both backends' minimum is set by the *same line* + (`p2 |[1] Raman, P. et al. Segme|`). Baselines are identical and sizes agree + to 0.005pt, so the residue is the *leading*: that block's membership differs + between backends. A grouping artifact, not a font-metric one. + +**Reading of gate 3.** Its stated purpose is that "disagreement surviving the fix +means the fix did not remove the convention". The surviving disagreement is +demonstrably *not* the convention in either case. I am treating the gate as met +on its purpose while recording plainly that its literal per-document threshold +is missed on those two, so the planner can overrule me on the reading rather +than on the facts. + +### Predictions, written before implementing (law 18 gate 5) + +**The incumbent moves, and here is exactly where.** pymupdf `margin_t` changes on +**12 of 16** documents: + +| moves | pymupdf margin_t | +|---|---| +| `f1_fpdf_brief`, `r1_reportlab_report` | +2.2 | +| `05_memo` | +1.7 | +| `03_tech_report_code` | +1.2 | +| `02_research_paper` | +1.1 | +| `l1_word_native` | +0.1 | +| `c5_graphics` | −0.6 | +| `c2_paper2col` | −0.7 | +| `c4_i18n`, `c7_code`, `c8_toc_links` | −0.9 | +| `c6_long` | **−3.0** | +| unchanged (clamped or identical) | `01_whitepaper`, `04_exec_brief`, `c1_whitepaper`, `c3_tables` | + +- I predict the **four unchanged documents are bit-identical** on the pymupdf + lane. That is a hard prediction and easy to falsify. +- For the other twelve I predict **net-neutral-to-better**, because the origin + now agrees with the formula the writer uses to place the first paragraph + against it — but **I cannot predict the sign per document and will not + pretend to.** `c6_long` at −3.0 is the largest mover and the one I would bet + on if something regresses. +- Gate 4 requirement: **zero pymupdf documents may verdict REGRESSION**, both + lanes. If that fails twice, revert and take fallback (c). +- pdfium: I predict `02_research_paper` improves (its 1.67pt origin error drops + to 0.64) and `01_whitepaper_market` does **not** move materially — its + `margin_t` is clamped identical on both backends, so its 0.53-vs-0.72 gap was + never this defect. If 01 is unmoved, fallback (c)'s wording applies to it + regardless of how 02 lands. + +### Gate 4 — FAILED, on the document I named + +Implemented as specified: `_text_top(block_lines, ln)` in `infer.py`, one +formula, no backend conditionals, only baselines/leadings/sizes and exactdoc's +own 0.21 and 1.16. Parser untouched — golden IR 7/7, purity 16/16. + +**The incumbent regressed.** pymupdf lane, before → after: + +| pymupdf document | before | after | Δ | +|---|---|---|---| +| **`c6_long`** | **0.76** | **0.45** | **−0.31** | +| `c4_i18n` | 0.42 | 0.39 | −0.03 | +| `04_exec_brief` | 0.22 | 0.20 | −0.02 | +| `f1_fpdf_brief` | 0.62 | 0.60 | −0.02 | +| `r1_reportlab_report` | 0.60 | 0.58 | −0.02 | +| `01_whitepaper_market` | 0.72 | 0.71 | −0.01 | +| the other 10 | | | 0.00 | + +`c6_long` at −0.31 is four times the comparator's 0.08 band: a **REGRESSION +verdict on the incumbent**, which law 18 gate 4 forbids outright. Both gate +lanes confirm it — refine within2pt **0.529 → 0.486**, dy50 0.68 → 0.80, though +pass counts held at 12/16 and 13/16 with 0 new / 0 stale (the lane thresholds +are page/live/recall, which this does not touch). + +My prediction named `c6_long` as the document to watch, and the four +margin-unchanged documents (`01_whitepaper`, `04_exec_brief`, `c1`, `c3`) were +predicted bit-identical — `01` moved 0.01 and `04` moved 0.02, so that hard +prediction is **half wrong**: clamping made their *margins* identical but their +content still shifted, because other pages of those documents moved. + +### Why, and why a second attempt is not the answer + +Diagnosed rather than retried. Under the shipped rule `c6_long`'s origin is set +by a single-line bullet block at box top 65.19; under the anchored rule it is a +body paragraph at 62.20, because `leading − 0.21·size` = 13.5pt is *larger* than +that text's ink ascent of ~11.3pt. + +The formula is not wrong. It returns the top of the first line's **box in the +flow model**, which is precisely what `para_top` means. The problem is that +**everything downstream of the origin is calibrated against a box-top origin** — +`_assemble_chunks` derives each element's `space_before` from the running +position, so moving only the origin desynchronises the origin from the spacing +chain measured against it. Making the vertical model baseline-consistent means +moving `margin_t`, `_para_box` and the `space_before` chain **together**, which +is a far larger change than this escalation granted (its scope is the `margin_t` +block and the `page_top` it feeds) and is not something to attempt at the end of +a session by iterating against a gate. + +**Reverted.** `infer.py` is back at HEAD; the shared pipeline carries no change +from this escalation. Per the ruling's stop condition I am taking pre-agreed +fallback **(c)** rather than spending the second attempt on a speculative +restructuring. + +### What the escalation bought, since it is not nothing + +- The convention is now **proven** to be the cause *and* proven not to be + fixable at the origin alone. That closes a line of enquiry rather than + leaving it open. +- `testkit/margin_probe.py` stays: it measures backend agreement on the page + origin under both derivations, and it is the instrument that would gate any + future attempt at the full vertical-chain change. +- The measured fact that **14 of 16 documents reach 0.000pt origin agreement** + under the baseline formula is the evidence that a *complete* baseline-anchored + vertical model would work — it is the partial application that fails. diff --git a/STATUS.md b/STATUS.md index fdd0b18..3fc59d2 100644 --- a/STATUS.md +++ b/STATUS.md @@ -5,27 +5,87 @@ Where something is unknown, it says so; where a measurement is untrustworthy, it says why. Baseline for all figures: 16-document corpus, `--refine` (the CLI default), -LibreOffice render-back, Windows/PyMuPDF. Reproduce with: +LibreOffice render-back, PyMuPDF. **CI Linux is the number of record** +(`.github/workflows/gate.yml`); the Windows column is kept beside it because +having two independent environments agree is itself evidence. Reproduce with: ```bash -python testkit/gen_corpus.py testkit/adv && python corpus/make_corpus.py +bash scripts/bootstrap.sh # Linux: provisions the oracles, reports what it found ``` ```bash -REFINE=lanes python testkit/runall.py testkit/adv corpus/pdfs +python testkit/gen_corpus.py testkit/adv --strict && python corpus/make_corpus.py +``` + +```bash +python testkit/corpus_manifest.py verify && python testkit/runall.py ``` --- ## 1. Where the converter stands -| Metric | no-refine lane | refine lane (shipped) | +| Metric | `raw` lane | `product` lane (shipped) | earlier CI run | Windows | +|---|---|---|---|---| +| Gate passed | 12/16 | 13/16 | 12 / 13 | 12 / 13 | +| Page count 1:1 | 13/16 | 15/16 | 13 / 15 | 13 / 15 | +| Live (editable) text | 0.9652 | 0.9652 | 0.965 | 0.965 | +| Words within 2pt of source | 0.3486 | **0.5118** | 0.366 / 0.529 | 0.361 / 0.510 | +| Median per-word vertical drift | 2.20pt | **0.62pt** | 2.20 / 0.68pt | 2.79 / 0.69pt | + +The first two columns are the **recorded baseline** — +`testkit/gate_baseline.json`, measured on the canonical Linux environment +(LibreOffice 24.2.7.2, Liberation metric fonts, PyMuPDF 1.28.0 / MuPDF 1.29.0, +pypdfium2 5.12.1), which the file names in full beside the numbers. Beside them, +an earlier CI run +([#30455217670](https://github.com/ebt55/exactdoc/actions/runs/30455217670)) and +Windows. + +The `within2pt` spread across those columns — 0.510 to 0.529 — is what different +LibreOffice builds and font sets cost, and it is why the gate's tolerances are +absolute-plus-proportional rather than exact. It is also why `dy_p50` gets a +proportional term: it is the one gated metric that is not a fraction, running +from 0.04pt to 101pt across the corpus, so a single absolute slack cannot serve +both ends. + +**Three environments** — different fonts, three LibreOffice builds, three +Chromium builds — agree on every structural number (which documents pass, page +counts, live text, drift) and differ only in the third decimal of `within2pt`. +The harness is portable; it was only its *provisioning* that was folklore. + +The gate is **both** a regression gate and, on demand, an absolute one, and it +runs fail-closed. Three documents have never cleared the thresholds, and naming +them precisely matters because an earlier version of this paragraph wrote +"D3, D4/graphics" and thereby merged two different documents with two different +causes: + +| Document | Fails | Defect | |---|---|---| -| Gate passed | 12/16 | 13/16 | -| Page count 1:1 | 13/16 | 15/16 | -| Live (editable) text | 0.965 | 0.965 | -| Words within 2pt of source | 0.361 | **0.510** | -| Median per-word vertical drift | 2.79pt | **0.69pt** | +| `c3_tables` | page count, word recall 0.331, live text 0.923 | D3 nested tables | +| `c5_graphics` | live text 0.707, word recall 0.678 (raw: also page count) | D10 rasterised regions | +| `04_exec_brief` | live text 0.941, doc recall 0.934 | D10 rasterised regions | +| `c1_whitepaper` | raw lane only: page count, word recall 0.767 | D4 rounded cards | + +Because those exist, `runall.py` used to exit non-zero on every run ever made and +the CI step had to ignore its own result. The record in +`testkit/gate_baseline.json` is now **numeric**: every gated metric of every +document, per lane, plus the defect ID each shortfall answers to. The gate asks +three separate questions of it — + +- **regression** — is anything worse than the recorded number beyond tolerance? + Every document, every metric, passing or not. This is the pull-request gate, + and it is what closes the hole where a known 0.941 could have slid to 0.10 + while staying green, because the old record stored only the metric's *name*. +- **absolute** (`--absolute`) — does every document clear its release threshold? + This is the release-qualification gate, and today it fails, by design and on + the record. +- **stale** — does a recorded shortfall now pass? Then the record is wrong, and a + wrong record silently re-admits the regression it exists to catch. + +Both lanes gate the exit code. Gating on the refined lane alone meant the raw +lane — the control, whose whole purpose is to be untainted — was the one nobody +had to answer for. Every false-green path the previous gate had is now a test in +`tests/test_gate_mutations.py`, which needs no corpus and no oracle. Two lanes are always reported because `refine()` tunes the layout against the same renderer the gate measures with. A refined-only number can improve because @@ -39,14 +99,20 @@ That is the honest generalisation number and it is worse than the corpus number ### By producer dialect -| Dialect | Docs | State | -|---|---|---| -| ReportLab | 6 | good — page match, 94–100% live text | -| Chromium / Skia | 8 | good after the P0 dialect work; was catastrophic | -| WeasyPrint | 1 | good — 10/10 pages, 98% live | -| fpdf2 | 1 | good | -| LibreOffice | 1 | fair | -| **LaTeX / pdfTeX** | **4** | **worst — see D1** | +| Dialect | Docs | In the gate corpus? | State | +|---|---|---|---| +| ReportLab | 6 | yes | good — page match, 94–100% live text | +| Chromium / Skia | 8 | yes | good after the P0 dialect work; was catastrophic | +| fpdf2 | 1 | yes | good | +| LibreOffice | 1 | yes | fair | +| WeasyPrint | 1 | **no** — a real document outside the corpus | good — 10/10 pages, 98% live | +| **LaTeX / pdfTeX** | **4** | **no** — the holdout | **worst — see D1** | + +The third column was missing and the rows summed to 17 for a 16-document corpus, +which is the kind of arithmetic that survives in prose and cannot survive in +`testkit/corpus_manifest.json` — the manifest names all 16, their generator and +their dialect, and the gate fails if the run and the manifest disagree in either +direction. --- @@ -79,21 +145,45 @@ partly-wrong answer, documented in §5. python testkit/elemheight.py testkit/real/arxiv_transformer.pdf ``` -### D2 — pdfium backend: fine-placement gap · **severity: high (blocks relicensing)** +### D2 — pdfium backend: fine-placement gap · **severity: low (no longer blocks relicensing)** + +**This used to be the only thing keeping exactdoc off Apache-2.0. It is not any +more.** The licence is inherited, not chosen: PyMuPDF is AGPL-3.0, so exactdoc +is. A permissive parser (pypdfium2, Apache-2.0) exists in +`exactdoc/parse_pdfium.py`, and it is now measured **not worse than the +incumbent on 14 of 16 corpus documents** — four exactly equal, two better. The +flip is scheduled, not blocked: see [ROADMAP.md](ROADMAP.md) §3.2. -**This is the only thing keeping exactdoc off Apache-2.0.** The licence is -inherited, not chosen: PyMuPDF is AGPL-3.0, so exactdoc is. A permissive parser -(pypdfium2, Apache-2.0) exists in `exactdoc/parse_pdfium.py` and is selectable, -but it places text worse, so it is not the default. +The two documents that remain are attributed to a font-metric convention no +permissive parser can reproduce, and are accepted as a documented divergence +rather than chased — the reasoning is below, under *The two that remain*. -The gap is **7 regressions**, down from 9. Words land on the right *pages* -(`word_recall` 0.96–1.00); they land a couple of points off within them. +The gap is **2 regressions**, down from 9 → 8 → 6 → 3 → 2. Fourteen of sixteen +documents are now at or better than the incumbent, four of them exactly equal +to it and two above it. | | within-2pt | median dy | |---|---|---| -| PyMuPDF (default) | **0.510** | 0.69pt | -| PyMuPDF + pdfium clip rendering | 0.476 | 1.31pt | -| pdfium parser | 0.291 | 2.02pt | +| PyMuPDF (default) | **0.511** | 0.69pt | +| pdfium parser, when this was first measured | 0.291 | 2.02pt | +| **pdfium parser, now** | **0.461** | — | + +**Acceptance for the flip, and it is now executable rather than stated:** +`testkit/parity_policy.json` carries the rule the test applies — comparison +margins, the two expected divergences with their rendered evidence, and these two +accepted shortfalls with **numeric floors**, recorded on the canonical +environment: + +| Document | PyMuPDF | pdfium floor | fails if | +|---|---|---|---| +| `01_whitepaper_market` | 0.719 | **0.533** | within-2pt drops below the floor, or the divergence disappears | +| `02_research_paper` | 0.761 | **0.569** | same | + +Both directions matter. An acceptance with no floor is an acceptance of anything, +and an acceptance that no longer describes reality is a stale record that hides +the next real regression on that document. The current verdict is **0 +regressions, 11 same, 1 better, 2 expected divergences, 2 accepted** — and the CI +step is required, not `continue-on-error`. #### What it is not @@ -146,14 +236,68 @@ a humanist sans. Matching it would mean reproducing a bug. #### What is left -Converge `_build_blocks` on PyMuPDF's grouping — `testkit/golden_ir.py` is the -specification. On the evidence above that should clear roughly three more -documents. `c7_code` and `03_tech_report_code` are explained by neither -geometry, grouping nor fonts, and still need a cause; both are code-heavy, so -intra-line span segmentation is the next place to look. +Converge `_build_blocks` on PyMuPDF's grouping. On the evidence above that +should clear roughly three more documents. + +**The contract is `backend_parity.py`, not the golden IR.** An earlier version +of this section called `golden_ir.py` "the specification". It is not, and +saying so was steering the port at the wrong target: this backend already +refuses to reproduce three PyMuPDF behaviours because they are bugs, and +PyMuPDF's grouping is not even stable across its own releases (measured: 1.24 +and 1.26 put `02_research_paper` p2 in 4 blocks, 1.28 in 7). The golden is a +microscope for locating a disagreement; the rendered-output gate decides +whether it matters. + +`c7_code` and `03_tech_report_code` were "explained by neither geometry, +grouping nor fonts". They are now attributed, and it was none of the four +suspected causes: **PDFium does not report leading indentation and PyMuPDF +synthesises it.** For ` def __init__(...)` PDFium's first character is `d` +at x=93.17 with no space before it; PyMuPDF reports the line starting at +x=72.25 with four leading spaces. PDFium synthesises spaces *between* +characters, where there is a gap to measure; at a line start there is nothing +to the left, so the indent vanishes and every glyph on the line is displaced. +Both documents now sit at or above the incumbent. + +#### The two that remain, and why + +`01_whitepaper_market` (0.53 against 0.72) and `02_research_paper` (0.57 +against 0.76) are attributed to a **font-metric convention difference that no +permissive parser can reproduce.** + +`infer()` derives the page's vertical origin from line-box *tops*. That is the +one vertical quantity two correct parsers legitimately disagree about, because +each reads it from font-metric tables the other does not have: + +| font | PyMuPDF above/below baseline, per size | pdfium | +|---|---|---| +| Helvetica | 1.075 / 0.299 | 0.905 / 0.211 | +| Times-Roman | 1.053 / 0.281 | 0.891 / 0.215 | +| **Symbol** | **1.010 / 0.293** | **1.010 / 0.293** | + +Symbol is the control: where both fall back to the *embedded* font's metrics +they agree to three decimals. Everywhere else PyMuPDF is using its own base-14 +table. The difference reaches `margin_t` (63.30 against 64.90 on +`02_research_paper`) and displaces every word on the page by a constant 1.5pt — +visible as two identical dy distributions offset by exactly that. + +**Parser-side exhaustion is proven, not assumed.** `FPDFFont_GetAscent` and +`FPDFFont_GetDescent` return exactly the ratios the loose box already uses; +pdfium exposes one vertical font metric and the parser is already using it. +Reproducing PyMuPDF's numbers would mean vendoring MuPDF's base-14 table into +the permissive tree, which the licence plan forbids and which is measurably +version-dependent. + +A shared-pipeline fix was granted, built and **reverted**: anchoring the origin +on baselines instead reached 0.000pt backend agreement on 14 of 16 documents, +but cost the *incumbent* `c6_long` 0.76 → 0.45, because the `space_before` +chain downstream is itself calibrated against a box-top origin. Making the +vertical model baseline-consistent means moving the origin, `_para_box` and the +spacing chain together — a larger change than this defect justifies on its own. +Evidence: `testkit/margin_probe.py`, and the escalation packet in the project's +planning documents. ```bash -python testkit/backend_parity.py --refine 3 +python testkit/backend_parity.py ``` ```bash @@ -196,6 +340,11 @@ On the *same* DOCX, with the `--target gdocs` static fix applied: | mean within-2pt | 0.404 | ~0.20 | | page match | 17/18 | 11/16 | +The LibreOffice column is from the 18-document corpus of the time and the Docs +column from the current 16; the two are not directly comparable and the +comparison has not been rerun on one corpus. The *direction* is the finding — +Docs is the harder target — not the ratio. + Docs has no "exact" line spacing, so its importer mistranslates `lineRule="exact"` — error scaling with font size (+45pt at 18pt type, +84pt at 22pt). `--target gdocs` emits multiples instead, which recovers most of it @@ -213,22 +362,64 @@ tiny, dense microtype) convert without crashing. ### D9 — `w:shd` emitted 17,112 times across 18 documents · **severity: low** +*(Counted on the 18-document corpus of the time; not recounted on the current +16. The order of magnitude is the point.)* + Shading applied very aggressively per-run/per-cell. File-size and complexity smell, not a correctness bug. +### D10 — text inside rasterised regions is not live text · **severity: medium** + +The defect ID the gate baseline needed. Two documents have never cleared the +0.95 live-text threshold and the reason was recorded only as prose, which meant +`04_exec_brief`'s 0.941 could have fallen to 0.10 without the gate noticing — +the old baseline stored the metric's *name*, not its value. + +| Document | live text | doc recall | with figure regions excluded | +|---|---|---|---| +| `c5_graphics` | 0.707 | 0.678 | **0.988** | +| `04_exec_brief` | 0.941 | 0.934 | **0.978** | +| `c3_tables` (D3) | 0.923 | 0.936 | 0.966 | + +The third column is `exactdoc/verify.py:audit()`, which excludes figure-region +text from its denominator — the converter's own view, and the one STATUS §4.5 +warns not to trust alone. Read only as an attribution it says: **rasterisation +is the dominant cause on all three and the whole cause on none.** `c5_graphics` +loses 28 points of coverage to a gradient band and an SVG chart that must +rasterise (§6.5), and recovers 28 of them when those regions are excluded. +`04_exec_brief` recovers most but not all of its 6 points. The residual is +**unattributed** and deliberately not guessed at. + +Not the same defect as D3: `c3_tables` fails structurally (word recall 0.331, +one page over), and its live-text shortfall is a symptom of the nested-table +flattening rather than of a figure. + +```bash +python testkit/runall.py --lane product --absolute +``` + --- ## 3. Pending work, in the order I would do it +**Sequence and distance live in [ROADMAP.md](ROADMAP.md).** This table is the +defect view; the roadmap is the plan view. + +**D2 is no longer a blocker.** The permissive parser is at 2 regressions from 9, +both attributed and accepted as a documented divergence, so the relicence can +proceed. That was the only thing gating it. + | # | Item | Blocks | Notes | |---|---|---|---| -| 1 | **D2 fine-placement gap** | Apache-2.0 relicensing | Now attributed, not guessed: converge `_build_blocks` against the golden IR (≈3 documents), then find what ails the code-heavy pair | -| 2 | **D1 LaTeX pagination** | core use case | Needs writer-side instrumentation (§5), not another hypothesis | -| 3 | **Un-gate the wrap correction** | fidelity | Written and measured (+20pt line agreement); needs predicted `n_lines` in the page-capacity model *before* the first write, or it costs a page | -| 4 | D4, D5, D6 | — | Bounded, independent | -| 5 | **Google Docs cover-band check** | a real claim in the README | One oracle run; may invalidate the design | -| 6 | D3 nested tables | — | | -| 7 | PyPI release | adoption | After 1 | +| ~~1~~ | ~~Superscript in the pdfium backend~~ | — | **Closed by measurement, no code written.** `backend_superscript.py`: the writer never sees the parser's flag — `dialect` and `infer` recover superscript from geometry, and all 16 documents agree at the layout level. ROADMAP §3.1 | +| 2 | **The flip and the relicence** | **the whole point of the project** | **Not mechanical.** `fitz` is on the default *runtime* path well past the parser: `docxout.py` imports it at module load and uses MuPDF text metrics for table fitting and MuPDF rasterisation for figure clips, `refine.py` extracts text through it, `verify.py` compares images with it, `ladder.py` measures with it. A wheel installed without PyMuPDF fails while importing the writer, before any backend selection happens. The backend has to be chosen once and carried through parse, write, refine and verify first | +| 3 | **D8 clean unsupported-input error** | the release | Encrypted/truncated PDFs; both files into CI | +| 4 | **PyPI release** | adoption | TestPyPI dry run first; release notes lead with the holdout | +| 5 | **D1 LaTeX pagination** | the holdout, and the core use case | Needs writer-side instrumentation (§5) — per-element emitted-vs-source height accounting inside `docxout` — not another hypothesis. Three attempts have each produced a partly-wrong answer | +| 6 | **The baseline-consistent vertical model** | the last 2 parity regressions, and probably much else | Move `margin_t`, `_para_box` and the `space_before` chain together. A partial version was granted, built and reverted (D2) — the origin alone desynchronises from the spacing calibrated against it | +| 7 | **Google Docs cover-band check** | a real claim in the README | One oracle run; may invalidate the design. The least-measured part of the stated product goal | +| 8 | Un-gate the wrap correction | fidelity | Needs predicted `n_lines` in the page-capacity model *before* the first write, or it costs a page | +| 9 | D3, D4, D5, D6, D9 | — | Bounded, independent | Not planned: OCR for scanned PDFs; CJK/RTL shaping beyond the reordering already done; forms. @@ -341,6 +532,15 @@ pattern is more useful than the individual fixes. | Probes matched non-unique strings | Measured body-text "ByteNet", not the table | Match on text that is unique on both sides | | Wrote a hypothesis into D2 as if it were a finding | "Most likely baseline or line-box geometry" survived a full revision of this file; the first direct measurement showed baselines identical on 4,734 of 4,734 lines | A plausible cause in a defect register is read as a known one. Mark it as a guess or measure it | | Imported `pypdfium2` without declaring it | `uv sync` evicted it; the parity gate began reporting `ModuleNotFoundError` | A gate that cannot run looks exactly like a gate that passes | +| Gated on *any* failure, with three documents that had never passed | `runall.py` returned non-zero on every run it ever made, so the CI step was marked `continue-on-error` and nothing was gated at all | A check that always fails carries the same information as one that always passes. Gate on the *delta* against a recorded set | +| Tokenised words on whitespace, which CJK does not use | A "word" was a whole rendered line; a one-character re-wrap lost it. `c4_i18n` scored `doc_recall` 0.83 on Linux and passed on Windows **with every character present in both** | The unit a metric counts in must be a unit the content actually has | +| Read golden drift as parser drift | A version-dependent difference (PyMuPDF 1.26 groups `02_research_paper` p2 into 4 blocks, 1.28 into 7) was recorded as cross-platform instability | A frozen artifact without a manifest of what froze it cannot tell you which of the two changed | +| Recorded the *names* of failing metrics, not their values | `04_exec_brief`'s live-text coverage was on record as "known failing" at 0.941. It could have fallen to 0.10 and stayed exactly as green. Same hole in `page_match`, a boolean that cannot tell one page over from forty | A known failure needs a *bound*, not a label. Record the number | +| Treated a missing measurement as a skip | `harness.evaluate()` returns `{"error": ...}` when the render fails and nothing read the key; absent metrics hit `if v is None: continue`. A renderer dying on all 16 documents scored zero failures | Fail closed. A metric that could not be computed is a failure, never a row to pass over | +| Never checked the corpus against a manifest | Measured in a bare container: the generator produced 3 of 16 documents, printed "the corpus is incomplete, numbers are NOT comparable", exited 0 — and the gate scored those 3 against a 16-document baseline and reported a pass | Prose that the next step ignores is not a safeguard. `--strict`, and a manifest the gate compares against in both directions | +| Wrote the oracle paths to a file nobody sourced | `bootstrap.sh` discovers Chromium and writes `scripts/env.sh`, then every subsequent shell — including each CI step — starts without it. CI only ever worked because the GitHub runner image happens to ship `/usr/bin/google-chrome`: provisioning by accident | Discovery has to be readable by the thing that needs it. `_paths.py` now reads the record itself | +| Let the executable rule and the ratified rule disagree | `backend_parity.py` exited on `regressions == 0` while ROADMAP and this file said two documents were formally accepted. The disagreement was resolved by marking the CI step `continue-on-error`, which retired the one gate the entire relicensing effort was aimed at | A gate whose policy lives in prose will be switched off, not corrected. Put the policy in a file the test reads | +| Injected a parser by assigning a module global | The instruments set `exactdoc.convert.parse_pdf`. That worked only because `convert` happened to hold the parser as a global; once the backend was selected through the seam, the assignment became a no-op that set an attribute nobody read — and an experiment that silently measures the default still prints a number | An injection point should be declared (`register_backend`), so removing it breaks loudly instead of quietly | Two compensators were built, measured, and **left switched off** because they did not pay: the quality ladder (line-locking) and the half-point wrap diff --git a/THEORY.md b/THEORY.md index 5588a1c..728699d 100644 --- a/THEORY.md +++ b/THEORY.md @@ -19,6 +19,53 @@ No fix below was accepted on theory alone; each one moved a measured number. --- +## Addition, 2026-07-30 — two things the permissive-parser port taught + +Both were learned the expensive way during the pdfium convergence work, and both +generalise beyond it. Recorded here so a future session does not re-derive them. + +### The renderer normalises whatever it is free to normalise + +Three separate changes made the pdfium IR structurally identical to PyMuPDF's — +span boundaries, injected whitespace, trailing spaces — and **all three moved +`within2pt` by exactly zero.** The third explained the other two: in justified +text the renderer redistributes inter-word space to fill the measure, so the +*number of spaces in the source has no bearing on where words land*. + +The general form: **a difference the renderer is free to normalise cannot show +up in a placement metric, however wrong it looks in the IR.** Whitespace, +run-splitting and span segmentation are all in that category for justified text. +This is not a reason to leave them wrong — the DOCX carries the text a user will +read and edit, and `live_text_cov` strips whitespace so it cannot see the +difference — but it *is* a reason not to expect them to move the gate, and a +reason to check which category a defect is in before spending a session on it. + +### Anchor everything on baselines, including the page origin + +§3.1 established that vertical placement is anchored on baselines because line +boxes are unreliable. That principle was applied to paragraphs and not to the +page: `infer()` still derives `margin_t` from the topmost line's box *top*. + +Line-box height turns out to be the single least portable quantity in the whole +model. Two correct parsers disagree about it because each reads it from font +metric tables the other does not have — PyMuPDF puts Helvetica's box 1.075× the +type size above the baseline, pdfium 0.905×, and on Symbol, where both fall back +to the *embedded* font's metrics, they agree to three decimals. That difference +propagates into the page origin and displaces every word on the page by a +constant. + +Completing the principle — deriving the origin from +`baseline − (leading − 0.21·size)` like everything else — was tried and +**reverted**. It reached exact backend agreement on 14 of 16 documents and still +made the *default* backend worse, because `space_before` is computed against the +running position and is therefore calibrated on the old origin. The lesson is +not "baselines were the wrong idea"; it is that **the vertical model is a chain, +and half-converting a chain desynchronises it.** Doing it properly means moving +the origin, `_para_box` and the spacing chain in one change — recorded in +[ROADMAP.md](ROADMAP.md) as the open item it is. + +--- + ## 1. The problem A PDF is a *painting*: absolutely positioned glyphs, paths and images with no @@ -372,10 +419,17 @@ here measured editability at all. And the corpus behind those numbers was one self-authored dialect, so it measured tuning, not generalisation — the current holdout figure on wild PDFs is **0/4**. -Corpus scores are reported in two lanes (refine on and off) for the same +Corpus scores are reported in two lanes (`product` and `raw`) for the same reason: `refine()` tunes against the same renderer the gate measures with, so a refined-only number can improve because the loop memorised the oracle. Only the -pair means anything. +pair means anything — and both now gate the exit code, because for a while only +the refined lane did, which left the control lane free to regress unanswered. + +One more failure of the same shape, and it is the reason `exactdoc/options.py` +exists: the numbers above were measured on a profile no shipping surface ran. The +API refined 0 times, the CLI 2, the quoted lane 3. A measurement that describes no +shipping configuration is a coincidence, however carefully it was taken. There is +one profile now, and every surface reads its defaults from it. ## 9. Is Python the limitation? diff --git a/exactdoc/__init__.py b/exactdoc/__init__.py index e69de29..2efa679 100644 --- a/exactdoc/__init__.py +++ b/exactdoc/__init__.py @@ -0,0 +1,47 @@ +"""exactdoc -- measurement-driven PDF to DOCX conversion. + + from exactdoc import convert + convert("paper.pdf", "paper.docx") + +The public surface is deliberately small: `convert`, the options profile that +supplies its defaults, and `__version__`. Everything else is internal and may +move between alpha releases. + +Names resolve lazily (PEP 562) so that `import exactdoc` costs nothing but this +docstring. That matters beyond startup time: the permissive-runtime work needs +`import exactdoc` to succeed on an installation with no PyMuPDF present, and an +eager re-export of the writer would defeat that before the backend seam ever +got a chance to choose. +""" +__all__ = ["convert", "ConversionOptions", "PRODUCT", "RAW", "__version__"] + + +def _version() -> str: + try: + from importlib.metadata import PackageNotFoundError, version + except ImportError: # pragma: no cover + return "0.0.0+unknown" + try: + return version("exactdoc") + except PackageNotFoundError: + # Running from a checkout that was never installed -- the normal state + # for the harness. Say so rather than inventing a number that would + # then be published in an evidence artifact. + return "0.0.0+source" + + +__version__ = _version() + + +def __getattr__(name): + if name == "convert": + from .convert import convert + return convert + if name in ("ConversionOptions", "PRODUCT", "RAW"): + from . import options + return getattr(options, name) + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + +def __dir__(): + return sorted(__all__) diff --git a/exactdoc/backend.py b/exactdoc/backend.py index 37659a1..eae6119 100644 --- a/exactdoc/backend.py +++ b/exactdoc/backend.py @@ -24,10 +24,28 @@ no line or block grouping at all, which means writing that clustering here. That is the actual plan, and it is why this seam exists. Writing the -clustering ourselves means we *control* the grouping, so the existing tuning -stops being a liability and becomes the specification: the port is correct -when it reproduces the frozen golden IR (testkit/golden_ir.py). A verifiable -port, not a rewrite. +clustering ourselves means we *control* the grouping. + +**What "correct" means here, and what it does not.** The port is correct when +`testkit/backend_parity.py` finds it no worse than PyMuPDF on the rendered +output. It is NOT "reproduce the frozen golden IR" -- that was the target for a +while, and it is the wrong finish line for three measured reasons: + + - This backend deliberately refuses to reproduce three PyMuPDF behaviours + because they are bugs: RTL returned in visual order (renders Arabic + backwards), gradients dropped (white text left invisible on white), and + Calibri reported as serif. + - PyMuPDF's grouping is not stable across its own releases. Measured: + 1.24.14 and 1.26.0 put page 2 of 02_research_paper in 4 blocks, 1.28.0 + puts it in 7. A target that moves with a dependency version cannot be a + specification. + - The golden is regenerated from a corpus that is itself regenerated, so it + describes an environment as much as a parser (hence the manifest it now + carries). + +The golden IR is a **microscope**: a fast, oracle-free, per-document diff for +finding *where* two parsers disagree. The parity gate is the **contract**. When +they disagree, the parity gate wins. Until then: **do not accept external contributions to parse.py.** Relicensing needs every contributor's consent, and the swap is confined to this one @@ -59,7 +77,21 @@ class Backend(Protocol): - """Structural interface. PyMuPDF is the only implementation today.""" + """Structural interface. Two shipped implementations, plus registrations. + + Selected once per conversion, by name, from `ConversionOptions.backend` -- + not by an environment variable read at an arbitrary depth, and not by + assigning over a module global. `EXACTDOC_BACKEND` still works and is now the + lowest-priority source. + + The seam stops at parsing and rendering, and that is the honest description + of where it stops being enough: the writer, the refiner and the verifier all + still reach for `fitz` directly, so a wheel installed without PyMuPDF fails + while importing `exactdoc.docxout`, before any of this gets a chance to + choose. Carrying the chosen backend through those three stages is the next + milestone (STATUS §3 item 2), and it is the real content of "the licence + flip", which was previously described as mechanical. + """ name: str @@ -125,20 +157,22 @@ class PDFiumBackend: paragraph assembly, and line boundaries decide which text a figure or table region absorbs. A cluster classified differently rasterises a page. - End-to-end that costs **7 regressions** on the parity gate (was 9 before the - serif-flag fix, and 15 before block convergence). testkit/exp_regroup.py - grafts PyMuPDF's block boundaries onto this backend's geometry and shows the - cost is bimodal: grouping is the entire cause on c6_long (0.23 -> 0.73) and - c8_toc_links (0.63 -> 1.00), and none of it on c7_code or - r1_reportlab_report, which do not move. - - So the remaining work is reproducing PyMuPDF's grouping decisions closely - enough that the downstream tuning still applies -- testkit/golden_ir.py is - the specification -- plus a cause for the code-heavy documents that grouping - does not explain. Superscript is still hardcoded False. - - This is the measured cost of relicensing. It is a re-tune, not a rewrite, - and it is bounded -- but it is not free, and it buys no fidelity. + End-to-end that cost 15 regressions before block convergence, then 9 before + the serif-flag fix, then 7, and it is now **0** against the ratified policy in + testkit/parity_policy.json: 11 same, 1 better, 2 expected divergences where + this backend is the correct one, and 2 accepted shortfalls bounded by recorded + numeric floors. testkit/exp_regroup.py grafts PyMuPDF's block boundaries onto + this backend's geometry and showed the cost was bimodal: grouping was the + entire cause on c6_long (0.23 -> 0.73) and c8_toc_links (0.63 -> 1.00), and + none of it on c7_code or r1_reportlab_report, which did not move. + + `superscript` is still hardcoded False, and measurement says leave it that + way: testkit/backend_superscript.py shows the writer never sees this flag -- + `dialect` and `infer` recover superscript from geometry, and all 16 corpus + documents agree at the layout level, including the one that has any. + + This was the measured cost of relicensing. It was a re-tune, not a rewrite, + and it bought no fidelity -- it bought a licence. """ name = "pdfium" @@ -174,10 +208,65 @@ def render_page(self, path: str, page_no: int, dpi: int = 110) -> Optional[bytes return buf.getvalue() -def get_backend(name: str = "pymupdf") -> Backend: - if name in ("pymupdf", "fitz", "default"): - return PyMuPDFBackend() - if name in ("pdfium", "pypdfium2"): - return PDFiumBackend() - raise ValueError("unknown backend %r (choose 'pymupdf' or the experimental " - "'pdfium'; see the module docstring)" % name) +_IMPLEMENTATIONS = {"pymupdf": PyMuPDFBackend, "pdfium": PDFiumBackend} +_EXPERIMENTAL = {} + + +class FunctionBackend: + """A backend built from a `parse_pdf` callable, for experiments. + + The instruments in `testkit/` need to convert the corpus through a parse + function that is neither shipped backend -- PDFium geometry with PyMuPDF's + block boundaries grafted on (`exp_regroup.py`), or PyMuPDF with the Chromium + bullet fix applied (`exp_chromefix.py`). They did it by assigning + `exactdoc.convert.parse_pdf`, which worked only because `convert` happened to + hold the parser as a module global. The moment the backend was selected + through the seam instead, that assignment became a no-op that set an + attribute nobody read -- and an experiment that silently measures the default + is worse than one that crashes, because it produces a number. + + So the seam takes registrations. Rendering falls through to a real backend, + because an experiment on grouping has no opinion about rasterising a clip. + """ + + experimental = True + + def __init__(self, name, parse, renderer=None, license=None): + self.name = name + self._parse = parse + self._renderer = renderer or PyMuPDFBackend() + self.license = license + + def parse_pdf(self, path: str, keep_image_data: bool = True) -> DocIR: + return self._parse(path, keep_image_data=keep_image_data) + + def render_clip(self, path, page_no, clip, dpi: int = 240): + return self._renderer.render_clip(path, page_no, clip, dpi=dpi) + + def render_page(self, path, page_no, dpi: int = 110): + return self._renderer.render_page(path, page_no, dpi=dpi) + + +def register_backend(name: str, parse, renderer=None) -> str: + """Make `parse` selectable as `backend=name`. Returns the name. + + For instruments and experiments only. Nothing in the package registers + anything, and a registered name is never a default. + """ + if name in _IMPLEMENTATIONS: + raise ValueError("%r is a shipped backend; pick another name" % name) + _EXPERIMENTAL[name] = FunctionBackend(name, parse, renderer=renderer) + return name + + +def get_backend(name: str = None) -> Backend: + """Instantiate a backend by name. Aliases resolve in options.py. + + Name resolution lives in one place on purpose: an unrecognised backend name + that quietly fell back to the default would report numbers for a parser + nobody selected. + """ + from .options import DEFAULT_BACKEND, canonical_backend + if name in _EXPERIMENTAL: + return _EXPERIMENTAL[name] + return _IMPLEMENTATIONS[canonical_backend(name or DEFAULT_BACKEND)]() diff --git a/exactdoc/cli.py b/exactdoc/cli.py index 0ecd356..a7d1356 100644 --- a/exactdoc/cli.py +++ b/exactdoc/cli.py @@ -1,36 +1,50 @@ -"""exactdoc command-line interface.""" -import os +"""exactdoc command-line interface. + +The single console entry point. Its defaults are not written here -- they come +from `exactdoc.options.PRODUCT`, so `exactdoc file.pdf`, `convert(file)` and +the CI product lane all run the same configuration. They used to run three. +""" import argparse +from .options import BACKENDS, PRODUCT, TARGETS -def main(argv=None): + +def build_parser(): ap = argparse.ArgumentParser( prog="exactdoc", - description="High-fidelity PDF -> DOCX converter tuned for " - "Claude-generated whitepapers. Output uses only Google " - "Docs-safe constructs.") + description="High-fidelity PDF -> DOCX converter. Output uses only " + "Google Docs-safe constructs.") ap.add_argument("pdf", nargs="+", help="input PDF file(s)") ap.add_argument("-o", "--out", help="output .docx path (single input only)") - ap.add_argument("--dpi", type=int, default=240, - help="raster DPI for vector figure regions (default 240)") - ap.add_argument("--target", default="libreoffice", - choices=["gdocs", "libreoffice", "none"], + ap.add_argument("--dpi", type=int, default=PRODUCT.dpi, + help="raster DPI for vector figure regions (default %(default)s)") + ap.add_argument("--target", default=PRODUCT.target, choices=list(TARGETS), help="which program the output should look right in. The " "closed loop optimises for this renderer, and the " "choice matters: a layout tuned for LibreOffice is " "measurably not tuned for Google Docs. 'gdocs' needs " - "Drive credentials (default: libreoffice)") - ap.add_argument("--refine", type=int, default=2, metavar="N", + "Drive credentials (default: %(default)s)") + ap.add_argument("--backend", default=PRODUCT.backend, choices=list(BACKENDS), + help="PDF parser (default: %(default)s). Overrides " + "EXACTDOC_BACKEND") + ap.add_argument("--refine", type=int, default=PRODUCT.refine_rounds, + metavar="N", help="closed-loop correction passes: render the DOCX back " "and correct page overflow and per-page offsets " "against what actually rendered (0 disables, " - "default 2)") + "default %(default)s -- the profile every published " + "number is measured on)") ap.add_argument("--verify", action="store_true", help="render the DOCX back to PDF (needs LibreOffice) and " "report per-page visual similarity + text coverage") ap.add_argument("--report-dir", default=None, help="directory for side-by-side comparison images") ap.add_argument("-v", "--verbose", action="store_true") + return ap + + +def main(argv=None): + ap = build_parser() args = ap.parse_args(argv) if args.out and len(args.pdf) > 1: ap.error("-o works with a single input") @@ -38,7 +52,8 @@ def main(argv=None): from .convert import convert for p in args.pdf: out = convert(p, args.out, dpi=args.dpi, refine_rounds=args.refine, - target=args.target, verbose=args.verbose) + target=args.target, backend=args.backend, + verbose=args.verbose) print("wrote", out) if args.verify: from .verify import verify, audit diff --git a/exactdoc/convert.py b/exactdoc/convert.py index 2813c9a..6434706 100644 --- a/exactdoc/convert.py +++ b/exactdoc/convert.py @@ -1,18 +1,41 @@ -"""End-to-end conversion API + CLI.""" +"""End-to-end conversion API. + +One entry point, one profile. Every default this function applies comes from +`exactdoc.options.PRODUCT`, so the API, the CLI, the CI lanes and the published +numbers cannot describe three different configurations again (see +options.py for what that cost). +""" import os -import sys -import argparse +from typing import Optional -from .parse import parse_pdf from .dialect import normalize from .infer import infer -from .docxout import write_docx +from .options import ConversionOptions, resolve + +def _select_backend(name: str): + """The backend, chosen once per conversion. + + `EXACTDOC_BACKEND` still works, but it is now the lowest-priority source: + an explicit `backend=` argument wins, because a gate that selects a parser + has to be able to say so in its own call rather than by mutating the + environment of the process it shares with everything else. + """ + from .backend import get_backend + return get_backend(name) -def convert(pdf_path: str, out_path: str = None, dpi: int = 240, - refine_rounds: int = 0, target: str = "libreoffice", - ladder: bool = False, verbose: bool = False) -> str: - """Convert a PDF to DOCX. + +def convert(pdf_path: str, out_path: Optional[str] = None, + dpi: Optional[int] = None, refine_rounds: Optional[int] = None, + target: Optional[str] = None, backend: Optional[str] = None, + ladder: Optional[bool] = None, verbose: Optional[bool] = None, + options: Optional[ConversionOptions] = None) -> str: + """Convert a PDF to DOCX. Returns the output path. + + Defaults come from `options.PRODUCT`: the pdfium/PyMuPDF backend it names, + the LibreOffice target, and its refine round count. Pass `options=` to + supply a whole profile, or individual keywords to override parts of it. A + `None` keyword means "take the profile's value", never "zero". `refine_rounds` > 0 enables the closed-loop pass: render the DOCX back and correct page overflow and per-page offsets against what actually rendered. @@ -22,47 +45,41 @@ def convert(pdf_path: str, out_path: str = None, dpi: int = 240, tuned for LibreOffice is measurably not tuned for Google Docs. If the chosen oracle is unavailable the conversion still succeeds, open-loop. """ + if backend is None: + env = os.environ.get("EXACTDOC_BACKEND", "").strip() + backend = env or None + opts = resolve(options, backend=backend, target=target, dpi=dpi, + refine_rounds=refine_rounds, ladder=ladder, verbose=verbose) if out_path is None: out_path = os.path.splitext(pdf_path)[0] + ".docx" - _bk = os.environ.get("EXACTDOC_BACKEND", "").strip().lower() - if _bk and _bk not in ("pymupdf", "fitz", "default"): - from .backend import get_backend - ir = normalize(get_backend(_bk).parse_pdf(pdf_path)) - else: - ir = normalize(parse_pdf(pdf_path)) + + bk = _select_backend(opts.backend) + ir = normalize(bk.parse_pdf(pdf_path)) lay = infer(ir) - if ladder: + if opts.ladder: from .ladder import apply_ladder, summarise rep = apply_ladder(lay) lay.ladder_report = rep - if verbose: + if opts.verbose: print(" ladder: " + summarise(rep)) - if refine_rounds > 0: + if opts.refine_rounds > 0: from .refine import refine from .targets import get_renderer - render, resolved = get_renderer(target) + render, resolved = get_renderer(opts.target) if render is not None: - if verbose: + if opts.verbose: print(" refining against: %s" % resolved) - return refine(lay, pdf_path, out_path, dpi=dpi, rounds=refine_rounds, - verbose=verbose, render=render, target=target) - return write_docx(lay, out_path, dpi=dpi, target=target) + return refine(lay, pdf_path, out_path, dpi=opts.dpi, + rounds=opts.refine_rounds, verbose=opts.verbose, + render=render, target=opts.target) + from .docxout import write_docx + return write_docx(lay, out_path, dpi=opts.dpi, target=opts.target) def main(argv=None): - ap = argparse.ArgumentParser( - prog="exactdoc", - description="High-fidelity PDF -> DOCX converter (Google Docs-safe output)") - ap.add_argument("pdf", nargs="+", help="input PDF file(s)") - ap.add_argument("-o", "--out", help="output .docx (single input only)") - ap.add_argument("--dpi", type=int, default=240, - help="raster DPI for vector figure regions (default 240)") - args = ap.parse_args(argv) - if args.out and len(args.pdf) > 1: - ap.error("-o works with a single input") - for p in args.pdf: - out = convert(p, args.out, dpi=args.dpi) - print("wrote", out) + """Deprecated alias. The console entry point is `exactdoc.cli:main`.""" + from .cli import main as _main + return _main(argv) if __name__ == "__main__": diff --git a/exactdoc/options.py b/exactdoc/options.py new file mode 100644 index 0000000..73676a6 --- /dev/null +++ b/exactdoc/options.py @@ -0,0 +1,139 @@ +"""One product profile, shared by the API, the CLI, CI, the docs and the evidence. + +There used to be three defaults, and none of them was the one the numbers came +from: + + Python API `convert()` 0 refine rounds + console CLI 2 refine rounds + CI "refined/shipped" lane 3 refine rounds + +The README quoted the three-round lane as the shipped default. So the published +0.529 within-2pt was measured on a profile that neither surface actually ran, +and "reproduce it with `convert()`" produced 0.366 -- the raw number -- with no +error anywhere to say why. A measurement that describes no shipping +configuration is not evidence, it is a coincidence. + +`PRODUCT` below is that one configuration. Every surface resolves its defaults +from it; the gate measures it by name; the docs quote its numbers. `RAW` is the +deliberate zero-refine sibling, kept because `refine()` tunes against the same +renderer the gate then measures with, so a refined-only figure can improve +because the loop memorised the oracle rather than because the converter got +better. Only the pair is meaningful, which is why both are named here rather +than passed as a number at three call sites. + +Changing `PRODUCT.refine_rounds` changes what users get AND what the published +numbers mean. Re-record the gate baseline in the same commit. +""" +import dataclasses +from typing import Optional + +# Every backend name the seam accepts, and the one that ships. `pdfium` becomes +# the default in the permissive-runtime phase; it is a real option today so that +# the parity gate can select it without monkey-patching `convert.parse_pdf`, +# which it used to do -- and which meant the gate measured a module it had +# mutated rather than the product. +BACKENDS = ("pymupdf", "pdfium") +TARGETS = ("none", "libreoffice", "gdocs") + +# Aliases accepted from users and environment variables. Kept narrow and +# explicit: a silently-unrecognised backend name would fall back to the default +# and report numbers for the wrong parser. +_BACKEND_ALIASES = {"fitz": "pymupdf", "mupdf": "pymupdf", "default": "pymupdf", + "pypdfium2": "pdfium"} +_TARGET_ALIASES = {"off": "none", "lo": "libreoffice", "soffice": "libreoffice", + "word": "libreoffice", "google": "gdocs", + "googledocs": "gdocs", "google-docs": "gdocs"} + + +def canonical_backend(name: str) -> str: + n = (name or "").strip().lower() + n = _BACKEND_ALIASES.get(n, n) + if n not in BACKENDS: + raise ValueError("unknown backend %r; choose from %s" + % (name, ", ".join(BACKENDS))) + return n + + +def canonical_target(name: str) -> str: + n = (name or "").strip().lower() + n = _TARGET_ALIASES.get(n, n) + if n not in TARGETS: + raise ValueError("unknown target %r; choose from %s" + % (name, ", ".join(TARGETS))) + return n + + +@dataclasses.dataclass(frozen=True) +class ConversionOptions: + """Immutable, validated conversion settings. + + Frozen on purpose. The writer's target mode used to be a module global that + `write_docx` set and restored, so two concurrent conversions with different + targets could each observe the other's encoding. Options that cannot be + mutated after validation are the first half of fixing that; passing them + down instead of reading a global is the second. + """ + + backend: str = "pymupdf" + target: str = "libreoffice" + refine_rounds: int = 3 + dpi: int = 240 + ladder: bool = False + verbose: bool = False + + def __post_init__(self): + object.__setattr__(self, "backend", canonical_backend(self.backend)) + object.__setattr__(self, "target", canonical_target(self.target)) + if not isinstance(self.refine_rounds, int) or self.refine_rounds < 0: + raise ValueError("refine_rounds must be a non-negative int, got %r" + % (self.refine_rounds,)) + if not isinstance(self.dpi, int) or not (36 <= self.dpi <= 1200): + raise ValueError("dpi must be an int in 36..1200, got %r" % (self.dpi,)) + + def replace(self, **kw) -> "ConversionOptions": + """A new options object with `kw` overridden. Revalidates.""" + return dataclasses.replace(self, **kw) + + def profile_id(self) -> str: + """Short stable name for reports: what was actually measured.""" + return "%s/%s/refine%d@%ddpi" % (self.backend, self.target, + self.refine_rounds, self.dpi) + + def as_dict(self) -> dict: + return dataclasses.asdict(self) + + +# The shipped configuration. This is the profile the README's numbers describe, +# the profile the CI "product" lane measures, and the profile a bare +# `convert(pdf)` or `exactdoc file.pdf` runs. +PRODUCT = ConversionOptions() + +# The uncontaminated comparison lane: no closed loop, so no chance of the +# oracle being memorised. Not a fallback and not a fast mode -- a control. +RAW = PRODUCT.replace(refine_rounds=0) + +# Kept for callers that want the name rather than the object. +DEFAULT_OPTIONS = PRODUCT +DEFAULT_BACKEND = PRODUCT.backend +DEFAULT_TARGET = PRODUCT.target +DEFAULT_REFINE_ROUNDS = PRODUCT.refine_rounds +DEFAULT_DPI = PRODUCT.dpi + +# Lane names. The gate, the baseline file and the evidence artifact all key on +# these, so they live with the profiles they describe rather than being spelled +# out as string literals in three files. +LANES = {"raw": RAW, "product": PRODUCT} + + +def resolve(options: Optional[ConversionOptions] = None, **overrides + ) -> ConversionOptions: + """The one place a surface turns partial arguments into full options. + + `None` overrides are dropped, so a CLI or API caller can pass every + argument it has and let the profile supply the rest. That is what keeps the + three surfaces from drifting apart again: none of them writes a default + value of its own. + """ + base = options if options is not None else PRODUCT + kw = {k: v for k, v in overrides.items() if v is not None} + return base.replace(**kw) if kw else base diff --git a/exactdoc/parse_pdfium.py b/exactdoc/parse_pdfium.py index 99db9eb..c6c61e9 100644 --- a/exactdoc/parse_pdfium.py +++ b/exactdoc/parse_pdfium.py @@ -41,7 +41,9 @@ # cells and the two halves of a two-column page do # exactly that, and merging them fused rows into # single lines (measured: 0.60x PyMuPDF's count). -BLOCK_GAP_FACTOR = 1.6 # line pitch multiple that ends a block +BLOCK_GAP_FACTOR = 1.15 # multiple of the BODY pitch that ends a block. See + # _body_pitch: the reference is the 20th-percentile + # gap, not the median, so the factor is close to 1. # Horizontal reach for joining lines that share a baseline. Deliberately # SHORT: it now only has to catch genuinely adjacent text, such as a list # marker and its item. Table rows are joined later by dialect._join_ruled_rows, @@ -49,6 +51,8 @@ # something no width threshold here can do, since the two have identical gap # distributions (median 4.7em each). BLOCK_SAME_ROW_EM = 1.2 +MONO_ADV_EM = 0.6 # advance of a monospaced glyph, as a fraction of size +SPACE_ADV_EM = 0.28 # advance of a space in a proportional face def _line_size(ln) -> float: @@ -61,7 +65,7 @@ def _hexcol(r, g, b): class _Char: __slots__ = ("u", "x0", "y0", "x1", "y1", "ox", "oy", "size", "font", - "flags", "color") + "flags", "color", "gen") @property def mono_hint(self) -> bool: @@ -115,16 +119,33 @@ def _page_chars(textpage, page_h) -> List[_Char]: # ink here made every line box start below the true ascent and shifted # the inferred top margin (measured 71.8pt against 67.8pt) and with it # every space_before on the page. + # ...and the same is true HORIZONTALLY, which this used to get wrong. + # PyMuPDF reports a line box that starts at the pen origin, so every + # line of a left-aligned page starts at exactly the same x. The ink box + # starts at the first glyph's ink, which moves with whichever letter + # happens to begin the line: measured on c6_long, 'L' +1.694pt, + # '1' +1.106, 'R' +0.504, 'T' +0.074, 'w' -0.011 against PyMuPDF's + # constant 61.500. That is the left side bearing, a different number + # per line, and it is unfixable downstream -- no per-page correction + # removes a per-character error. It is why c6_long could reach an IR + # identical to PyMuPDF's on lines, spans, text, spaces, styles and block + # boundaries and still score 0.46 against 0.76. + # + # Probed before being relied on (§12 law 15): loose.left equals + # FPDFText_GetCharOrigin's x to 0.000 on every character sampled, and + # equals PyMuPDF's line x0 exactly. lr = raw.FS_RECTF() if raw.FPDFText_GetLooseCharBox(textpage.raw, i, ctypes.byref(lr)): ly0, ly1 = float(lr.bottom), float(lr.top) + lx0, lx1 = float(lr.left), float(lr.right) else: ly0, ly1 = float(b.value), float(t.value) + lx0, lx1 = float(l.value), float(r_.value) c = _Char() c.u = chr(u) # flip y: PDFium is bottom-left origin, the IR is top-left - c.x0, c.x1 = float(l.value), float(r_.value) + c.x0, c.x1 = lx0, max(lx1, lx0) c.y0, c.y1 = page_h - max(ly1, float(t.value)), page_h - min(ly0, float(b.value)) c.ox, c.oy = float(ox.value), page_h - float(oy.value) # FPDFText_GetFontSize reports the size BEFORE the text matrix. Chromium @@ -142,6 +163,7 @@ def _page_chars(textpage, page_h) -> List[_Char]: size *= vs except Exception: pass + c.gen = generated # a generated space has no font of its own; inherit the run it joins if generated and out: prev = out[-1] @@ -156,6 +178,31 @@ def _page_chars(textpage, page_h) -> List[_Char]: c.flags = int(flags.value) c.color = _hexcol(cr.value, cg.value, cb.value) out.append(c) + + # A generated space carries no box of its own worth the name: PDFium + # reports it degenerate, `x..x` at a single coordinate, so the branch above + # ends up giving it [previous character's end, that coordinate] -- 1.70pt + # wide on c7_code where the real advance is 5.10pt. The remaining 3.401pt + # surfaces as a gap to the next character, and downstream that gap is + # indistinguishable from a producer who positioned words instead of emitting + # a space: a SECOND space gets synthesised on top of the one already there, + # giving `def··rerank` where PyMuPDF has `def·rerank`. + # + # It is worth exactly ONE space, though, and no more. Running it all the way + # to the next character also closes the gap between two table cells that + # happen to have a generated space in it, and _build_lines splits a row into + # visual lines on precisely that gap (LINE_SPLIT_EM): measured, that fused + # cells back into single lines and cost 01_whitepaper_market 130 lines -> 105 + # and 03_tech_report_code 73 -> 53. So the space is given one space advance, + # capped at wherever the next character actually starts. + for i, c in enumerate(out[:-1]): + if not c.gen: + continue + nxt = out[i + 1] + if abs(nxt.oy - c.oy) >= 0.5 or nxt.x0 <= c.x1: + continue + adv = (MONO_ADV_EM if c.mono_hint else SPACE_ADV_EM) * max(c.size, 1.0) + c.x1 = min(nxt.x0, max(c.x1, c.x0 + adv)) return out @@ -279,11 +326,36 @@ def _build_lines(chars: List[_Char]) -> List[Line]: for row in vis_rows: if any(_is_rtl(c.u[:1] or " ") for c in row): row = _reorder_rtl(row) + # PDFium synthesises a space at the end of a line, where the producer + # merely stopped drawing. It is line-break decoration, not content, and + # PyMuPDF does not report it: measured on 01_whitepaper_market, 25% of + # lines differed from PyMuPDF's text by exactly one trailing space -- + # `|Tier·|` against `|Tier|`, `|•·|` against `|•|`. Dropped after any + # RTL reordering, so "trailing" means the end of the logical text. + while row and row[-1].u.isspace(): + row = row[:-1] + if not row: + continue spans, cur, cur_key = [], [], None for c in row: k = _style(c) gap = (c.x0 - cur[-1].x1) if cur else 0.0 - if cur and (k != cur_key or gap > SPAN_GAP_EM * max(c.size, 1.0)): + # A span ends where the STYLE ends. A gap does not end it: a gap + # with the same style on both sides is a space the producer drew by + # positioning, and the branch below turns it into one. + # + # This condition used to also split on `gap > SPAN_GAP_EM`, which + # ran BEFORE the space-insertion branch and so consumed the gap + # instead of bridging it. Measured on c7_code: 79 of 79 intra-line + # span boundaries sat between spans of identical style, every one at + # a 3.401pt gap -- exactly one space at that size -- giving 105 spans + # for 26 lines where PyMuPDF gives 26. The text came out right and + # reached the writer as four runs per line instead of one. + # + # Nothing is left unbounded by dropping it: _build_lines has already + # split the LINE at LINE_SPLIT_EM (1.10em) further up, so every gap + # still under consideration here is small enough for spaces to span. + if cur and k != cur_key: spans.append((cur, cur_key)) cur, cur_key = [], None if not cur: @@ -296,10 +368,21 @@ def _build_lines(chars: List[_Char]) -> List[Line]: # drift on a listing. Monospace advances are ~0.6em, so the # count is recoverable from the gap; proportional text is # ~0.28em and rarely runs more than one. - adv = (0.6 if cur[-1].mono_hint else 0.28) * max(c.size, 1.0) + adv = (MONO_ADV_EM if cur[-1].mono_hint else SPACE_ADV_EM) * max(c.size, 1.0) n_sp = int(round(gap / adv)) if adv > 0 else 1 if cur[-1].u.isspace() or c.u.isspace(): - n_sp = min(n_sp, 1) if not cur[-1].mono_hint else n_sp + # A space is already there, and in proportional text that is + # the whole answer however far the gap has been stretched. + # Justified text pulls its word gaps to 7.84pt at 9.5pt type + # on 02_research_paper and PyMuPDF still reports ONE space; + # adding to it gave `Speculative··decoding` on 22% of that + # document's lines and 12% of 01_whitepaper_market's, + # displacing every word after it along the line. + # + # Monospace keeps counting: there a run length is code + # indentation, and collapsing it once cost 19 unmatched + # words and 40pt of horizontal drift on a listing. + n_sp = n_sp if cur[-1].mono_hint else 0 if n_sp >= 1: cur[-1].u += " " * min(n_sp, 24) cur.append(c) @@ -326,9 +409,92 @@ def _build_lines(chars: List[_Char]) -> List[Line]: max(s.bbox[2] for s in sp_objs), max(s.bbox[3] for s in sp_objs)) lines.append(Line(spans=sp_objs, bbox=lb)) lines.sort(key=lambda l: (round(l.bbox[1], 1), l.bbox[0])) + _reconstruct_indents(lines) return lines +def _reconstruct_indents(lines: List[Line]) -> None: + """Put back the leading indentation PDFium does not report. + + PDFium synthesises the spaces a producer drew by positioning -- but only + BETWEEN two characters, because that is the only place a gap exists to + measure. At the start of a line there is nothing to the left, so the indent + is simply absent: measured on c7_code, the raw character stream for + ` def __init__(...)` begins with 'd' at x=93.17 and contains no space at + all, while PyMuPDF reports the same line starting at x=72.25 with four + leading spaces. + + Downstream that is not a cosmetic difference. The line box starts at the + first ink instead of at the code block's left edge, so the paragraph is + written at the wrong x and every glyph on the line is displaced by the + indent. It is the whole of the code-heavy gap the defect register left + unattributed: c7_code within-2pt 0.91 -> 0.16 with 16 of its 26 lines + failing to pair with their PyMuPDF counterparts at all. + + Reconstruction needs a left edge to measure from, and the block's own + minimum will not do -- a block whose every line is indented (a continuation + inside a function body) would measure zero indent. The reference is the + leftmost line of the surrounding *monospace run*: consecutive mono lines, + which is exactly the extent of one code listing, ended by the first + proportional line. On c7_code that yields 72.25 for both listings, and the + two are separated by their heading. + + Restricted to monospace deliberately. Indentation is load-bearing in code + and decorative almost everywhere else, a proportional font has no single + advance width to divide by, and the measured defect is entirely in code + blocks. A proportional first-line indent stays where it is: expressed by + the line box, as it already was. + + Lines that SHARE a baseline are excluded, and that exclusion is not a + detail. A configuration table whose cells are set in a monospace face puts + three of them on one baseline at x=61, 153 and 223; read as a listing, the + second and third are "indented" by 18 and 32 spaces and get dragged back to + the left margin. Measured, when this function did that: + 03_tech_report_code within-2pt 0.23 -> 0.03. A line alone on its baseline is + a line of a listing; several lines on one baseline are the cells of a row, + and their x is a column position rather than an indent. + """ + # A baseline carrying more than one line is a row of cells, not a listing. + rows = {} + for ln in lines: + rows.setdefault(round(ln.baseline, 1), []).append(ln) + solo = {id(ln) for group in rows.values() if len(group) == 1 for ln in group} + + def flush(run): + if len(run) < 2: + return + left = min(l.bbox[0] for l in run) + for ln in run: + size = max((s.size for s in ln.spans), default=10.0) + adv = MONO_ADV_EM * max(size, 1.0) + n = int(round((ln.bbox[0] - left) / adv)) if adv > 0 else 0 + if n < 1: + continue + first = ln.spans[0] + first.text = " " * min(n, 40) + first.text + first.bbox = (left, first.bbox[1], first.bbox[2], first.bbox[3]) + first.origin = (left, first.origin[1]) + ln.bbox = (left, ln.bbox[1], ln.bbox[2], ln.bbox[3]) + + run = [] + for ln in lines: + inked = [s for s in ln.spans if s.text.strip()] + mono = bool(inked) and all(s.mono for s in inked) and id(ln) in solo + # A listing is contiguous. Two listings separated by other content share + # no left edge, and the run must not straddle the gap between them. + if mono and run: + pitch = max(_line_size(ln), _line_size(run[-1]), 1.0) + if ln.baseline - run[-1].baseline > 3.0 * pitch: + flush(run) + run = [] + if mono: + run.append(ln) + else: + flush(run) + run = [] + flush(run) + + def _column_split(lines: List[Line]) -> Optional[float]: """The x of a genuine column gutter on this page, or None. @@ -405,16 +571,80 @@ def _build_blocks(lines: List[Line], page_w: float = 612.0) -> List[TextBlock]: return _build_blocks_one(lines, col_x) +def _body_pitch(lines: List[Line]) -> float: + """The pitch of ordinary body text: the 20th-percentile line gap. + + This is the reference the block-split test multiplies, and it used to be the + MEDIAN gap. The median is biased upward by exactly the thing it is meant to + exclude -- the gaps *between* blocks are in the sample, and so are a table's + row pitches -- so on a page of paragraphs the tolerance grew until the + paragraph boundary fitted inside it. Measured on c6_long: body pitch 15pt, + paragraph boundaries 19.5-23.2pt, and median x 1.6 admitted anything up to + ~24pt, fusing 72 of 201 lines into the wrong block. + + The 20th percentile approximates the tightest recurring pitch on the page, + which is what body text sets, and is unmoved by however many wide gaps sit + above it. + + Evidence for the choice, and for the factor (testkit/block_gaps.py, which + labels 685 consecutive line pairs with PyMuPDF's own answer): + + gap <= median * 1.60 355/685 wrong <- shipped before this + gap <= p20 * 1.30 178/685 + gap <= p20 * 1.15 140/685 <- this + gap <= p20 * 1.05 154/685 + per-page adaptive cut 322/682 + + Note what that table also says: no fixed factor is *right*. Every document + separates cleanly on its own, at its own ratio (1.00 to 1.24 across the + corpus), and no single value serves all of them -- 140 of 685 stay wrong. + A per-page adaptive cut was measured before being written and is worse. + """ + gaps = sorted(b.baseline - a.baseline for a, b in zip(lines, lines[1:]) + if 0 < b.baseline - a.baseline < 60) + if not gaps: + return 12.0 + return gaps[max(0, int(0.2 * len(gaps)) - 1)] + + +def _pitch_by_size(lines: List[Line]) -> dict: + """Body pitch per type size, because a page can set more than one. + + The page-wide 20th percentile fixed the median's upward bias but inherited + the same shape of error in the other direction: on c7_code the code + listings run at an 11.25pt pitch, which drags the page percentile below the + 15.0pt pitch of the body text, and body paragraphs then split into one block + per line. Measured, that is the whole of that document's remaining gap -- + 3 boundary disagreements, all `pdfium SPLITS where PyMuPDF merges` at + exactly gap=15.0. + + Text of one size shares one leading, so the reference is computed within + each size and only falls back to the page when a size has too few samples + to be worth trusting. This is not the sliding window that was tried and + reverted (SESSIONS.md, `local_pitch`): a window has no idea what it is + averaging over and cut 02_research_paper's paragraphs in half, whereas a + size bucket is a property of the text itself. + """ + buckets = {} + for a, b in zip(lines, lines[1:]): + d = b.baseline - a.baseline + if not (0 < d < 60): + continue + buckets.setdefault(round(_line_size(a), 1), []).append(d) + out = {} + for size, gaps in buckets.items(): + if len(gaps) < 3: + continue + gaps.sort() + out[size] = gaps[max(0, int(0.2 * len(gaps)) - 1)] + return out + + def _build_blocks_one(lines: List[Line], col_x) -> List[TextBlock]: if not lines: return [] - pitches = [] - for a, b in zip(lines, lines[1:]): - d = b.baseline - a.baseline - if 0 < d < 60: - pitches.append(d) - pitches.sort() - typical = pitches[len(pitches) // 2] if pitches else 12.0 + typical = _body_pitch(lines) + by_size = _pitch_by_size(lines) blocks, cur = [], [lines[0]] for prev, ln in zip(lines, lines[1:]): @@ -456,7 +686,8 @@ def _build_blocks_one(lines: List[Line], col_x) -> List[TextBlock]: same = not (min(prev.bbox[2], ln.bbox[2]) <= col_x <= max(prev.bbox[0], ln.bbox[0])) else: - same = (0 < gap <= typical * BLOCK_GAP_FACTOR) and overlap > 0 + ref = by_size.get(round(_line_size(prev), 1), typical) + same = (0 < gap <= ref * BLOCK_GAP_FACTOR) and overlap > 0 if same: cur.append(ln) else: @@ -482,6 +713,33 @@ def _build_blocks_one(lines: List[Line], col_x) -> List[TextBlock]: SEG_MOVETO = raw.FPDF_SEGMENT_MOVETO # 2 +def _obj_matrix(obj): + """(a, b, c, d, e, f) for a page object, or None when unavailable.""" + try: + m = raw.FS_MATRIX() + if raw.FPDFPageObj_GetMatrix(obj.raw, ctypes.byref(m)): + return (m.a, m.b, m.c, m.d, m.e, m.f) + except Exception: + pass + return None + + +def _apply_matrix(m, x, y): + if m is None: + return x, y + a, b, c, d, e, f = m + return a * x + c * y + e, b * x + d * y + f + + +def _matrix_scale(m): + """Uniform scale factor of a matrix, for converting stroke widths.""" + if m is None: + return 1.0 + a, b, c, d, _, _ = m + s = abs(a * d - b * c) ** 0.5 + return s if s > 1e-9 else 1.0 + + def _classify(pts, w, h) -> str: has_curve = any(t == SEG_BEZIERTO for _, _, t in pts) # a rectangle arrives as MOVETO + 3-4 LINETO @@ -494,6 +752,13 @@ def _classify(pts, w, h) -> str: # which promotes their cluster straight to "figure": two callout accent # bars were enough to rasterise both callouts and 23% of a document's # text. + # + # The points reaching here are in PAGE space. They did not used to be: + # PDFium reports segment points in OBJECT space, so this test compared + # object-space dx/dy against page-space w/h. On a Chromium document that + # is every path on the page -- measured, 578 of the corpus's 612 path + # objects carry a non-identity matrix, and raw points miss the true + # bounds by up to 5438pt (testkit/backend_paths.py). if len(pts) <= 3: xs = [x for x, _, _ in pts] ys = [y for _, y, _ in pts] @@ -570,8 +835,15 @@ def _page_paths(page, page_h) -> List[DrawCmd]: if not raw.FPDFPageObj_GetBounds(obj.raw, ctypes.byref(l), ctypes.byref(b), ctypes.byref(r_), ctypes.byref(t)): continue - bbox = (float(l.value), page_h - float(t.value), - float(r_.value), page_h - float(b.value)) + bounds_bbox = (float(l.value), page_h - float(t.value), + float(r_.value), page_h - float(b.value)) + # Segment points are in OBJECT space: the path object's own matrix has + # to be applied before they mean anything on the page. Skipping it is + # not a small error -- measured across the corpus, 578 of 612 path + # objects carry a non-identity matrix (every path on every Chromium + # document) and untransformed points miss the true bounds by up to + # 5438pt. testkit/backend_paths.py measures this and keeps measuring it. + mat = _obj_matrix(obj) n = raw.FPDFPath_CountSegments(obj.raw) pts = [] for i in range(max(0, n)): @@ -580,8 +852,8 @@ def _page_paths(page, page_h) -> List[DrawCmd]: continue sx = ctypes.c_float(); sy = ctypes.c_float() raw.FPDFPathSegment_GetPoint(seg, ctypes.byref(sx), ctypes.byref(sy)) - pts.append((float(sx.value), page_h - float(sy.value), - raw.FPDFPathSegment_GetType(seg))) + px, py = _apply_matrix(mat, float(sx.value), float(sy.value)) + pts.append((px, page_h - py, raw.FPDFPathSegment_GetType(seg))) fillmode = ctypes.c_int(); stroke = ctypes.c_int() raw.FPDFPath_GetDrawMode(obj.raw, ctypes.byref(fillmode), ctypes.byref(stroke)) fr = ctypes.c_uint(); fg = ctypes.c_uint() @@ -594,10 +866,31 @@ def _page_paths(page, page_h) -> List[DrawCmd]: ctypes.byref(sb), ctypes.byref(sa)) sw = ctypes.c_float() raw.FPDFPageObj_GetStrokeWidth(obj.raw, ctypes.byref(sw)) + # ...in object space too, like the points, so it scales with the matrix. + stroke_w = float(sw.value) * _matrix_scale(mat) has_fill = fillmode.value != 0 and fa.value > 0 has_stroke = bool(stroke.value) and sa.value > 0 kind = "fillstroke" if (has_fill and has_stroke) else \ "stroke" if has_stroke else "fill" + + # GetBounds returns the INK envelope: a stroked path inflated by its + # line width in every direction. PyMuPDF returns the geometric path, and + # every threshold downstream was tuned against that. The difference + # decides structure, not appearance: a 0.75pt box border arrives 1.5pt + # wide instead of zero-width, and infer.py's table detector reads that + # bar as a column -- measured on 03_tech_report_code, a code listing was + # built as a two-column table with a 3.0pt first column and its line + # breaks discarded, where PyMuPDF builds role=code with all ten lines. + # + # Curves keep the envelope: their control points hull wider than the + # drawn curve, so for those GetBounds is the better estimate. Paths that + # yield no points keep it too. + bbox = bounds_bbox + if pts and not any(t == SEG_BEZIERTO for _, _, t in pts): + xs = [p[0] for p in pts] + ys = [p[1] for p in pts] + bbox = (min(xs), min(ys), max(xs), max(ys)) + w, h = bbox[2] - bbox[0], bbox[3] - bbox[1] fill = _hexcol(fr.value, fg.value, fb.value) if has_fill else None stroke_c = _hexcol(sr.value, sg.value, sb.value) if has_stroke else None @@ -605,7 +898,7 @@ def _page_paths(page, page_h) -> List[DrawCmd]: # Producers emit borders twice; keep one. parse.py dedupes the same way. sig = (tuple(round(v, 1) for v in bbox), kind, fill, stroke_c, - round(float(sw.value), 2), len(pts)) + round(stroke_w, 2), len(pts)) if sig in seen: continue seen.add(sig) @@ -642,7 +935,7 @@ def _page_paths(page, page_h) -> List[DrawCmd]: shape = "vline" out.append(DrawCmd( kind=kind, shape=shape, bbox=bbox, fill=fill, stroke=stroke_c, - width=float(sw.value), opacity=opacity, n_items=max(1, len(pts)))) + width=stroke_w, opacity=opacity, n_items=max(1, len(pts)))) return out diff --git a/exactdoc/targets.py b/exactdoc/targets.py index d48fc5c..38c1e33 100644 --- a/exactdoc/targets.py +++ b/exactdoc/targets.py @@ -21,8 +21,9 @@ import os from typing import Callable, Optional -TARGETS = ("none", "libreoffice", "gdocs") -DEFAULT = "libreoffice" +from .options import DEFAULT_TARGET, TARGETS, canonical_target + +DEFAULT = DEFAULT_TARGET def _libreoffice_render(docx_path: str, tmp_dir: str) -> Optional[str]: @@ -53,15 +54,20 @@ def render(docx_path, tmp_dir): def get_renderer(target: str): - """-> (render_callable | None, resolved_target_name).""" - t = (target or DEFAULT).lower() - if t in ("none", "off"): + """-> (render_callable | None, resolved_target_name). + + A resolved name that differs from the requested one is a *fallback*, and + the caller has to be able to see it: 'libreoffice' resolving to 'none' + means the conversion ran open-loop, which is a different product than the + one the user asked for. Reporting that explicitly is REL-01's job; this + function's contract is to name the target it actually resolved to. + """ + t = canonical_target(target or DEFAULT) + if t == "none": return None, "none" - if t in ("gdocs", "google", "googledocs", "google-docs"): + if t == "gdocs": return _gdocs_render_factory(), "gdocs" - if t in ("libreoffice", "lo", "word", "soffice"): - from .verify import SOFFICE - if SOFFICE is None: - return None, "none" - return _libreoffice_render, "libreoffice" - raise ValueError("unknown target %r; choose from %s" % (target, ", ".join(TARGETS))) + from .verify import SOFFICE + if SOFFICE is None: + return None, "none" + return _libreoffice_render, "libreoffice" diff --git a/pyproject.toml b/pyproject.toml index 4fc28db..9da4662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "exactdoc" -version = "0.2.0" +version = "0.1.0a1" description = "High-fidelity PDF to DOCX that survives Google Docs, with a render-back diff that proves it" readme = "README.md" requires-python = ">=3.9" @@ -16,7 +16,7 @@ keywords = [ "whitepaper", "research-paper", ] classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", "Programming Language :: Python :: 3", diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100644 index 0000000..99afda4 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# Provision a Linux machine to run exactdoc's measurement harness. +# +# The harness is this project's real product, and until now its dependencies +# were folklore: LibreOffice, headless Chromium and the metric-compatible font +# families are all required, none of them was declared anywhere, and the corpus +# generator crashed rather than said so. An executor who cannot run the gate is +# flying blind, and a gate that cannot run looks exactly like a gate that +# passes -- this repository has already paid for that lesson once (STATUS.md §5). +# +# bash scripts/bootstrap.sh provision, then report +# bash scripts/bootstrap.sh --report report only, change nothing +# bash scripts/bootstrap.sh --strict exit 1 if any capability is missing +# +# Idempotent: safe to re-run, installs only what is absent. Writes +# scripts/env.sh with the SOFFICE/CHROME paths it found; source it, or export +# them yourself. Nothing here ships in the wheel -- these are dev/CI oracles. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(dirname "$HERE")" +VENV="$ROOT/.venv" +cd "$ROOT" + +REPORT_ONLY=0 +STRICT=0 +for a in "$@"; do + case "$a" in + --report) REPORT_ONLY=1 ;; + --strict) STRICT=1 ;; + *) echo "unknown option: $a" >&2; exit 2 ;; + esac +done + +say() { printf '\n=== %s\n' "$*"; } +have() { command -v "$1" >/dev/null 2>&1; } + +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + have sudo && SUDO="sudo" +fi + +# --------------------------------------------------------------- package layer +PKG="" +if have apt-get; then PKG=apt +elif have dnf; then PKG=dnf +elif have apk; then PKG=apk +fi + +pkg_install() { + [ "$REPORT_ONLY" -eq 1 ] && return 0 + [ -z "$PKG" ] && return 1 + case "$PKG" in + apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends "$@" ;; + dnf) $SUDO dnf install -y "$@" ;; + apk) $SUDO apk add --no-cache "$@" ;; + esac +} + +if [ "$REPORT_ONLY" -eq 0 ] && [ "$PKG" = apt ]; then + say "refreshing the package index" + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq || true +fi + +# ------------------------------------------------------------ Python interpreter +# A stock container image has no python at all, so this cannot be assumed. +if ! have python3 && [ "$REPORT_ONLY" -eq 0 ]; then + say "python3" + case "$PKG" in + apt) pkg_install python3 python3-venv python3-pip ca-certificates curl ;; + dnf) pkg_install python3 python3-pip ca-certificates curl ;; + apk) pkg_install python3 py3-pip ca-certificates curl ;; + esac +fi + +# ---------------------------------------------------------------------- fonts +# LibreOffice renders the DOCX we measure. Without metric-compatible families +# it substitutes something else, every line wraps differently, and the fidelity +# numbers move for a reason that has nothing to do with the converter. +say "fonts (Liberation + DejaVu -- metric-compatible with Arial/Times/Courier)" +if have fc-list && [ -n "$(fc-list 2>/dev/null | grep -i liberation | head -1)" ]; then + echo "already present" +else + case "$PKG" in + apt) pkg_install fontconfig fonts-liberation fonts-dejavu-core ;; + dnf) pkg_install fontconfig liberation-fonts dejavu-sans-fonts dejavu-serif-fonts ;; + apk) pkg_install fontconfig font-liberation font-dejavu ;; + *) echo "no known package manager -- install Liberation and DejaVu by hand" ;; + esac + have fc-cache && [ "$REPORT_ONLY" -eq 0 ] && $SUDO fc-cache -f >/dev/null 2>&1 +fi + +# ----------------------------------------------------------------- LibreOffice +# The render-back oracle: --verify, --refine and the whole gate need it. +say "LibreOffice (the render-back oracle)" +find_soffice() { + for c in "${SOFFICE:-}" /usr/bin/soffice /usr/lib/libreoffice/program/soffice \ + /opt/libreoffice*/program/soffice "$ROOT"/.tools/squashfs-root/opt/libreoffice*/program/soffice; do + [ -n "$c" ] && [ -x "$c" ] && "$c" --version >/dev/null 2>&1 \ + && { echo "$c"; return 0; } + done + p="$(command -v soffice 2>/dev/null)" || return 1 + [ -n "$p" ] && "$p" --version >/dev/null 2>&1 && { echo "$p"; return 0; } + return 1 +} +SOFFICE_PATH="$(find_soffice || true)" +if [ -n "$SOFFICE_PATH" ]; then + echo "already present: $SOFFICE_PATH" +else + case "$PKG" in + apt) pkg_install libreoffice-writer ;; + apk) pkg_install libreoffice-writer ;; + dnf) pkg_install libreoffice-writer || true ;; + esac + SOFFICE_PATH="$(find_soffice)" +fi +if [ -z "$SOFFICE_PATH" ] && [ "$REPORT_ONLY" -eq 0 ]; then + # Fallback for distributions that do not package it (Amazon Linux 2023 does + # not). The AppImage is extracted rather than mounted: no FUSE in containers. + say "LibreOffice not packaged here -- extracting the AppImage" + mkdir -p "$ROOT/.tools" && cd "$ROOT/.tools" + if [ ! -d squashfs-root ]; then + if have curl; then + curl -fsSL -o lo.AppImage \ + https://appimages.libreitalia.org/LibreOffice-fresh.standard-x86_64.AppImage \ + && chmod +x lo.AppImage && ./lo.AppImage --appimage-extract >/dev/null + else + echo "curl not available; install LibreOffice by hand" + fi + fi + cd "$ROOT" + SOFFICE_PATH="$(find_soffice)" +fi + +# ------------------------------------------------------------- Python packages +# uv when it is available (uv.lock is the pinned truth); otherwise a plain venv, +# because modern distributions mark the system interpreter externally-managed +# and `pip install -e .` into it simply refuses (PEP 668). +# +# This runs BEFORE Chromium on purpose: the no-root fallback for Chromium is a +# Playwright download, and Playwright needs somewhere to be installed. +say "Python packages (converter + test harness + permissive backend)" +if [ "$REPORT_ONLY" -eq 0 ]; then + if have uv; then + uv sync --extra test --extra pdfium + else + [ -d "$VENV" ] || python3 -m venv "$VENV" + "$VENV/bin/pip" install --quiet --upgrade pip + "$VENV/bin/pip" install --quiet -e ".[test,pdfium]" + fi +fi + +pyrun() { + if have uv; then uv run python "$@" + elif [ -x "$VENV/bin/python" ]; then "$VENV/bin/python" "$@" + else python3 "$@"; fi +} +pyhas() { pyrun -c "import $1" >/dev/null 2>&1; } + +# -------------------------------------------------------------------- Chromium +# Generates the Chromium/Skia half of the corpus (8 of 16 documents). Not +# needed to convert a PDF, only to build the corpus. +say "Chromium (generates the Chromium/Skia corpus documents)" +# Executable is not the same as working. Ubuntu's `chromium-browser` apt package +# is a snap shim: it installs, it is on PATH, it is executable, and every +# invocation exits 1 with "requires the chromium snap to be installed" -- which +# in a container is unreachable. Probing with --version is the difference +# between a capability report that is true and one that is merely optimistic. +find_chrome() { + for c in "${CHROME:-}" /usr/bin/google-chrome /usr/bin/chromium \ + /usr/bin/chromium-browser \ + "$HOME"/.cache/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell; do + [ -n "$c" ] && [ -x "$c" ] && "$c" --version >/dev/null 2>&1 \ + && { echo "$c"; return 0; } + done + for c in chromium google-chrome; do + p="$(command -v $c 2>/dev/null)" || continue + [ -n "$p" ] && "$p" --version >/dev/null 2>&1 && { echo "$p"; return 0; } + done + return 1 +} +CHROME_PATH="$(find_chrome || true)" +if [ -n "$CHROME_PATH" ]; then + echo "already present: $CHROME_PATH" +elif [ "$REPORT_ONLY" -eq 0 ]; then + case "$PKG" in + # Not apt: on Ubuntu both `chromium` and `chromium-browser` are snap shims. + dnf) pkg_install chromium || true ;; + apk) pkg_install chromium || true ;; + esac + CHROME_PATH="$(find_chrome || true)" +fi +if [ -z "$CHROME_PATH" ] && [ "$REPORT_ONLY" -eq 0 ]; then + # Playwright's headless shell prints PDFs correctly and installs without root + # -- the fallback used when the distribution ships Chromium only as a snap. + say "no system Chromium -- trying Playwright's headless shell" + # --with-deps because the downloaded shell links against system libraries + # (libatk, libnss, ...) that a slim image does not carry: without them the + # binary is present, executable, and exits 127 on every invocation. + DEPS="" + { [ "$(id -u)" -eq 0 ] || [ -n "$SUDO" ]; } && [ "$PKG" = apt ] && DEPS="--with-deps" + if have uv; then + uv pip install --quiet playwright \ + && uv run python -m playwright install $DEPS chromium --only-shell + elif [ -x "$VENV/bin/python" ]; then + "$VENV/bin/pip" install --quiet playwright \ + && "$VENV/bin/python" -m playwright install $DEPS chromium --only-shell + fi + CHROME_PATH="$(find_chrome || true)" +fi + +# ----------------------------------------------------------------------- report +say "capability report" +status() { printf ' %-22s %s\n' "$1" "$2"; } +MISSING=0 +mark() { if [ -n "$2" ]; then status "$1" "OK $2"; else status "$1" "MISSING"; MISSING=$((MISSING+1)); fi; } + +FONTS_OK="" +have fc-list && [ -n "$(fc-list 2>/dev/null | grep -i liberation | head -1)" ] && FONTS_OK="Liberation found" +mark "fonts" "$FONTS_OK" +mark "soffice" "$SOFFICE_PATH" +mark "chromium" "$CHROME_PATH" +PYMUPDF_OK=""; pyhas fitz && PYMUPDF_OK="importable" +PDFIUM_OK=""; pyhas pypdfium2 && PDFIUM_OK="importable" +RL_OK=""; pyhas reportlab && RL_OK="importable" +mark "pymupdf" "$PYMUPDF_OK" +mark "pypdfium2" "$PDFIUM_OK" +mark "reportlab/fpdf2" "$RL_OK" + +if [ "$REPORT_ONLY" -eq 0 ]; then + { + echo "# Written by scripts/bootstrap.sh -- source this before running the harness." + [ -n "$SOFFICE_PATH" ] && echo "export SOFFICE=\"$SOFFICE_PATH\"" + [ -n "$CHROME_PATH" ] && echo "export CHROME=\"$CHROME_PATH\"" + } > "$HERE/env.sh" + status "wrote" "scripts/env.sh" +fi + +cat <` and the very next command +reported `chromium=MISSING` and generated 3 of 16 documents. CI escaped it only +because the GitHub runner image ships `/usr/bin/google-chrome`, which is +provisioning by accident. + +A missing tool makes `gen_corpus.py` skip the documents it produces and say so. +Without `--strict` it exits 0, which is right for a contributor on a thin machine +and wrong for the environment of record — so **CI passes `--strict`**, and a skip +is a failure there. A tool that is *present and failing* exits 1 either way. +Ubuntu's `chromium-browser` package is the case that motivates that second +distinction: a snap shim that installs, sits on `PATH`, and fails on every +invocation inside a container. ## Metrics @@ -55,7 +150,11 @@ distinguish a document from a photograph of a document. | File | Purpose | |---|---| | `harness.py` | the metrics; importable, or `python harness.py src.pdf out.docx workdir` | -| `runall.py` | batch convert + score + CI gate | +| `runall.py` | convert + score both lanes; produces the numbers, applies no policy of its own | +| `gate.py` | **the decision.** Pure over already-measured results, so it can be mutation-tested without a corpus | +| `corpus_manifest.py` | `verify` / `update` the exact 16-document manifest | +| `evidence.py` | the one artifact every published number traces to: commit, dependency and oracle versions, profile, corpus, both lanes, parity | +| `backend_superscript.py` | does PDFium's hardcoded `superscript=False` reach a DOCX? Measured: no | | `gen_corpus.py` | adversarial corpus across 4 producer dialects (Chromium/Skia, ReportLab, fpdf2, LibreOffice) | | `probe.py` | dump a PDF's producer, fonts, drawings and what `infer()` decided | | `wrapdiag.py` | compare source vs rendered wrap geometry; find the first diverging line per page | @@ -65,11 +164,39 @@ distinguish a document from a photograph of a document. | `exp_chromefix.py` | A/B prototype of the Chromium-dialect fix (monkey-patched, edits nothing) | | `exp_sweep.py` | sweeps the wrap-width correction, measuring line-break agreement | +### Backend-comparison instruments + +Built during the permissive-parser port, each because a question could not +otherwise be answered. They compare the two backends on the same document and +need no oracle, so they run in seconds. + +| File | Answers | +|---|---| +| `backend_parity.py` | **the swap's verdict**, against `parity_policy.json`. Marks each document REGRESSION / same / BETTER / expected-div / accepted. `--only ` for one document; `--update-policy` to re-record the accepted floors | +| `backend_geom.py` | is the *geometry* the same? (baselines, leadings, sizes, fonts) | +| `backend_spans.py` | is the *line content* the same? span boundaries, text, injected space runs, mono flags, style keys | +| `backend_paths.py` | which coordinate space are path points in? Answer: object space — 578 of 612 corpus paths carry a non-identity matrix, and untransformed points miss by up to 5438pt | +| `block_gaps.py` | does a block-split threshold exist? Plots the two distributions it must separate and scores candidate rules against PyMuPDF's own answers | +| `residual.py` | is the remaining placement error systematic or scatter? Reports the **ceiling** a perfect anchoring fix could reach, and `--hist` shows whether the error is a few lines displaced by a whole leading or every line off by a fraction | +| `margin_probe.py` | do the backends agree on the page's vertical origin, and would they under a baseline-anchored derivation? | +| `golden_ir.py` | frozen per-document parser digests. A **microscope** for locating a disagreement — `backend_parity.py` is the contract that decides whether it matters | +| `exp_regroup.py` | grafts PyMuPDF's block boundaries onto the other backend's geometry, to isolate grouping from everything else | + +Two habits these encode, both learned expensively: + +- **Probe a native API's quantity before building on it.** Object space vs page + space, ink envelope vs geometric path, before-matrix vs after-matrix font + sizes — this API has all three traps and the project has hit all three. +- **A subset run never decides.** `--only` exists for iteration speed; a change + is judged on the full corpus, because a two-document run once looked clean + while costing a third document 0.55. + ## Producer dialects `gen_corpus.py` generates from four engines because **a PDF's producer changes -its structure more than its content does**. The corpus in -`exactdoc_v1.1/corpus/` is five ReportLab documents — one dialect, authored by -the same process that was tuned against it. Chromium/Skia, the likely producer -for anything printed from a browser, was absent and is where the tool fails -hardest. +its structure more than its content does**. `corpus/make_corpus.py` adds five +ReportLab documents — one dialect, authored by the same process that was tuned +against it. Chromium/Skia, the likely producer for anything printed from a +browser, was absent from the original corpus and is where the tool failed +hardest. The two generators together make the 16-document gate corpus: +8 Chromium/Skia, 6 ReportLab, 1 fpdf2, 1 LibreOffice. diff --git a/testkit/_paths.py b/testkit/_paths.py index ae3542d..15d7171 100644 --- a/testkit/_paths.py +++ b/testkit/_paths.py @@ -1,5 +1,20 @@ -"""Shared path discovery for the testkit (no hard-coded machine paths).""" -import os, sys, glob, shutil +"""Shared path discovery for the testkit (no hard-coded machine paths). + +`scripts/bootstrap.sh` writes the oracle paths it found into `scripts/env.sh` +and tells you to source it. Nobody sources it: each CI step is its own shell, +and so is every command a contributor pastes. Measured in a bare +`ubuntu:24.04` container -- bootstrap reported `chromium OK ` +and the very next command reported `chromium=MISSING` and generated 3 of 16 +corpus documents, exit code 0. CI only escaped it because the GitHub runner +image happens to ship `/usr/bin/google-chrome`, which is provisioning by +accident. + +So this module reads `scripts/env.sh` itself. Discovery order is: an explicitly +exported variable, then what bootstrap recorded, then the search path. An +exported value always wins -- overriding the record is how you test another +build of the oracle. +""" +import os, sys, glob, re, shutil HERE = os.path.dirname(os.path.abspath(__file__)) PROJECT = os.path.dirname(HERE) @@ -9,10 +24,34 @@ if HERE not in sys.path: sys.path.insert(0, HERE) +ENV_SH = os.path.join(PROJECT, "scripts", "env.sh") + + +def _recorded(): + """{name: path} from scripts/env.sh, if bootstrap has run here.""" + out = {} + try: + with open(ENV_SH) as f: + for line in f: + m = re.match(r'\s*export\s+(\w+)\s*=\s*"?([^"\n]+)"?\s*$', line) + if m: + out[m.group(1)] = m.group(2) + except OSError: + pass + return out + + +RECORDED = _recorded() + def _first(cands, env=None): - if env and os.environ.get(env) and os.path.exists(os.environ[env]): - return os.environ[env] + if env: + v = os.environ.get(env) + if v and os.path.exists(v): + return v + v = RECORDED.get(env) + if v and os.path.exists(v): + return v for c in cands: if os.path.exists(c): return c @@ -27,7 +66,9 @@ def _first(cands, env=None): r"C:\Program Files (x86)\LibreOffice\program\soffice.exe", "/usr/bin/soffice", "/opt/libreoffice26.2/program/soffice", "/Applications/LibreOffice.app/Contents/MacOS/soffice", "soffice", -], env="SOFFICE") +] + sorted(glob.glob(os.path.join(PROJECT, ".tools", "squashfs-root", "opt", + "libreoffice*", "program", "soffice"))), + env="SOFFICE") CHROME = _first([ r"C:\Program Files\Google\Chrome\Application\chrome.exe", @@ -36,4 +77,7 @@ def _first(cands, env=None): "/usr/bin/google-chrome", "/usr/bin/chromium", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "google-chrome", "chromium", -], env="CHROME") +] + sorted(glob.glob(os.path.expanduser( + "~/.cache/ms-playwright/chromium_headless_shell-*/" + "chrome-headless-shell-linux64/chrome-headless-shell"))), + env="CHROME") diff --git a/testkit/backend_parity.py b/testkit/backend_parity.py index 4cb1675..5905a44 100644 --- a/testkit/backend_parity.py +++ b/testkit/backend_parity.py @@ -1,65 +1,99 @@ -"""The acceptance test for replacing the parser. +"""The acceptance test for replacing the parser -- and the policy it enforces. -The swap is done when pypdfium2 is not WORSE than PyMuPDF -- not when it is +The swap is done when pypdfium2 is not WORSE than PyMuPDF, not when it is perfect. Several corpus documents already fail on PyMuPDF (nested tables, rasterised SVG charts), and chasing those while believing they are swap regressions would burn the schedule on pre-existing bugs. -So this converts both backends on the same corpus with the same settings and -prints them side by side, marking each document REGRESSION / same / BETTER. -Exit code is non-zero only if a document is worse under pdfium. +So this converts both backends over the manifest corpus with the same profile +and marks each document REGRESSION / same / BETTER / expected-div / accepted. python testkit/backend_parity.py python testkit/backend_parity.py --refine 3 + python testkit/backend_parity.py --refine 3 --only c7_code + python testkit/backend_parity.py --update-policy # record the floors + +**The policy is `parity_policy.json`, not this docstring.** It used to be the +other way round: the code exited on `regressions == 0` while ROADMAP §3.2 and +STATUS D2 said two named documents were formally accepted divergences. An +executable rule that contradicts the ratified rule has one outcome -- the step +gets marked `continue-on-error` so the build stays usable, which is what +happened, and from then on nothing was gated at all. Two accepted shortfalls now +live in the policy file with numeric floors: worsening past a floor fails, and +so does clearing the divergence entirely, because an acceptance that no longer +describes reality is a stale record. + +--only takes substrings and narrows the run. The full run converts 16 documents +twice and renders both, which is minutes; a single document is seconds. It exists +so a hypothesis about one document costs that document. `--only` can never report +the swap as acceptable: the verdict needs the whole corpus, and the exit code +says so. """ import argparse -import glob +import json import os import sys import _paths # noqa: F401 +import evidence +import gate import harness +import runall + +ROOT = os.path.dirname(os.path.abspath(__file__)) +PROJECT = os.path.dirname(ROOT) +POLICY_PATH = os.path.join(ROOT, "parity_policy.json") + +# Compared in priority order, matching the gate's own: a wrong page count is the +# loudest failure, rasterised text is unrecoverable, page-level placement next, +# and fine placement last -- but *present*, because leaving within2pt out was a +# real hole. Measured: a swap this harness called clean cost within-2pt +# 0.510 -> 0.291 and median drift 0.69pt -> 2.02pt, invisibly. +DIMENSIONS = ("page_err", "live_text_cov", "word_recall", "within2pt") +LOWER_IS_BETTER = ("page_err",) + + +def load_policy(path=POLICY_PATH): + with open(path) as f: + return json.load(f) + + +def _clean(d): + return {k: v for k, v in d.items() if not k.startswith("_")} + -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Documents where the two backends genuinely disagree about what CORRECT means, -# and pdfium was verified to be the right one. The harness measures agreement -# with the incumbent, so on these it would otherwise demand that a bug be -# reproduced. Nothing goes in here without rendered evidence. -# -# c4_i18n -- PDFium reports RTL glyphs in visual order and this backend -# reorders them to logical, which is what a DOCX must carry: Word and -# LibreOffice apply their own bidi to logical text. PyMuPDF returns visual -# order, so its DOCX renders Arabic BACKWARDS. Verified by rendering the -# source and both outputs side by side: the source and the pdfium output -# read 'تتدهور جودة الاسترجاع...', the PyMuPDF output reads -# '...التضمين نموذج معايرة' -- the same words in reverse. -# c5_graphics -- the page opens with a gradient band carrying white text. -# PyMuPDF does not report the gradient at all, so exactdoc emits the band's -# text with no background: white on white, invisible. PDFium reports the -# pattern flattened to a grey fill, so the band survives and its text is -# legible, at the cost of grey instead of blue. Verified by rendering: -# PyMuPDF's output simply has no band, pdfium's has a grey one with the -# heading readable. -# -# Note what this exposes in the metric: live_text_cov scored PyMuPDF HIGHER -# (0.71 against 0.68), because invisible white text still counts as live -# while pdfium's legible text is partly inside a rasterised region. Text -# coverage cannot see contrast, so on documents with knocked-out text it -# rewards losing the background. Worth remembering before trusting it alone. -EXPECTED_DIVERGENCE = { - "c4_i18n": "RTL: pdfium emits logical order, PyMuPDF emits visual (renders " - "backwards). Verified visually.", - "c5_graphics": "gradient band: PyMuPDF drops it and its white text becomes " - "invisible; pdfium keeps it legible. Verified visually.", -} +def dims(res): + return {"page_err": abs(res["out_pages"] - res["src_pages"]), + "live_text_cov": res.get("live_text_cov", 0.0), + "word_recall": res.get("word_recall", 0.0), + "within2pt": res.get("within2pt", 0.0)} + + +def compare(a, b, margins): + """-> (verdict, dimension) where verdict is 'worse' | 'better' | None.""" + da, db = dims(a), dims(b) + for name in DIMENSIONS: + margin = margins.get(name, 0) + delta = db[name] - da[name] + if abs(delta) <= margin: + continue + if name in LOWER_IS_BETTER: + return ("worse" if delta > 0 else "better"), name + return ("worse" if delta < 0 else "better"), name + return None, None def run(backend, srcs, out_root, refine): - import exactdoc.convert as C - from exactdoc.parse import parse_pdf as mu - from exactdoc.parse_pdfium import parse_pdf as pf - C.parse_pdf = mu if backend == "pymupdf" else pf + """Convert the corpus with one backend. No monkey-patching. + + This used to reassign `exactdoc.convert.parse_pdf`, so the gate measured a + module it had mutated rather than the product, and the mutation silently + bypassed whatever backend selection `convert()` would have done itself. + """ + from exactdoc.options import PRODUCT + from exactdoc.convert import convert + + options = PRODUCT.replace(backend=backend, refine_rounds=refine) out = os.path.join(out_root, backend) os.makedirs(out, exist_ok=True) pairs = [] @@ -67,7 +101,7 @@ def run(backend, srcs, out_root, refine): n = os.path.splitext(os.path.basename(s))[0] dx = os.path.join(out, n + ".docx") try: - C.convert(s, dx, refine_rounds=refine) + convert(s, dx, options=options) pairs.append((s, dx, n)) except Exception as e: print(" CONVERT FAIL [%s] %-22s %s" % (backend, n[:22], str(e)[:50])) @@ -75,82 +109,202 @@ def run(backend, srcs, out_root, refine): res = {} for s, dx, n in pairs: try: - res[n] = harness.evaluate(s, dx, os.path.join(out, "r"), - save_images=False) + res[os.path.basename(s)] = harness.evaluate( + s, dx, os.path.join(out, "r"), save_images=False) except Exception as e: print(" EVAL FAIL [%s] %-22s %s" % (backend, n[:22], str(e)[:50])) return res -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--refine", type=int, default=0) +def adjudicate(ref, cand, policy, subset=False): + """Apply the policy. -> (rows, summary dict).""" + margins = _clean(policy.get("margins", {})) + divergence = _clean(policy.get("expected_divergence", {})) + accepted = _clean(policy.get("accepted_shortfalls", {})) + rows, failures = [], [] + counts = {"regressions": 0, "same": 0, "better": 0, "expected_div": 0, + "accepted": 0, "missing": 0} + + for doc_id in sorted(set(ref) | set(cand)): + A, B = ref.get(doc_id), cand.get(doc_id) + if not A or not B: + counts["missing"] += 1 + failures.append(("missing", doc_id, + "scored under %s only -- a document that cannot be " + "compared is not a document that agrees" + % ("reference" if A else "candidate"))) + rows.append({"document": doc_id, "verdict": "MISSING"}) + continue + state, dim = compare(A, B, margins) + row = {"document": doc_id, "reference": dims(A), "candidate": dims(B), + "dimension": dim} + + if doc_id in divergence: + row["verdict"] = "expected-div" + counts["expected_div"] += 1 + elif doc_id in accepted: + spec = accepted[doc_id] + row["verdict"] = "accepted" + row["defect"] = spec.get("defect") + counts["accepted"] += 1 + if not spec.get("defect"): + failures.append(("undocumented", doc_id, + "accepted shortfall with no defect ID")) + floors = spec.get("floors") + if floors is None: + failures.append(("unrecorded", doc_id, + "accepted with no numeric floors -- record them " + "with --update-policy on the canonical " + "environment. An unbounded acceptance is an " + "acceptance of anything")) + else: + cd = dims(B) + for name, floor in sorted(_clean(floors).items()): + v = cd.get(name) + if v is None: + continue + tol = gate.METRICS.get(name, {}).get("tol", 0) + bad = (v > floor + tol) if name in LOWER_IS_BETTER \ + else (v < floor - tol) + if bad: + failures.append(("below-floor", doc_id, + "%s %.4g against a ratified floor of " + "%.4g" % (name, v, floor))) + if state != "worse": + failures.append(("stale", doc_id, + "accepted as worse, but the candidate is no " + "longer worse (%s). A stale acceptance hides " + "the next real regression on this document" + % (state or "equal"))) + elif state == "worse": + row["verdict"] = "REGRESSION" + counts["regressions"] += 1 + failures.append(("regression", doc_id, + "worse on %s: %.4g -> %.4g" % + (dim, dims(A)[dim], dims(B)[dim]))) + elif state == "better": + row["verdict"] = "BETTER" + counts["better"] += 1 + else: + row["verdict"] = "same" + counts["same"] += 1 + rows.append(row) + + ok = not failures and not subset + summary = dict(counts) + summary.update({"ok": ok, "subset": subset, + "failures": [{"kind": k, "document": d, "detail": v} + for k, d, v in failures]}) + return rows, summary + + +def record_policy(ref, cand, policy, path=POLICY_PATH): + """Write the measured floors for each accepted shortfall.""" + accepted = policy.get("accepted_shortfalls", {}) + for doc_id, spec in accepted.items(): + if doc_id.startswith("_") or doc_id not in cand: + continue + spec["floors"] = {k: round(v, 4) for k, v in dims(cand[doc_id]).items()} + spec["reference_at_record"] = {k: round(v, 4) + for k, v in dims(ref[doc_id]).items()} + with open(path, "w") as f: + json.dump(policy, f, indent=1, sort_keys=True) + f.write("\n") + print("recorded floors for %d accepted shortfall(s) in %s" + % (sum(1 for k in accepted if not k.startswith("_")), path)) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--refine", type=int, default=None, + help="refine rounds for both backends (default: the product " + "profile's)") ap.add_argument("--out", default=None) - a = ap.parse_args() + ap.add_argument("--only", nargs="+", default=None, + help="substrings; run only the matching documents") + ap.add_argument("--update-policy", action="store_true", + help="record the accepted shortfalls' numeric floors") + ap.add_argument("--evidence", default=None, + help="evidence JSON to merge the parity verdict into") + a = ap.parse_args(argv) - srcs = sorted(glob.glob(os.path.join(ROOT, "corpus", "pdfs", "*.pdf"))) - srcs += sorted(glob.glob(os.path.join(ROOT, "testkit", "adv", "*.pdf"))) + from exactdoc.options import PRODUCT + refine = PRODUCT.refine_rounds if a.refine is None else a.refine + + manifest = gate.load_manifest() + if manifest is None: + print("no corpus manifest -- parity cannot know which documents it " + "should have compared") + return 2 + srcs, problems = runall.resolve_corpus(manifest) + for kind, doc, why in problems: + print("CORPUS %-11s %-28s %s" % (kind, doc[:28], why)) if not srcs: print("no corpus; run the generators first") return 2 - out_root = a.out or os.path.join(ROOT, "testkit", "parity") - mu = run("pymupdf", srcs, out_root, a.refine) - pf = run("pdfium", srcs, out_root, a.refine) + subset = False + if a.only: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.only)] + if not srcs: + print("--only matched no document") + return 2 + subset = True + print("subset run: %s -- this cannot report the swap as acceptable, " + "only the full corpus can" + % ", ".join(os.path.basename(s) for s in srcs)) - print("\n%-22s %-16s %-16s %s" % ("document", "pymupdf", "pdfium", "verdict")) - worse = same = better = 0 - for n in sorted(set(mu) | set(pf)): - A, B = mu.get(n), pf.get(n) - if not A or not B: - print("%-22s %-16s %-16s MISSING" % (n[:22], bool(A), bool(B))) - worse += 1 + policy = load_policy() + out_root = a.out or os.path.join(ROOT, "parity") + ref_name = policy.get("reference_backend", "pymupdf") + cand_name = policy.get("candidate_backend", "pdfium") + print("reference %s vs candidate %s, refine %d, %d document(s)" + % (ref_name, cand_name, refine, len(srcs))) + + ref = run(ref_name, srcs, out_root, refine) + cand = run(cand_name, srcs, out_root, refine) + + if a.update_policy: + record_policy(ref, cand, policy) + policy = load_policy() + + rows, summary = adjudicate(ref, cand, policy, subset=subset) + print("\n%-22s %-22s %-22s %s" + % ("document", ref_name, cand_name, "verdict")) + for row in rows: + if row["verdict"] == "MISSING": + print("%-22s %-22s %-22s MISSING" % (row["document"][:22], "-", "-")) continue - # Lexicographic on (page error, live text, placement), matching the - # gate's own priorities: page_match is its first criterion because a - # wrong page count is the loudest failure, live text next because - # rasterised text is unrecoverable, placement last. - # - # A flat "any live-text drop is a regression" rule reported c5_graphics - # as worse for a 0.03 dip while it gained a correct page count and 0.45 - # of placement -- that document rasterises an SVG chart by design, so - # small live-text differences there are noise, not loss. Comparing - # dimensions in priority order says what a reader would say. - dp_a = abs(A["out_pages"] - A["src_pages"]) - dp_b = abs(B["out_pages"] - B["src_pages"]) - lv_a, lv_b = A["live_text_cov"], B["live_text_cov"] - pl_a, pl_b = A.get("word_recall", 0), B.get("word_recall", 0) - # within2pt is FINE placement, and leaving it out was a real hole: - # a backend can put every word on the right page (word_recall) while - # putting none of them in the right spot. Measured that way, a swap - # that this harness called clean cost within-2pt 0.510 -> 0.291 and - # median drift 0.69pt -> 2.02pt across the gate, invisibly. - w2_a, w2_b = A.get("within2pt", 0), B.get("within2pt", 0) - if dp_b != dp_a: - worse_doc = dp_b > dp_a - elif abs(lv_b - lv_a) > 0.05: - worse_doc = lv_b < lv_a - elif abs(pl_b - pl_a) > 0.05: - worse_doc = pl_b < pl_a - elif abs(w2_b - w2_a) > 0.08: - worse_doc = w2_b < w2_a - else: - worse_doc = None - if n in EXPECTED_DIVERGENCE: - v, same = "expected-div", same + 1 - elif worse_doc is True: - v, worse = "REGRESSION", worse + 1 - elif worse_doc is False: - v, better = "BETTER", better + 1 - else: - v, same = "same", same + 1 - print("%-20s %s/%-2s l%.2f p%.2f w%.2f %s/%-2s l%.2f p%.2f w%.2f %s" % ( - n[:20], A["src_pages"], A["out_pages"], lv_a, pl_a, w2_a, - B["src_pages"], B["out_pages"], lv_b, pl_b, w2_b, v)) - - print("\n%d regressions, %d same, %d better" % (worse, same, better)) - print("swap is acceptable when regressions == 0") - return 1 if worse else 0 + r, c = row["reference"], row["candidate"] + fmt = "pg%+d l%.2f p%.2f w%.2f" + print("%-22s %-22s %-22s %s" + % (row["document"][:22], + fmt % (r["page_err"], r["live_text_cov"], r["word_recall"], + r["within2pt"]), + fmt % (c["page_err"], c["live_text_cov"], c["word_recall"], + c["within2pt"]), + row["verdict"])) + + print("\n%d regression(s), %d same, %d better, %d expected-divergence, " + "%d accepted, %d missing" + % (summary["regressions"], summary["same"], summary["better"], + summary["expected_div"], summary["accepted"], summary["missing"])) + for f in summary["failures"]: + print(" %-13s %-28s %s" % (f["kind"], f["document"][:28], f["detail"])) + if subset: + print("subset run: exit code reports findings among what it ran, and " + "cannot report the swap as acceptable") + print("PASS" if summary["ok"] else "FAIL") + + ev_path = a.evidence or os.path.join(ROOT, "batch", "evidence.json") + parity = dict(summary) + parity.update({"reference_backend": ref_name, "candidate_backend": cand_name, + "refine_rounds": refine, "documents": rows}) + evidence.merge(ev_path, parity=parity) + + if a.update_policy: + return 0 + return 0 if summary["ok"] else 1 if __name__ == "__main__": diff --git a/testkit/backend_paths.py b/testkit/backend_paths.py new file mode 100644 index 0000000..6faa91c --- /dev/null +++ b/testkit/backend_paths.py @@ -0,0 +1,210 @@ +"""Which coordinate space are PDFium's path segment points in? + +Written because getting this wrong cost a session. `parse_pdfium._page_paths` +took path bounding boxes from `FPDFPageObj_GetBounds` (page space, and the INK +envelope: a stroked path inflated by its line width) while `_classify`, +`_rect_pts` and the frame-edge decomposition read `FPDFPath_GetPathSegment` +points -- which PDFium reports in **object space**, before the path object's own +transform. On a Chromium document every path carries a non-identity matrix, so +those two families of numbers are not comparable, and an experiment that +replaced the bbox with a raw-points bbox silently scaled the whole page. + +The failure mode is invisible at the microscope: pick two paths to eyeball and +you may well pick identity-matrix ones, which agree perfectly. It is only +visible across a corpus, which is what this measures. + + python testkit/backend_paths.py # the whole corpus + python testkit/backend_paths.py 03_tech c7_code # named documents + python testkit/backend_paths.py --show 6 c1 # per-path detail + +Reads: raw points, matrix-transformed points, and GetBounds, per path object. +Reports how often each reconstructs GetBounds, and by how much they miss. +""" +import argparse +import ctypes +import glob +import os +import sys + +import pypdfium2 as pdfium +import pypdfium2.raw as raw + +import _paths # noqa: F401 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# GetBounds inflates a stroked path by its line width in each direction, so a +# geometric bbox reconstructs it only after that envelope is added back. 0.6pt +# of slack absorbs bezier control points and PDFium's own rounding. +TOL = 0.6 + + +def _matrix(obj): + """(a, b, c, d, e, f) for a path object, or None if unavailable.""" + try: + m = raw.FS_MATRIX() + if raw.FPDFPageObj_GetMatrix(obj.raw, ctypes.byref(m)): + return (m.a, m.b, m.c, m.d, m.e, m.f) + except Exception: + pass + return None + + +def _is_identity(m): + if m is None: + return True + a, b, c, d, e, f = m + return (abs(a - 1) < 1e-6 and abs(b) < 1e-6 and abs(c) < 1e-6 + and abs(d - 1) < 1e-6 and abs(e) < 1e-6 and abs(f) < 1e-6) + + +def _apply(m, x, y): + if m is None: + return x, y + a, b, c, d, e, f = m + return a * x + c * y + e, b * x + d * y + f + + +def _bbox(points): + xs = [p[0] for p in points] + ys = [p[1] for p in points] + return (min(xs), min(ys), max(xs), max(ys)) + + +def _miss(geom, bounds, sw): + """How far the geometric bbox is from GetBounds once the stroke envelope + is added back. 0.0 means it reconstructs it exactly. + + `sw` must be 0 for an unstroked path: PDFium reports a stroke WIDTH of 1.0 + on fill-only objects, which is not a stroke and does not inflate anything. + Inflating by it regardless made every filled rect miss by exactly 1.00pt -- + a suspiciously round constant, which is what gave the mistake away. + """ + inflated = (geom[0] - sw, geom[1] - sw, geom[2] + sw, geom[3] + sw) + return max(abs(inflated[i] - bounds[i]) for i in range(4)) + + +def _stroke_width(obj, m): + """Stroke width in PAGE space, or 0.0 when the path is not stroked.""" + fillmode = ctypes.c_int(); stroke = ctypes.c_int() + if not raw.FPDFPath_GetDrawMode(obj.raw, ctypes.byref(fillmode), + ctypes.byref(stroke)): + return 0.0 + if not stroke.value: + return 0.0 + sa = ctypes.c_uint() + r_ = ctypes.c_uint(); g = ctypes.c_uint(); b = ctypes.c_uint() + raw.FPDFPageObj_GetStrokeColor(obj.raw, ctypes.byref(r_), ctypes.byref(g), + ctypes.byref(b), ctypes.byref(sa)) + if sa.value == 0: + return 0.0 + sw = ctypes.c_float() + raw.FPDFPageObj_GetStrokeWidth(obj.raw, ctypes.byref(sw)) + w = float(sw.value) + # The width is in object space like the points, so it scales with the matrix. + if m is not None: + a, b_, c, d, _, _ = m + scale = (abs(a * d - b_ * c)) ** 0.5 + if scale > 1e-9: + w *= scale + return w + + +def report(path, show=0): + name = os.path.splitext(os.path.basename(path))[0] + pdf = pdfium.PdfDocument(path) + n_paths = n_nonid = raw_ok = tx_ok = 0 + raw_worst = tx_worst = 0.0 + detail = [] + for pno in range(len(pdf)): + page = pdf[pno] + for obj in page.get_objects(): + try: + if raw.FPDFPageObj_GetType(obj.raw) != raw.FPDF_PAGEOBJ_PATH: + continue + except Exception: + continue + l = ctypes.c_float(); b = ctypes.c_float() + r_ = ctypes.c_float(); t = ctypes.c_float() + if not raw.FPDFPageObj_GetBounds(obj.raw, ctypes.byref(l), + ctypes.byref(b), ctypes.byref(r_), + ctypes.byref(t)): + continue + bounds = (float(l.value), float(b.value), + float(r_.value), float(t.value)) + pts = [] + for i in range(max(0, raw.FPDFPath_CountSegments(obj.raw))): + seg = raw.FPDFPath_GetPathSegment(obj.raw, i) + if not seg: + continue + sx = ctypes.c_float(); sy = ctypes.c_float() + raw.FPDFPathSegment_GetPoint(seg, ctypes.byref(sx), ctypes.byref(sy)) + pts.append((float(sx.value), float(sy.value))) + if not pts: + continue + m = _matrix(obj) + sw_page = _stroke_width(obj, m) + + n_paths += 1 + if not _is_identity(m): + n_nonid += 1 + rb = _bbox(pts) + tb = _bbox([_apply(m, x, y) for x, y in pts]) + rmiss = _miss(rb, bounds, sw_page) + tmiss = _miss(tb, bounds, sw_page) + raw_ok += rmiss <= TOL + tx_ok += tmiss <= TOL + raw_worst = max(raw_worst, rmiss) + tx_worst = max(tx_worst, tmiss) + if len(detail) < show: + detail.append((m, rb, tb, bounds, sw_page, rmiss, tmiss)) + pdf.close() + + print("\n== %s" % name) + print(" path objects %d" % n_paths) + print(" non-identity matrix %d of %d" % (n_nonid, n_paths)) + print(" raw points reconstruct GetBounds %d of %d (worst miss %.2fpt)" + % (raw_ok, n_paths, raw_worst)) + print(" matrix-transformed reconstruct GetBounds %d of %d (worst miss %.2fpt)" + % (tx_ok, n_paths, tx_worst)) + for m, rb, tb, bounds, sw, rmiss, tmiss in detail: + print(" matrix %s" % (None if m is None else + "(%.3f %.3f %.3f %.3f %.1f %.1f)" % m)) + print(" raw %s miss %.2f" % (_fmt(rb), rmiss)) + print(" matrix %s miss %.2f" % (_fmt(tb), tmiss)) + print(" bounds %s stroke %.2f" % (_fmt(bounds), sw)) + return {"paths": n_paths, "nonid": n_nonid, "raw_ok": raw_ok, + "tx_ok": tx_ok, "raw_worst": raw_worst, "tx_worst": tx_worst} + + +def _fmt(b): + return "x=%7.2f..%-7.2f y=%7.2f..%-7.2f" % (b[0], b[2], b[1], b[3]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("names", nargs="*") + ap.add_argument("--show", type=int, default=0, + help="print N per-path examples") + a = ap.parse_args() + srcs = sorted(glob.glob(os.path.join(ROOT, "corpus", "pdfs", "*.pdf"))) + srcs += sorted(glob.glob(os.path.join(ROOT, "testkit", "adv", "*.pdf"))) + if a.names: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.names)] + if not srcs: + print("no matching corpus documents") + return 2 + tot = {"paths": 0, "nonid": 0, "raw_ok": 0, "tx_ok": 0} + for s in srcs: + r = report(s, a.show) + for k in tot: + tot[k] += r[k] + print("\n%-24s %d path objects, %d with a non-identity matrix" + % ("CORPUS TOTAL", tot["paths"], tot["nonid"])) + print("%-24s raw points reconstruct %d, matrix-transformed reconstruct %d" + % ("", tot["raw_ok"], tot["tx_ok"])) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/backend_spans.py b/testkit/backend_spans.py new file mode 100644 index 0000000..13e8557 --- /dev/null +++ b/testkit/backend_spans.py @@ -0,0 +1,177 @@ +"""Diff the two backends at SPAN level, inside matched lines. + +backend_geom.py answered "is the geometry the same?" -- yes: baselines +identical on 4,734 of 4,734 lines, leadings 99-100%, sizes to 0.005pt. +exp_regroup.py answered "is it the block grouping?" -- for about half the +failing documents, yes. Two documents are explained by neither, both +code-heavy (c7_code 0.92 -> 0.16, 03_tech_report_code 0.46 -> 0.23), and the +cause has never been measured. This instrument exists to stop that being +guessed at. + +It pairs lines across backends by baseline and x, then reports, per document: + + spans/line do the two agree on where a line divides into runs? + text is the CHARACTER content of the matched line identical? + space runs how many runs of >=2 spaces, and how wide -- the synthesised + indentation that _build_lines reconstructs from a gap + mono flags does one backend think the line is monospaced and the other not + style keys font/size/bold/italic/serif per span + +Text differences are printed with the space runs made visible, because the +suspected failure mode is invisible: the same words with a different number of +spaces between them reads identically in a terminal. + + python testkit/backend_spans.py # the whole corpus + python testkit/backend_spans.py c7_code 03_tech # named documents + python testkit/backend_spans.py --show 12 c7_code # with example lines +""" +import argparse +import glob +import os +import sys +from collections import Counter + +import _paths # noqa: F401 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +BASELINE_TOL = 0.6 # pt; the backends agree on baselines far tighter +X_TOL = 2.0 # pt; line start + + +def _lines(ir): + """[(page_no, baseline, x0, line)] flattened, in reading order.""" + out = [] + for pno, p in enumerate(ir.pages, 1): + for b in p.blocks: + for ln in b.lines: + out.append((pno, ln.baseline, ln.bbox[0], ln)) + out.sort(key=lambda t: (t[0], round(t[1], 1), t[2])) + return out + + +def _pair(a_lines, b_lines): + """Match lines across backends on (page, baseline, x). Unmatched reported.""" + used, pairs, unmatched = set(), [], [] + b_by_page = {} + for i, (pno, base, x0, ln) in enumerate(b_lines): + b_by_page.setdefault(pno, []).append(i) + for pno, base, x0, ln in a_lines: + best, best_d = None, None + for i in b_by_page.get(pno, ()): + if i in used: + continue + _, bb, bx, _ = b_lines[i] + if abs(bb - base) > BASELINE_TOL or abs(bx - x0) > X_TOL: + continue + d = abs(bb - base) + abs(bx - x0) / 10.0 + if best_d is None or d < best_d: + best, best_d = i, d + if best is None: + unmatched.append((pno, base, ln)) + else: + used.add(best) + pairs.append((ln, b_lines[best][3])) + return pairs, unmatched + + +def _space_runs(text): + """[(start_index, length)] for every run of 2+ spaces.""" + runs, i = [], 0 + while i < len(text): + if text[i] == " ": + j = i + while j < len(text) and text[j] == " ": + j += 1 + if j - i >= 2: + runs.append((i, j - i)) + i = j + else: + i += 1 + return runs + + +def _visible(text): + return text.replace(" ", "·") # middle dot: spaces you can count + + +def _stylekey(sp): + return (sp.font, round(sp.size, 2), sp.bold, sp.italic, sp.mono, sp.serif) + + +def report(path, show=0): + from exactdoc.parse import parse_pdf as mu + from exactdoc.parse_pdfium import parse_pdf as pf + + name = os.path.splitext(os.path.basename(path))[0] + a, b = _lines(mu(path, keep_image_data=False)), _lines(pf(path, keep_image_data=False)) + pairs, unmatched = _pair(a, b) + + n_span_diff = n_text_diff = n_mono_diff = n_style_diff = n_space_diff = 0 + span_delta = Counter() + examples = [] + for la, lb in pairs: + ta = "".join(s.text for s in la.spans) + tb = "".join(s.text for s in lb.spans) + d_span = len(lb.spans) - len(la.spans) + if d_span: + n_span_diff += 1 + span_delta[d_span] += 1 + ra, rb = _space_runs(ta), _space_runs(tb) + differs_text = ta != tb + if differs_text: + n_text_diff += 1 + if [n for _, n in ra] != [n for _, n in rb]: + n_space_diff += 1 + if any(s.mono for s in la.spans) != any(s.mono for s in lb.spans): + n_mono_diff += 1 + if [_stylekey(s) for s in la.spans] != [_stylekey(s) for s in lb.spans]: + n_style_diff += 1 + if differs_text and len(examples) < show: + examples.append((ta, tb, ra, rb)) + + n = max(1, len(pairs)) + print("\n== %s" % name) + print(" lines mupdf %d, pdfium %d, matched %d, unmatched %d" + % (len(a), len(b), len(pairs), len(unmatched))) + print(" span count diff %d/%d lines (%.0f%%) deltas %s" + % (n_span_diff, len(pairs), 100.0 * n_span_diff / n, + dict(span_delta.most_common(5)))) + print(" TEXT diff %d/%d lines (%.0f%%)" + % (n_text_diff, len(pairs), 100.0 * n_text_diff / n)) + print(" space-run diff %d/%d lines (%.0f%%)" + % (n_space_diff, len(pairs), 100.0 * n_space_diff / n)) + print(" mono flag diff %d/%d lines" % (n_mono_diff, len(pairs))) + print(" style key diff %d/%d lines" % (n_style_diff, len(pairs))) + for ta, tb, ra, rb in examples: + print(" mupdf |%s| runs=%s" % (_visible(ta)[:96], [n for _, n in ra])) + print(" pdfium |%s| runs=%s" % (_visible(tb)[:96], [n for _, n in rb])) + return {"text_diff": n_text_diff, "pairs": len(pairs), + "space_diff": n_space_diff, "span_diff": n_span_diff} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("names", nargs="*", help="substrings; default is everything") + ap.add_argument("--show", type=int, default=0, + help="print N example lines whose text differs") + a = ap.parse_args() + + srcs = sorted(glob.glob(os.path.join(ROOT, "corpus", "pdfs", "*.pdf"))) + srcs += sorted(glob.glob(os.path.join(ROOT, "testkit", "adv", "*.pdf"))) + if a.names: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.names)] + if not srcs: + print("no matching corpus documents") + return 2 + for s in srcs: + try: + report(s, a.show) + except Exception as e: + print("\n== %s\n FAILED: %s: %s" + % (os.path.basename(s), type(e).__name__, e)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/backend_superscript.py b/testkit/backend_superscript.py new file mode 100644 index 0000000..ab4f545 --- /dev/null +++ b/testkit/backend_superscript.py @@ -0,0 +1,129 @@ +"""Does PDFium's hardcoded `superscript=False` cost anything? + +`parse_pdfium.py` reports `superscript=False` for every span, where `parse.py` +reads PyMuPDF's font flag. ROADMAP §3.1 queued "implement it in the backend" +ahead of the licence flip. But the writer does not read the parser's flag -- it +reads the *layout*, and two shared stages set the flag themselves from geometry +alone: `dialect._merge_row_lines` and `infer` both promote a small fragment +sitting above its host line's baseline (measured inside the em box, raised more +than 0.12x the host size). Neither looks at the backend. + +So the question is not "does PDFium report superscript" but "does the DOCX carry +the same superscript runs either way", and that is measurable before writing any +backend code. This script answers it at both levels: + + parse spans flagged by the parser itself + layout runs flagged after normalize() + infer(), which is what reaches the + writer, plus the text of those runs so a count that matches by + accident is still visible as a mismatch + + python testkit/backend_superscript.py + python testkit/backend_superscript.py --only 02_research_paper + +Exit non-zero if the layout-level flags disagree: that is the only level where +disagreement can reach a user. +""" +import argparse +import os +import sys + +import _paths # noqa: F401 +import gate +import runall + + +def parse_level(backend, path): + from exactdoc.backend import get_backend + ir = get_backend(backend).parse_pdf(path, keep_image_data=False) + hits = [] + for p in ir.pages: + for b in p.blocks: + for l in b.lines: + for s in l.spans: + if s.superscript: + hits.append((p.number, s.text)) + return ir, hits + + +def layout_level(ir): + """Superscript runs as the writer will see them: after the shared stages.""" + from exactdoc.dialect import normalize + from exactdoc.infer import infer + lay = infer(normalize(ir)) + hits = [] + for pg in lay.pages: + for ch in pg.chunks: + for el in ch.elements: + for r in getattr(el, "runs", ()) or (): + if getattr(r, "superscript", False): + hits.append((getattr(el, "page_no", pg.number), r.text)) + return hits + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--only", nargs="+", default=None) + ap.add_argument("--reference", default="pymupdf") + ap.add_argument("--candidate", default="pdfium") + a = ap.parse_args(argv) + + manifest = gate.load_manifest() + if manifest is None: + print("no corpus manifest") + return 2 + srcs, problems = runall.resolve_corpus(manifest) + for kind, doc, why in problems: + print("CORPUS %-11s %-28s %s" % (kind, doc[:28], why)) + if a.only: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.only)] + if not srcs: + print("no documents to compare") + return 2 + + print("%-24s %-17s %-17s %s" + % ("document", "parse ref/cand", "layout ref/cand", "verdict")) + disagree = [] + for s in srcs: + name = os.path.basename(s) + try: + ir_a, pa = parse_level(a.reference, s) + ir_b, pb = parse_level(a.candidate, s) + la, lb = layout_level(ir_a), layout_level(ir_b) + except Exception as e: + print("%-24s %s: %s" % (name[:24], type(e).__name__, str(e)[:40])) + disagree.append((name, "error")) + continue + ta = sorted(t.strip() for _, t in la if t.strip()) + tb = sorted(t.strip() for _, t in lb if t.strip()) + same = ta == tb + verdict = "same" if same else "DIFFERS" + if not same: + disagree.append((name, "%d vs %d runs" % (len(ta), len(tb)))) + print("%-24s %-17s %-17s %s" + % (name[:24], "%d/%d" % (len(pa), len(pb)), + "%d/%d" % (len(la), len(lb)), verdict)) + if not same: + only_a = [t for t in ta if t not in tb][:6] + only_b = [t for t in tb if t not in ta][:6] + if only_a: + print(" only %s: %s" % (a.reference, only_a)) + if only_b: + print(" only %s: %s" % (a.candidate, only_b)) + + print("\n%d of %d document(s) disagree at the layout level" + % (len(disagree), len(srcs))) + if not disagree: + print("The parser flag is not load-bearing: normalize() and infer() " + "recover superscript from geometry, so the backend hardcode costs " + "nothing that reaches a DOCX. ROADMAP §3.1 is answered by " + "measurement rather than by implementation.") + return 0 + for name, why in disagree: + print(" %-28s %s" % (name[:28], why)) + print("\nThe flag IS load-bearing on the documents above: implement it in " + "the candidate backend, then re-run this.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/block_gaps.py b/testkit/block_gaps.py new file mode 100644 index 0000000..3fe9cf6 --- /dev/null +++ b/testkit/block_gaps.py @@ -0,0 +1,246 @@ +"""Does a block-split threshold exist? Plot the two distributions it separates. + +Protocol §12.6: never tune a threshold before plotting the distributions it is +supposed to separate. This repository has already proved once that a threshold +CANNOT exist for one such decision (table-cell gaps vs coincidental +same-baseline gaps: identical medians, 4.7em each), and a session was lost +tuning it before that was known. + +The decision here is `_build_blocks_one`'s: + + same_block <=> gap <= reference_pitch * BLOCK_GAP_FACTOR + +So for every consecutive pair of pdfium lines this measures `gap / reference` +and labels the pair with PyMuPDF's answer -- same block or not -- by pairing +lines across backends geometrically. Two distributions come out. If they +separate, a threshold exists and the plot says where; if they overlap, no +value of BLOCK_GAP_FACTOR can be right and the decision needs a different +signal. + +Several candidate references are scored side by side, because the previous +attempt at this failed by changing the reference (a local median) without +checking it on the whole corpus -- it fixed a code listing and cut +02_research_paper's paragraphs in half. + + python testkit/block_gaps.py # corpus + python testkit/block_gaps.py c6_long # one document +""" +import argparse +import glob +import os +import sys +from collections import defaultdict + +import _paths # noqa: F401 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _lines(ir): + out = [] + for pno, p in enumerate(ir.pages, 1): + for bi, b in enumerate(p.blocks): + for ln in b.lines: + out.append({"page": pno, "block": id(b), "base": ln.baseline, + "x0": ln.bbox[0], "x1": ln.bbox[2], + "size": max((s.size for s in ln.spans), default=10.0)}) + out.sort(key=lambda r: (r["page"], round(r["base"], 1), r["x0"])) + return out + + +def _pair(a, b): + """MuPDF line -> pdfium line, matched on page/baseline/x.""" + used, out = set(), {} + for i, ra in enumerate(a): + best, bd = None, None + for j, rb in enumerate(b): + if j in used or rb["page"] != ra["page"]: + continue + if abs(rb["base"] - ra["base"]) > 0.6 or abs(rb["x0"] - ra["x0"]) > 2.0: + continue + d = abs(rb["base"] - ra["base"]) + abs(rb["x0"] - ra["x0"]) / 10.0 + if bd is None or d < bd: + best, bd = j, d + if best is not None: + used.add(best) + out[j_key(ra)] = b[best] + return out + + +def j_key(r): + return (r["page"], round(r["base"], 1), round(r["x0"], 1)) + + +def _median(xs): + xs = sorted(xs) + return xs[len(xs) // 2] if xs else 0.0 + + +def references(page_lines): + """Candidate reference pitches for one page: {name: value}.""" + gaps = [b["base"] - a["base"] for a, b in zip(page_lines, page_lines[1:])] + valid = [g for g in gaps if 0 < g < 60] + if not valid: + return {} + med = _median(valid) + # the modal pitch, to 0.5pt -- body text dominates a page by line count even + # when a table's rows drag the median + hist = defaultdict(int) + for g in valid: + hist[round(g * 2) / 2.0] += 1 + modal = max(hist.items(), key=lambda kv: (kv[1], -kv[0]))[0] + # the smallest pitch that is not noise: the 20th percentile + p20 = sorted(valid)[max(0, int(0.2 * len(valid)) - 1)] + return {"median": med, "modal": modal, "p20": p20} + + +def collect(path): + from exactdoc.parse import parse_pdf as mu + from exactdoc.parse_pdfium import parse_pdf as pf + + A = _lines(mu(path, keep_image_data=False)) + B = _lines(pf(path, keep_image_data=False)) + m = _pair(A, B) + + rows = [] + by_page = defaultdict(list) + for r in B: + by_page[r["page"]].append(r) + for pno, pl in by_page.items(): + refs = references(pl) + if not refs: + continue + for a, b in zip(pl, pl[1:]): + gap = b["base"] - a["base"] + if not (0 < gap < 60): + continue + ka = [k for k, v in m.items() if v is a] + kb = [k for k, v in m.items() if v is b] + if not ka or not kb: + continue + mu_a = next(x for x in A if j_key(x) == ka[0]) + mu_b = next(x for x in A if j_key(x) == kb[0]) + rows.append({"same": mu_a["block"] == mu_b["block"], + "gap": gap, "refs": refs, + "pagekey": (os.path.basename(path), pno)}) + return rows + + +def page_cuts(path): + """{(document, page): adaptive cut} from the page's own gaps.""" + from exactdoc.parse_pdfium import parse_pdf as pf + out = {} + by_page = defaultdict(list) + for r in _lines(pf(path, keep_image_data=False)): + by_page[r["page"]].append(r) + for pno, pl in by_page.items(): + gaps = [b["base"] - a["base"] for a, b in zip(pl, pl[1:])] + gaps = [g for g in gaps if 0 < g < 60] + out[(os.path.basename(path), pno)] = otsu(gaps) + return out + + +def summarise(name, rows): + if not rows: + print(" %s: no comparable pairs" % name) + return + print("\n== %s (%d consecutive line pairs)" % (name, len(rows))) + print(" %-8s %-28s %-28s %s" + % ("ref", "gap/ref WITHIN a block", "gap/ref AT a boundary", "separable?")) + for ref in ("median", "modal", "p20"): + same = sorted(r["gap"] / r["refs"][ref] for r in rows + if r["same"] and r["refs"].get(ref)) + diff = sorted(r["gap"] / r["refs"][ref] for r in rows + if not r["same"] and r["refs"].get(ref)) + if not same or not diff: + continue + # best achievable split, and what it costs + best_err, best_t = None, None + for t in [x / 100.0 for x in range(50, 400)]: + err = sum(1 for v in same if v > t) + sum(1 for v in diff if v <= t) + if best_err is None or err < best_err: + best_err, best_t = err, t + verdict = ("YES at %.2f (%d/%d misclassified)" + % (best_t, best_err, len(same) + len(diff))) if best_err == 0 \ + else "overlap: best %.2f still misses %d of %d" % ( + best_t, best_err, len(same) + len(diff)) + print(" %-8s p50 %.2f max %.2f (n=%-4d) p50 %.2f min %.2f (n=%-4d) %s" + % (ref, _median(same), max(same), len(same), + _median(diff), min(diff), len(diff), verdict)) + + +def otsu(gaps): + """Unsupervised 1-D split of a page's gaps into 'inside' and 'between'. + + The per-document tables show every document separating cleanly at its OWN + ratio and no single ratio serving them all, so the question is whether the + split point can be found from the page itself rather than fixed in advance. + This is Otsu's method on the sorted gaps: pick the cut maximising + between-class variance. + """ + xs = sorted(gaps) + if len(xs) < 4: + return None + best, cut = None, None + for i in range(1, len(xs)): + lo, hi = xs[:i], xs[i:] + if not lo or not hi: + continue + mlo = sum(lo) / len(lo) + mhi = sum(hi) / len(hi) + v = len(lo) * len(hi) * (mlo - mhi) ** 2 + if best is None or v > best: + best, cut = v, (xs[i - 1] + xs[i]) / 2.0 + return cut + + +def evaluate_rules(rows, pages): + """Score the shipped rule and an adaptive one against PyMuPDF's answer.""" + print("\n-- rule scoring (labels from PyMuPDF's own block boundaries) --") + + # shipped: gap <= median_pitch * 1.6 + err = sum(1 for r in rows + if (r["gap"] <= r["refs"]["median"] * 1.6) != r["same"]) + print(" shipped gap <= median * 1.60 %d/%d wrong (%.0f%%)" + % (err, len(rows), 100.0 * err / max(1, len(rows)))) + + for f in (1.6, 1.3, 1.15, 1.05): + e = sum(1 for r in rows if (r["gap"] <= r["refs"]["p20"] * f) != r["same"]) + print(" fixed gap <= p20 * %.2f %d/%d wrong (%.0f%%)" + % (f, e, len(rows), 100.0 * e / max(1, len(rows)))) + + # adaptive: per-page Otsu cut on that page's own gaps + e = miss = 0 + for r in rows: + cut = pages.get(r["pagekey"]) + if cut is None: + miss += 1 + continue + if (r["gap"] <= cut) != r["same"]: + e += 1 + print(" adaptive per-page Otsu cut %d/%d wrong (%.0f%%)%s" + % (e, len(rows) - miss, 100.0 * e / max(1, len(rows) - miss), + "" if not miss else " [%d pairs on pages too small to cut]" % miss)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("names", nargs="*") + a = ap.parse_args() + srcs = sorted(glob.glob(os.path.join(ROOT, "corpus", "pdfs", "*.pdf"))) + srcs += sorted(glob.glob(os.path.join(ROOT, "testkit", "adv", "*.pdf"))) + if a.names: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.names)] + allrows, cuts = [], {} + for s in srcs: + rows = collect(s) + allrows += rows + cuts.update(page_cuts(s)) + summarise(os.path.splitext(os.path.basename(s))[0], rows) + summarise("CORPUS", allrows) + evaluate_rules(allrows, cuts) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/corpus_manifest.json b/testkit/corpus_manifest.json new file mode 100644 index 0000000..901831a --- /dev/null +++ b/testkit/corpus_manifest.json @@ -0,0 +1,118 @@ +{ + "_note": "The exact corpus the gate baseline was recorded against. Written and verified by testkit/corpus_manifest.py; regenerate the measured fields with `python testkit/corpus_manifest.py update` on the canonical environment. `src_pages` is an identity check, not a metric: the corpus is regenerated from scripts, so a generator change that alters a document silently re-bases every number measured from it. The PDFs are NOT byte-stable (ReportLab and Chromium both embed timestamps), so a content hash would fail on every run -- page count plus generator identity is what can honestly be pinned.", + "documents": { + "01_whitepaper_market.pdf": { + "dialect": "reportlab", + "generator": "corpus/make_corpus.py", + "path": "corpus/pdfs", + "src_pages": 3, + "why": "cover band, callouts, bar chart, ruled tables -- the Claude-style whitepaper this project started from" + }, + "02_research_paper.pdf": { + "dialect": "reportlab", + "generator": "corpus/make_corpus.py", + "path": "corpus/pdfs", + "src_pages": 2, + "why": "two-column academic layout with figures and a references list" + }, + "03_tech_report_code.pdf": { + "dialect": "reportlab", + "generator": "corpus/make_corpus.py", + "path": "corpus/pdfs", + "src_pages": 2, + "why": "monospace code blocks inside flowing prose" + }, + "04_exec_brief.pdf": { + "dialect": "reportlab", + "generator": "corpus/make_corpus.py", + "path": "corpus/pdfs", + "src_pages": 2, + "why": "dense one-page brief; its live-text coverage is a recorded shortfall" + }, + "05_memo.pdf": { + "dialect": "reportlab", + "generator": "corpus/make_corpus.py", + "path": "corpus/pdfs", + "src_pages": 1, + "why": "the minimal case -- if this one moves, something fundamental moved" + }, + "c1_whitepaper.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 2, + "why": "rounded-corner stat cards (D4) and a full-bleed cover band" + }, + "c2_paper2col.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "true multi-column sections, the case a naive reading order destroys" + }, + "c3_tables.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 3, + "why": "nested tables (D3) -- a recorded shortfall on both backends" + }, + "c4_i18n.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "CJK, Arabic and Hebrew: script continua and RTL reordering" + }, + "c5_graphics.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "gradient band with knocked-out white text; rasterised SVG chart" + }, + "c6_long.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 7, + "why": "long enough that a per-page offset compounds into a page count error" + }, + "c7_code.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "leading indentation, which PDFium does not report and PyMuPDF synthesises" + }, + "c8_toc_links.pdf": { + "dialect": "chromium", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "table of contents with tab leaders and internal hyperlinks" + }, + "f1_fpdf_brief.pdf": { + "dialect": "fpdf2", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "core-14 fonts with no FontDescriptor -- the serif-flag case" + }, + "l1_word_native.pdf": { + "dialect": "libreoffice", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "a PDF printed from a word processor: the round trip back to its own kind" + }, + "r1_reportlab_report.pdf": { + "dialect": "reportlab", + "generator": "testkit/gen_corpus.py", + "path": "testkit/adv", + "src_pages": 1, + "why": "a second ReportLab dialect written by a different generator than corpus/" + } + }, + "schema": 1 +} diff --git a/testkit/corpus_manifest.py b/testkit/corpus_manifest.py new file mode 100644 index 0000000..5d886d6 --- /dev/null +++ b/testkit/corpus_manifest.py @@ -0,0 +1,142 @@ +"""Verify -- or re-measure -- the corpus manifest. + + python testkit/corpus_manifest.py verify # is this the recorded corpus? + python testkit/corpus_manifest.py update # fill in the measured fields + +The manifest exists because the corpus is *generated*, and a generated corpus is +a moving target. Every published number is measured against 16 documents that no +longer exist as files anywhere -- they are rebuilt from `gen_corpus.py` and +`make_corpus.py` before each run, by whatever Chromium and ReportLab happen to +be installed. Nothing checked that the rebuild produced the same 16 documents. +Measured in a bare container: the generator produced 3 of 16, printed "the +corpus is incomplete", exited 0, and the gate went on to score those 3 against a +16-document baseline and report a pass. + +What can honestly be pinned, and what cannot: + + * the document set -- exactly, and that is the check that was missing; + * the generator that owns each document, and its dialect; + * the source page count, which is the cheapest identity fact that moves when a + generator change alters a document; + * NOT a content hash. Both generators embed a creation timestamp, so the bytes + differ on every run. A hash here would fail every time and be deleted within + a week, which is worse than no hash at all. +""" +import json +import os +import sys + +import _paths # noqa: F401 +from _paths import PROJECT + +PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "corpus_manifest.json") + + +def load(path=PATH): + with open(path) as f: + return json.load(f) + + +def _page_count(path): + """Page count via whichever parser is installed. Both agree on this.""" + try: + import pypdfium2 as pdfium + doc = pdfium.PdfDocument(path) + try: + return len(doc) + finally: + doc.close() + except ImportError: + pass + import fitz + doc = fitz.open(path) + try: + return doc.page_count + finally: + doc.close() + + +def verify(manifest=None, path=PATH): + """-> list of (kind, document, detail). Empty means the corpus matches.""" + manifest = manifest or load(path) + problems, seen = [], {} + for doc_id, spec in sorted(manifest.get("documents", {}).items()): + p = os.path.join(PROJECT, spec["path"], doc_id) + if doc_id in seen: + problems.append(("duplicate", doc_id, "two entries share a basename")) + continue + seen[doc_id] = p + if not os.path.exists(p): + problems.append(("missing", doc_id, "expected at %s (generator: %s)" + % (spec["path"], spec.get("generator", "?")))) + continue + want = spec.get("src_pages") + if want is None: + problems.append(("unmeasured", doc_id, + "src_pages is null -- run `corpus_manifest.py " + "update` on the canonical environment")) + continue + got = _page_count(p) + if got != want: + problems.append(("identity", doc_id, + "%d source pages, manifest says %d -- the generator " + "changed this document, so every number measured " + "from it was re-based" % (got, want))) + for d in sorted(set(s["path"] for s in manifest.get("documents", {}).values())): + import glob + for p in sorted(glob.glob(os.path.join(PROJECT, d, "*.pdf"))): + if os.path.basename(p) not in seen: + problems.append(("unexpected", os.path.basename(p), + "present in %s but not in the manifest" % d)) + return problems + + +def update(path=PATH): + manifest = load(path) + changed = [] + for doc_id, spec in sorted(manifest["documents"].items()): + p = os.path.join(PROJECT, spec["path"], doc_id) + if not os.path.exists(p): + print(" SKIP %-28s not present" % doc_id) + continue + got = _page_count(p) + if spec.get("src_pages") != got: + changed.append((doc_id, spec.get("src_pages"), got)) + spec["src_pages"] = got + print(" %-28s %d pages" % (doc_id, got)) + with open(path, "w") as f: + json.dump(manifest, f, indent=1, sort_keys=True) + f.write("\n") + for doc_id, was, now in changed: + print("CHANGED %-28s %s -> %s" % (doc_id, was, now)) + if changed: + print("\n%d document(s) changed identity. Re-record the gate baseline in " + "the same commit, or the numbers describe the previous corpus." + % len(changed)) + return 0 + + +def main(argv=None): + argv = argv if argv is not None else sys.argv[1:] + cmd = argv[0] if argv else "verify" + if cmd == "update": + return update() + if cmd != "verify": + print(__doc__) + return 2 + problems = verify() + m = load() + print("corpus manifest: %d documents" % len(m.get("documents", {}))) + for kind, doc, why in problems: + print(" %-11s %-28s %s" % (kind, doc[:28], why)) + if problems: + print("\n%d problem(s). Numbers from a corpus that is not the recorded " + "corpus are not comparable to the baseline." % len(problems)) + return 1 + print(" every document present, and each is the document on record") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/edge_cases.py b/testkit/edge_cases.py index 71301c4..a95c0d1 100644 --- a/testkit/edge_cases.py +++ b/testkit/edge_cases.py @@ -87,10 +87,18 @@ def _huge_text(d): cases["truncated"] = corrupt from exactdoc.convert import convert +from exactdoc.options import RAW + +# Explicitly the zero-refine profile. What is under test here is whether parse +# and write survive a hostile input, and running the closed loop over a truncated +# PDF would add an oracle dependency and minutes per case to a check that is +# supposed to be fast and offline. Naming the profile also means this file does +# not silently change meaning when the product default changes -- which it just +# did, from 0 rounds to 3. print("%-22s %s" % ("case", "result")) for name, path in cases.items(): try: - o = convert(path, os.path.join(OUT, name + ".docx")) + o = convert(path, os.path.join(OUT, name + ".docx"), options=RAW) sz = os.path.getsize(o) import zipfile from lxml import etree diff --git a/testkit/evidence.py b/testkit/evidence.py new file mode 100644 index 0000000..3a08bd1 --- /dev/null +++ b/testkit/evidence.py @@ -0,0 +1,206 @@ +"""One artifact that every published number traces to. + +The README, ROADMAP and STATUS each carried numbers, each with prose about +which environment and profile produced them, and they had already drifted apart: +one said `12 same / 2 better` where another said `13 same / 1 better`, and the +README's headline within-2pt was measured on a refine profile that no shipping +surface actually ran. Prose cannot be diffed and cannot be verified, so the fix +is not more careful prose -- it is a machine-readable record, addressed by +commit, that the docs quote and CI attaches to the run. + + python testkit/evidence.py # environment only, to stdout + python testkit/evidence.py --out evidence.json + +`runall.py` and `backend_parity.py` fold their lanes into the same file, so a +release can be checked against exactly one artifact: + + commit + dirty flag what code produced this + environment OS, Python, dependency versions, oracle versions + profile the ConversionOptions actually measured + corpus the manifest, and whether it verified + lanes per-document metrics and the gate verdict + parity the backend comparison verdict + package the installed-artifact smoke status +""" +import json +import os +import platform +import re +import subprocess +import sys + +import _paths # noqa: F401 +from _paths import CHROME, PROJECT, SOFFICE + +SCHEMA = 1 + + +def _run(cmd, timeout=60): + try: + p = subprocess.run(cmd, capture_output=True, timeout=timeout) + return (p.stdout or b"").decode("utf-8", "replace").strip() + except (OSError, subprocess.SubprocessError): + return "" + + +def git_state(): + """Commit, branch and whether the tree was dirty when this was measured. + + A dirty tree is recorded, not rejected: measuring uncommitted work is the + normal development loop. It is a release gate's job to refuse it, and it + cannot refuse what it was never told. + """ + def g(*args): + return _run(["git", "-C", PROJECT] + list(args)) + status = g("status", "--porcelain") + return {"commit": g("rev-parse", "HEAD"), + "short": g("rev-parse", "--short", "HEAD"), + "branch": g("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_paths": sorted(l[3:] for l in status.splitlines())[:40]} + + +def dependency_versions(): + """Installed versions of everything whose output can move a metric.""" + out = {} + for name in ("pymupdf", "pypdfium2", "python-docx", "numpy", "pillow", + "lxml", "reportlab", "fpdf2"): + try: + from importlib.metadata import PackageNotFoundError, version + out[name] = version(name) + except Exception: + out[name] = None + # The bundled native libraries matter more than the wrapper versions: the + # goldens are pinned to a PyMuPDF version because 1.26 and 1.28 group the + # same page differently (STATUS.md §5). + try: + import fitz + out["mupdf"] = getattr(fitz, "mupdf_version", None) or \ + getattr(fitz, "VersionBind", None) + except Exception: + out["mupdf"] = None + try: + import pypdfium2 + out["pdfium"] = str(getattr(pypdfium2, "PDFIUM_INFO", ""))or None + except Exception: + out["pdfium"] = None + return out + + +def oracle_versions(): + """The renderers. Their build decides the fidelity numbers, so name it.""" + lo = _run([SOFFICE, "--version"]) if SOFFICE else "" + ch = _run([CHROME, "--version"]) if CHROME else "" + fonts = _run(["fc-list"]) + liberation = sorted(set( + re.findall(r"(Liberation \w+)", fonts)))[:6] if fonts else [] + return {"soffice_path": SOFFICE, "soffice_version": lo.splitlines()[0] if lo else None, + "chrome_path": CHROME, "chrome_version": ch.splitlines()[0] if ch else None, + "metric_fonts": liberation} + + +def environment(): + return { + "os": platform.system().lower(), + "os_release": platform.release(), + "machine": platform.machine(), + "python": sys.version.split()[0], + "dependencies": dependency_versions(), + "oracles": oracle_versions(), + "canonical": platform.system().lower() == "linux", + } + + +def new(profile=None): + """A fresh evidence document. Lanes and verdicts are added as they run.""" + return {"schema": SCHEMA, "git": git_state(), "environment": environment(), + "profile": profile, "corpus": None, "lanes": {}, "parity": None, + "package": None} + + +def merge(path, **sections): + """Fold sections into the evidence file at `path`, creating it if absent. + + Separate processes produce the lanes, the parity verdict and the package + smoke result, and a release needs them in one artifact. Merging rather than + rewriting is what lets CI run them as independent steps without one step's + success erasing another's. + + A `None` section is *skipped*, not written. Without that, the plain + `evidence.py --out` step -- which exists to fill in the environment when + nothing else has -- passed the empty template's `parity: None` and + `corpus: None` straight over the verdicts the two preceding steps had already + recorded. Measured: a full green run ended with an evidence file that had + forgotten its own parity result. An artifact whose job is to be the single + source of a release claim must not have a write path that quietly empties it. + """ + doc = {} + if os.path.exists(path): + try: + with open(path) as f: + doc = json.load(f) + except ValueError: + doc = {} + if not doc: + doc = new() + for k, v in sections.items(): + if v is None: + continue + if k == "lanes" and isinstance(v, dict): + doc.setdefault("lanes", {}).update(v) + else: + doc[k] = v + d = os.path.dirname(os.path.abspath(path)) + if d: + os.makedirs(d, exist_ok=True) + with open(path, "w") as f: + json.dump(doc, f, indent=1, sort_keys=True) + return path + + +def summarise(doc): + """The lines a human should read before believing a release claim.""" + g, e = doc.get("git", {}), doc.get("environment", {}) + out = ["commit %s%s on %s" % (g.get("short") or "(no git)", + " (DIRTY)" if g.get("dirty") else "", + g.get("branch") or "?"), + "env %s %s, python %s%s" % ( + e.get("os"), e.get("machine"), e.get("python"), + "" if e.get("canonical") else " [NOT the canonical environment]"), + "oracle %s" % ((e.get("oracles") or {}).get("soffice_version") or "none")] + if doc.get("profile"): + out.append("profile %s" % doc["profile"].get("profile_id", "?")) + for lane, data in sorted((doc.get("lanes") or {}).items()): + agg = data.get("aggregate") or {} + verdict = data.get("verdict") or {} + out.append("lane %-8s %s pagematch %s/%s <2pt %s live %s dy50 %s" + % (lane, "PASS" if verdict.get("ok") else "FAIL", + agg.get("page_match_count"), agg.get("n"), + agg.get("mean_within2pt"), agg.get("mean_live_text"), + agg.get("median_dy_p50"))) + p = doc.get("parity") or {} + if p: + out.append("parity %s %s regression(s), %s same, %s better" + % ("PASS" if p.get("ok") else "FAIL", p.get("regressions"), + p.get("same"), p.get("better"))) + pk = doc.get("package") or {} + if pk: + out.append("package %s %s" % ("PASS" if pk.get("ok") else "FAIL", + pk.get("detail", ""))) + return "\n".join(out) + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--out", default=None, help="write/merge JSON here") + a = ap.parse_args() + doc = new() + if a.out: + merge(a.out, **{k: v for k, v in doc.items() if k != "schema"}) + with open(a.out) as f: + doc = json.load(f) # summarise the artifact, not the template + print("wrote", a.out) + print(summarise(doc)) + if not a.out: + print(json.dumps(doc, indent=1, sort_keys=True)) diff --git a/testkit/exp_chromefix.py b/testkit/exp_chromefix.py index 96a2132..2b83a7b 100644 --- a/testkit/exp_chromefix.py +++ b/testkit/exp_chromefix.py @@ -82,8 +82,16 @@ def patched_parse(path, keep_image_data=True): P.parse_pdf = patched_parse -import exactdoc.convert as C -C.parse_pdf = patched_parse +# Registered on the backend seam, not assigned over `exactdoc.convert.parse_pdf`: +# that assignment stopped having any effect once the backend was selected through +# the seam, and an experiment that quietly measures the unpatched parser still +# prints a number. +from exactdoc.backend import register_backend # noqa: E402 +from exactdoc.convert import convert # noqa: E402 +from exactdoc.options import PRODUCT # noqa: E402 + +_OPTIONS = PRODUCT.replace( + backend=register_backend("chromefix", patched_parse), refine_rounds=0) if __name__ == "__main__": @@ -100,7 +108,7 @@ def patched_parse(path, keep_image_data=True): for s in srcs: n = os.path.splitext(os.path.basename(s))[0] dx = os.path.join(out, n + ".docx") - C.convert(s, dx) + convert(s, dx, options=_OPTIONS) pairs.append((s, dx)) harness.batch_docx_to_pdf([d for _, d in pairs], os.path.join(out, "r")) rows = [] diff --git a/testkit/exp_regroup.py b/testkit/exp_regroup.py index b1b5798..d06cff2 100644 --- a/testkit/exp_regroup.py +++ b/testkit/exp_regroup.py @@ -95,10 +95,25 @@ def hybrid_parse(path, keep_image_data=True): def run(lane, srcs, out_root, refine): - import exactdoc.convert as C - from exactdoc.parse import parse_pdf as mu - from exactdoc.parse_pdfium import parse_pdf as px - C.parse_pdf = {"pymupdf": mu, "pdfium": px, "hybrid": hybrid_parse}[lane] + # The hybrid lane is registered on the backend seam rather than assigned over + # `exactdoc.convert.parse_pdf`. That assignment only ever worked because + # `convert` held the parser as a module global; once the backend is selected + # through the seam it is a no-op that sets an attribute nobody reads, and the + # lane would silently measure the default parser while reporting itself as + # the hybrid. + from exactdoc.backend import register_backend + from exactdoc.convert import convert + from exactdoc.options import PRODUCT + + if lane == "hybrid": + try: + register_backend("hybrid-regroup", hybrid_parse) + except ValueError: + pass + backend = "hybrid-regroup" + else: + backend = lane + options = PRODUCT.replace(backend=backend, refine_rounds=refine) out = os.path.join(out_root, lane) os.makedirs(out, exist_ok=True) pairs = [] @@ -106,7 +121,7 @@ def run(lane, srcs, out_root, refine): n = os.path.splitext(os.path.basename(s))[0] dx = os.path.join(out, n + ".docx") try: - C.convert(s, dx, refine_rounds=refine) + convert(s, dx, options=options) pairs.append((s, dx, n)) except Exception as e: # noqa: BLE001 print(" CONVERT FAIL [%s] %-20s %s" % (lane, n[:20], str(e)[:44])) diff --git a/testkit/exp_sweep.py b/testkit/exp_sweep.py index 4218ffd..48711da 100644 --- a/testkit/exp_sweep.py +++ b/testkit/exp_sweep.py @@ -60,6 +60,10 @@ def line_match(src_pdf, out_pdf): srcs += sorted(glob.glob(os.path.join(d, "*.pdf"))) root = os.path.dirname(os.path.abspath(__file__)) from exactdoc.convert import convert + # Zero refine, explicitly: this sweep measures what the wrap-width correction + # does to line-break agreement, and the closed loop would correct over the top + # of the very effect being swept. + from exactdoc.options import RAW print("%-8s %-6s | %-28s %s" % ("alpha", "quant", "doc", "line_match pages <2pt")) for quant, alpha in [(False, 0.0), (True, 0.0), (True, -0.004), (True, 0.004), @@ -73,7 +77,7 @@ def line_match(src_pdf, out_pdf): for s in srcs: n = os.path.splitext(os.path.basename(s))[0] dx = os.path.join(out, n + ".docx") - convert(s, dx) + convert(s, dx, options=RAW) pairs.append((s, dx)) harness.batch_docx_to_pdf([d for _, d in pairs], os.path.join(out, "r")) for s, dx in pairs: diff --git a/testkit/gate.py b/testkit/gate.py new file mode 100644 index 0000000..c49cd1c --- /dev/null +++ b/testkit/gate.py @@ -0,0 +1,402 @@ +"""The gate's decision, separated from the run that produces the numbers. + +Everything here is a pure function over already-measured results. That is the +point: a gate whose only expression is inside a 200-line runner that converts 16 +documents and shells out to LibreOffice cannot be tested, and an untested gate +is a claim. `tests/test_gate_mutations.py` feeds this module synthetic result +sets -- a deleted document, a renderer error, a missing metric, a known failure +sliding further -- and asserts each one comes back red. None of those tests need +an oracle, a corpus, or a minute. + +What the previous gate could not see, all of it measured or read off the code: + + * `harness.evaluate()` returns `{"error": ...}` when the render fails, and + nothing looked for that key. A renderer that died on every document scored + zero failures. + * A metric that was absent was skipped (`if v is None: continue`), so losing + `within2pt` removed the check instead of failing it. + * The baseline stored only the NAMES of failing metrics. `04_exec_brief`'s + live-text coverage was recorded as "known failing" at 0.941; it could have + fallen to 0.10 and stayed exactly as green. + * `page_match` is a boolean, so a document already failing it could go from + one page over to forty and register no change. + * Nothing checked that the 16 expected documents were the 16 documents + measured. The corpus generator exits 0 after skipping 8 of them. + * `REFINE=lanes` returned only the refined lane's status, so a raw-lane + regression could not fail the build. + +The rule set below is deliberately three separate questions, because they have +different answers: + + regression is anything worse than the number on record, beyond tolerance? + Applies to every document and every metric, passing or not. + This is the pull-request gate. + absolute does every document clear the release threshold? Documents + recorded BELOW a threshold are known shortfalls and must carry + a defect ID. This is the release-qualification gate. + stale does a recorded shortfall now pass? Then the record is wrong, + and a wrong record silently re-admits the regression it exists + to catch. +""" +import json +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +BASELINE_PATH = os.path.join(HERE, "gate_baseline.json") +MANIFEST_PATH = os.path.join(HERE, "corpus_manifest.json") + +# Metric direction and how much cross-environment noise is tolerated. +# +# `threshold` is the absolute release bar. `None` means "not ratified yet": the +# metric is still gated against its recorded number, but no absolute claim is +# made about it. Writing a number here is a product decision, so an unratified +# one is left visibly empty rather than guessed at -- an invented threshold that +# the corpus happens to pass is indistinguishable from no threshold at all. +# +# `tol` is the regression slack, sized from measurement: three environments +# (CI Linux, a local ubuntu:24.04 container, Windows) agree on every structural +# number and differ in the third decimal of within2pt (STATUS.md §1). The +# tolerances are an order of magnitude above that noise and an order of +# magnitude below any regression this project has actually shipped. +# +# `rel` adds a proportional term, and `dy_p50` needs one. It is the only gated +# metric that is not a fraction in [0, 1]: it runs from 0.04pt on +# `02_research_paper` to 101pt on `c1_whitepaper` in the raw lane. A flat 0.5pt +# slack is generous at the bottom of that range and absurdly tight at the top, +# where two LibreOffice builds can disagree by more than that on a drift already +# two orders of magnitude past the threshold anyone cares about. The tolerance is +# max(tol, rel x recorded), so the absolute floor governs the small numbers and +# the proportional term governs the large ones. +HIGHER, LOWER, BOOL = "higher", "lower", "bool" +METRICS = { + "page_err": {"dir": LOWER, "threshold": 0, "tol": 0}, + "live_text_cov": {"dir": HIGHER, "threshold": 0.95, "tol": 0.010}, + "doc_recall": {"dir": HIGHER, "threshold": 0.95, "tol": 0.010}, + "word_recall": {"dir": HIGHER, "threshold": 0.90, "tol": 0.020}, + "within2pt": {"dir": HIGHER, "threshold": None, "tol": 0.050}, + "dy_p50": {"dir": LOWER, "threshold": None, "tol": 0.500, + "rel": 0.10}, + "raster_frac": {"dir": LOWER, "threshold": None, "tol": 0.020}, +} + + +def tolerance(spec, reference): + """The slack allowed against `reference`, absolute and proportional.""" + tol = spec.get("tol", 0.0) + rel = spec.get("rel") + if rel and isinstance(reference, (int, float)): + return max(tol, rel * abs(reference)) + return tol + +# Aggregates are the headline numbers -- the ones that reach the README -- so +# they are gated as well. A set of per-document changes that each stay inside +# tolerance can still move the mean, and the mean is what gets published. +AGGREGATES = { + "page_match_count": {"dir": HIGHER, "threshold": None, "tol": 0}, + "mean_within2pt": {"dir": HIGHER, "threshold": None, "tol": 0.020}, + "mean_live_text": {"dir": HIGHER, "threshold": None, "tol": 0.005}, + "median_dy_p50": {"dir": LOWER, "threshold": None, "tol": 0.300}, +} + +# Keys harness.evaluate() must have produced for a result to be scoreable at +# all. Absence is a failure, never a skip. +REQUIRED_KEYS = ("src_pages", "out_pages", "live_text_cov", "doc_recall", + "word_recall", "within2pt", "dy_p50", "raster_frac", + "mean_ssim", "renderer") + +FATAL_KEYS = ("convert_error", "eval_error", "error") + + +class Verdict(object): + """What the gate decided, and why. `ok` is the exit code's only source.""" + + def __init__(self, lane): + self.lane = lane + self.failures = [] # (kind, document, detail) + self.notes = [] + + def fail(self, kind, doc, detail): + self.failures.append((kind, doc, detail)) + + def note(self, text): + self.notes.append(text) + + @property + def ok(self): + return not self.failures + + def kinds(self): + return sorted(set(k for k, _, _ in self.failures)) + + def report(self): + out = [] + for n in self.notes: + out.append(" %s" % n) + for kind, doc, detail in self.failures: + out.append(" %-12s %-30s %s" % (kind, (doc or "-")[:30], detail)) + if not self.failures: + out.append(" gate PASS (%s)" % self.lane) + else: + out.append(" gate FAIL (%s): %d finding(s) -- %s" + % (self.lane, len(self.failures), ", ".join(self.kinds()))) + return "\n".join(out) + + def as_dict(self): + return {"lane": self.lane, "ok": self.ok, + "failures": [{"kind": k, "document": d, "detail": v} + for k, d, v in self.failures], + "notes": list(self.notes)} + + +# --------------------------------------------------------------- derived facts +def metric_values(result): + """The gated metrics of one harness result, including derived ones. + + `page_err` is derived rather than read: `page_match` is a boolean, and a + boolean cannot record that a document went from one page over to forty. The + magnitude is the thing that has to be gated. + """ + vals = {} + for k in METRICS: + if k == "page_err": + if "src_pages" in result and "out_pages" in result: + vals[k] = abs(int(result["out_pages"]) - int(result["src_pages"])) + continue + if k in result and isinstance(result[k], (int, float)): + vals[k] = float(result[k]) + return vals + + +def worse(direction, value, reference, tol=0.0): + """Is `value` worse than `reference` by more than `tol`?""" + if direction == LOWER: + return value > reference + tol + if direction == BOOL: + return bool(reference) and not bool(value) + return value < reference - tol + + +def clears(direction, value, threshold): + if threshold is None: + return True + if direction == LOWER: + return value <= threshold + if direction == BOOL: + return bool(value) is bool(threshold) + return value >= threshold + + +def aggregates(results): + """Lane-level numbers, computed only from results that scored.""" + import statistics as st + ok = [r for r in results if not any(k in r for k in FATAL_KEYS)] + if not ok: + return {} + def mean(key): + vals = [r[key] for r in ok if isinstance(r.get(key), (int, float))] + return round(st.mean(vals), 4) if vals else None + dys = [r["dy_p50"] for r in ok if isinstance(r.get("dy_p50"), (int, float))] + return { + "n": len(ok), + "page_match_count": sum(1 for r in ok if r.get("page_match") is True), + # "13/16 passed" is the number the README quotes, so it comes from here + # rather than from a reader counting rows. It is not in AGGREGATES: it is + # a function of the per-document thresholds, every one of which is + # already gated, so gating it again would only double-report. + "gate_pass_count": sum(1 for r in ok if all( + clears(spec["dir"], v, spec["threshold"]) + for name, spec in METRICS.items() + for v in [metric_values(r).get(name)] if v is not None)), + "mean_within2pt": mean("within2pt"), + "mean_live_text": mean("live_text_cov"), + "median_dy_p50": round(st.median(dys), 3) if dys else None, + } + + +# ------------------------------------------------------------------- the gate +def check(lane, results, manifest=None, baseline=None, absolute=False): + """Score one lane. Returns a Verdict. + + `absolute` adds the release-qualification questions to the pull-request + ones: without it a known shortfall may stay below its threshold, with it + every document must clear every ratified threshold. CI runs both -- the + regression form as a required check, the absolute form as the release gate + -- because they answer different questions and conflating them is how "the + gate is green" came to mean "nothing got measurably worse today". + """ + v = Verdict(lane) + baseline = baseline or {} + docs_baseline = baseline.get("documents", {}) + defects = baseline.get("shortfall_defects", {}) + by_id = {} + + # 1. identity. Two inputs with the same basename overwrite each other's + # DOCX and each other's result row, so the second silently replaces the + # first and the count still looks right. + for r in results: + rid = r.get("src") + if not rid: + v.fail("malformed", None, "a result has no 'src' key") + continue + if rid in by_id: + v.fail("duplicate", rid, "measured twice -- output and result " + "identity collide on the basename") + continue + by_id[rid] = r + + if manifest: + expected = set(manifest.get("documents", {})) + for missing in sorted(expected - set(by_id)): + v.fail("missing", missing, "in the corpus manifest, not in the run") + for extra in sorted(set(by_id) - expected): + v.fail("unexpected", extra, "measured but not in the corpus manifest") + for doc_id, spec in sorted(manifest.get("documents", {}).items()): + r = by_id.get(doc_id) + want = spec.get("src_pages") + if r is not None and want is not None and "src_pages" in r \ + and int(r["src_pages"]) != int(want): + v.fail("identity", doc_id, + "source is %s pages, manifest says %s -- this is not the " + "document the baseline was recorded against" + % (r["src_pages"], want)) + + # 2. integrity. A result that carries an error key is a failure, not a row + # to be skipped: the renderer dying on every document used to score zero. + for doc_id, r in sorted(by_id.items()): + fatal = [k for k in FATAL_KEYS if k in r] + if fatal: + v.fail("error", doc_id, "%s: %s" % (fatal[0], str(r[fatal[0]])[:120])) + continue + for k in REQUIRED_KEYS: + if k not in r: + v.fail("no-metric", doc_id, + "required metric %r absent -- a metric that cannot be " + "computed is a failure, not a skip" % k) + + # 3. per-document thresholds and floors. + for doc_id, r in sorted(by_id.items()): + if any(k in r for k in FATAL_KEYS): + continue + recorded = docs_baseline.get(doc_id) + vals = metric_values(r) + if recorded is None: + v.fail("unrecorded", doc_id, + "no numeric baseline for this document in lane %r -- record " + "one with GATE_BASELINE=update on the canonical environment" + % lane) + for name, spec in sorted(METRICS.items()): + if name not in vals: + continue + value = vals[name] + ref = (recorded or {}).get(name) + known_shortfall = (ref is not None + and not clears(spec["dir"], ref, spec["threshold"])) + + tol = tolerance(spec, ref) + if ref is not None and worse(spec["dir"], value, ref, tol): + v.fail("regression", doc_id, + "%s %.4g -> %.4g (recorded %.4g, tolerance %.3g)" + % (name, ref, value, ref, tol)) + elif ref is None and recorded is not None: + # The document is on record but this metric is not: the record + # predates the metric, and an ungated metric is how within2pt + # once hid a 0.510 -> 0.291 regression in plain sight. + v.fail("unrecorded", doc_id, "%s has no recorded value" % name) + + if not clears(spec["dir"], value, spec["threshold"]): + if not known_shortfall: + v.fail("threshold", doc_id, + "%s %.4g misses %s and is not a recorded shortfall" + % (name, value, spec["threshold"])) + elif doc_id not in defects: + v.fail("undocumented", doc_id, + "recorded below the %s threshold with no defect ID; " + "add one to shortfall_defects and to STATUS.md" % name) + elif known_shortfall: + v.fail("stale", doc_id, + "%s %.4g now clears %s but is recorded as %.4g -- a stale " + "record re-admits the regression it exists to catch" + % (name, value, spec["threshold"], ref)) + + if absolute and not clears(spec["dir"], value, spec["threshold"]): + v.fail("unqualified", doc_id, + "%s %.4g misses the release threshold %s" + % (name, value, spec["threshold"])) + + # 4. aggregates -- the published numbers. + agg, ref_agg = aggregates(results), baseline.get("aggregate", {}) + for name, spec in sorted(AGGREGATES.items()): + value, ref = agg.get(name), ref_agg.get(name) + if value is None: + v.fail("no-metric", None, "aggregate %r could not be computed" % name) + continue + if ref is None: + v.fail("unrecorded", None, "aggregate %r has no recorded value" % name) + continue + tol = tolerance(spec, ref) + if worse(spec["dir"], value, ref, tol): + v.fail("regression", None, + "aggregate %s %.4g -> %.4g (tolerance %.3g)" + % (name, ref, value, tol)) + if absolute and not clears(spec["dir"], value, spec["threshold"]): + v.fail("unqualified", None, + "aggregate %s %.4g misses the release threshold %s" + % (name, value, spec["threshold"])) + + v.note("%d document(s) measured, %d expected" + % (len(by_id), len(manifest.get("documents", {})) if manifest else len(by_id))) + return v + + +# ------------------------------------------------------------------- baseline +def record(lane, results): + """The numeric record for one lane: every gated metric, every document.""" + docs = {} + for r in results: + if any(k in r for k in FATAL_KEYS) or not r.get("src"): + continue + docs[r["src"]] = {k: round(v, 4) for k, v in metric_values(r).items()} + return {"documents": docs, "aggregate": aggregates(results)} + + +def load(path=BASELINE_PATH): + if not os.path.exists(path): + return {} + with open(path) as f: + return json.load(f) + + +def load_lane(lane, path=BASELINE_PATH): + return load(path).get("lanes", {}).get(lane, {}) + + +def load_manifest(path=MANIFEST_PATH): + if not os.path.exists(path): + return None + with open(path) as f: + return json.load(f) + + +def save_lane(lane, data, path=BASELINE_PATH, environment=None): + """Write one lane's record, preserving the others and the defect IDs.""" + doc = load(path) or {} + doc["schema"] = 2 + doc["_note"] = ( + "Numeric per-document baseline for every gated metric, per lane, " + "measured on the canonical environment (see .github/workflows/gate.yml). " + "The gate asks three questions of it: nothing worse than these numbers " + "beyond tolerance (regression), everything clears its threshold unless " + "recorded below it (absolute), and nothing recorded below a threshold " + "now passes (stale). Regenerate deliberately with GATE_BASELINE=update, " + "never to silence a failure, and say so in the commit message.") + lanes = doc.setdefault("lanes", {}) + prev = lanes.get(lane, {}) + entry = {"documents": data["documents"], "aggregate": data["aggregate"]} + # Defect IDs are human knowledge and survive a re-record; the numbers do not. + entry["shortfall_defects"] = prev.get("shortfall_defects", {}) + if environment: + entry["environment"] = environment + lanes[lane] = entry + with open(path, "w") as f: + json.dump(doc, f, indent=1, sort_keys=True) + return path diff --git a/testkit/gate_baseline.json b/testkit/gate_baseline.json new file mode 100644 index 0000000..1de6587 --- /dev/null +++ b/testkit/gate_baseline.json @@ -0,0 +1,389 @@ +{ + "_note": "Numeric per-document baseline for every gated metric, per lane, measured on the canonical environment (see .github/workflows/gate.yml). The gate asks three questions of it: nothing worse than these numbers beyond tolerance (regression), everything clears its threshold unless recorded below it (absolute), and nothing recorded below a threshold now passes (stale). Regenerate deliberately with GATE_BASELINE=update, never to silence a failure, and say so in the commit message.", + "lanes": { + "product": { + "aggregate": { + "gate_pass_count": 13, + "mean_live_text": 0.9652, + "mean_within2pt": 0.5118, + "median_dy_p50": 0.62, + "n": 16, + "page_match_count": 15 + }, + "documents": { + "01_whitepaper_market.pdf": { + "doc_recall": 0.9677, + "dy_p50": 0.5, + "live_text_cov": 0.9595, + "page_err": 0, + "raster_frac": 0.0405, + "within2pt": 0.7194, + "word_recall": 0.9677 + }, + "02_research_paper.pdf": { + "doc_recall": 0.9586, + "dy_p50": 0.04, + "live_text_cov": 0.9736, + "page_err": 0, + "raster_frac": 0.0264, + "within2pt": 0.7614, + "word_recall": 0.9586 + }, + "03_tech_report_code.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.54, + "live_text_cov": 0.9827, + "page_err": 0, + "raster_frac": 0.0173, + "within2pt": 0.4602, + "word_recall": 1.0 + }, + "04_exec_brief.pdf": { + "doc_recall": 0.9337, + "dy_p50": 3.51, + "live_text_cov": 0.9406, + "page_err": 0, + "raster_frac": 0.0594, + "within2pt": 0.2249, + "word_recall": 0.9337 + }, + "05_memo.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.59, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.6386, + "word_recall": 1.0 + }, + "c1_whitepaper.pdf": { + "doc_recall": 0.9697, + "dy_p50": 29.36, + "live_text_cov": 0.9654, + "page_err": 0, + "raster_frac": 0.0346, + "within2pt": 0.1812, + "word_recall": 0.9697 + }, + "c2_paper2col.pdf": { + "doc_recall": 1.0, + "dy_p50": 29.2, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.1948, + "word_recall": 1.0 + }, + "c3_tables.pdf": { + "doc_recall": 0.9359, + "dy_p50": 4.8, + "live_text_cov": 0.9226, + "page_err": 1, + "raster_frac": 0.0774, + "within2pt": 0.0, + "word_recall": 0.331 + }, + "c4_i18n.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.15, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.416, + "word_recall": 1.0 + }, + "c5_graphics.pdf": { + "doc_recall": 0.6778, + "dy_p50": 0.5, + "live_text_cov": 0.7067, + "page_err": 0, + "raster_frac": 0.2933, + "within2pt": 0.6885, + "word_recall": 0.6778 + }, + "c6_long.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.65, + "live_text_cov": 0.9923, + "page_err": 0, + "raster_frac": 0.0077, + "within2pt": 0.758, + "word_recall": 1.0 + }, + "c7_code.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.7, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.9148, + "word_recall": 1.0 + }, + "c8_toc_links.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.1, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 1.0, + "word_recall": 1.0 + }, + "f1_fpdf_brief.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.0, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.621, + "word_recall": 1.0 + }, + "l1_word_native.pdf": { + "doc_recall": 0.9931, + "dy_p50": 14.69, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.014, + "word_recall": 0.9931 + }, + "r1_reportlab_report.pdf": { + "doc_recall": 1.0, + "dy_p50": 1.5, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.5959, + "word_recall": 1.0 + } + }, + "environment": { + "canonical": true, + "dependencies": { + "fpdf2": "2.8.7", + "lxml": "6.1.1", + "mupdf": "1.29.0", + "numpy": "2.5.1", + "pdfium": "152.0.7947.0", + "pillow": "12.3.0", + "pymupdf": "1.28.0", + "pypdfium2": "5.12.1", + "python-docx": "1.2.0", + "reportlab": "5.0.0" + }, + "machine": "x86_64", + "oracles": { + "chrome_path": "/root/.cache/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-linux64/chrome-headless-shell", + "chrome_version": "Google Chrome for Testing 149.0.7827.55", + "metric_fonts": [ + "Liberation Mono", + "Liberation Sans", + "Liberation Serif" + ], + "soffice_path": "/usr/bin/soffice", + "soffice_version": "LibreOffice 24.2.7.2 420(Build:2)" + }, + "os": "linux", + "os_release": "5.15.167.4-microsoft-standard-WSL2", + "python": "3.12.3" + }, + "shortfall_defects": { + "04_exec_brief.pdf": "D10", + "c3_tables.pdf": "D3", + "c5_graphics.pdf": "D10" + } + }, + "raw": { + "aggregate": { + "gate_pass_count": 12, + "mean_live_text": 0.9652, + "mean_within2pt": 0.3486, + "median_dy_p50": 2.2, + "n": 16, + "page_match_count": 13 + }, + "documents": { + "01_whitepaper_market.pdf": { + "doc_recall": 0.9677, + "dy_p50": 3.68, + "live_text_cov": 0.9595, + "page_err": 0, + "raster_frac": 0.0405, + "within2pt": 0.2208, + "word_recall": 0.9677 + }, + "02_research_paper.pdf": { + "doc_recall": 0.9586, + "dy_p50": 1.09, + "live_text_cov": 0.9736, + "page_err": 0, + "raster_frac": 0.0264, + "within2pt": 0.6701, + "word_recall": 0.9586 + }, + "03_tech_report_code.pdf": { + "doc_recall": 1.0, + "dy_p50": 4.08, + "live_text_cov": 0.9827, + "page_err": 0, + "raster_frac": 0.0173, + "within2pt": 0.0588, + "word_recall": 1.0 + }, + "04_exec_brief.pdf": { + "doc_recall": 0.9337, + "dy_p50": 9.35, + "live_text_cov": 0.9406, + "page_err": 0, + "raster_frac": 0.0594, + "within2pt": 0.0533, + "word_recall": 0.9337 + }, + "05_memo.pdf": { + "doc_recall": 1.0, + "dy_p50": 4.29, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.1566, + "word_recall": 1.0 + }, + "c1_whitepaper.pdf": { + "doc_recall": 0.9697, + "dy_p50": 100.96, + "live_text_cov": 0.9654, + "page_err": 1, + "raster_frac": 0.0346, + "within2pt": 0.0, + "word_recall": 0.7667 + }, + "c2_paper2col.pdf": { + "doc_recall": 1.0, + "dy_p50": 26.8, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.039, + "word_recall": 1.0 + }, + "c3_tables.pdf": { + "doc_recall": 0.9359, + "dy_p50": 2.5, + "live_text_cov": 0.9226, + "page_err": 1, + "raster_frac": 0.0774, + "within2pt": 0.0, + "word_recall": 0.3137 + }, + "c4_i18n.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.15, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.416, + "word_recall": 1.0 + }, + "c5_graphics.pdf": { + "doc_recall": 0.6778, + "dy_p50": 0.9, + "live_text_cov": 0.7067, + "page_err": 1, + "raster_frac": 0.2933, + "within2pt": 0.8, + "word_recall": 0.1667 + }, + "c6_long.pdf": { + "doc_recall": 1.0, + "dy_p50": 1.3, + "live_text_cov": 0.9923, + "page_err": 0, + "raster_frac": 0.0077, + "within2pt": 0.6738, + "word_recall": 1.0 + }, + "c7_code.pdf": { + "doc_recall": 1.0, + "dy_p50": 1.9, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.5568, + "word_recall": 1.0 + }, + "c8_toc_links.pdf": { + "doc_recall": 1.0, + "dy_p50": 0.1, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 1.0, + "word_recall": 1.0 + }, + "f1_fpdf_brief.pdf": { + "doc_recall": 1.0, + "dy_p50": 1.2, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.6048, + "word_recall": 1.0 + }, + "l1_word_native.pdf": { + "doc_recall": 0.9931, + "dy_p50": 26.69, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.007, + "word_recall": 0.9931 + }, + "r1_reportlab_report.pdf": { + "doc_recall": 1.0, + "dy_p50": 1.1, + "live_text_cov": 1.0, + "page_err": 0, + "raster_frac": 0.0, + "within2pt": 0.3212, + "word_recall": 1.0 + } + }, + "environment": { + "canonical": true, + "dependencies": { + "fpdf2": "2.8.7", + "lxml": "6.1.1", + "mupdf": "1.29.0", + "numpy": "2.5.1", + "pdfium": "152.0.7947.0", + "pillow": "12.3.0", + "pymupdf": "1.28.0", + "pypdfium2": "5.12.1", + "python-docx": "1.2.0", + "reportlab": "5.0.0" + }, + "machine": "x86_64", + "oracles": { + "chrome_path": "/root/.cache/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-linux64/chrome-headless-shell", + "chrome_version": "Google Chrome for Testing 149.0.7827.55", + "metric_fonts": [ + "Liberation Mono", + "Liberation Sans", + "Liberation Serif" + ], + "soffice_path": "/usr/bin/soffice", + "soffice_version": "LibreOffice 24.2.7.2 420(Build:2)" + }, + "os": "linux", + "os_release": "5.15.167.4-microsoft-standard-WSL2", + "python": "3.12.3" + }, + "shortfall_defects": { + "04_exec_brief.pdf": "D10", + "c1_whitepaper.pdf": "D4", + "c3_tables.pdf": "D3", + "c5_graphics.pdf": "D10" + } + } + }, + "schema": 2 +} \ No newline at end of file diff --git a/testkit/gen_corpus.py b/testkit/gen_corpus.py index 084742e..8ce29d8 100644 --- a/testkit/gen_corpus.py +++ b/testkit/gen_corpus.py @@ -9,7 +9,8 @@ OUT = sys.argv[1] if len(sys.argv) > 1 else "adv" HTML = os.path.join(OUT, "_html") -os.makedirs(HTML, exist_ok=True) +# Directories are created in main(), not at import: importing this module to +# test its degradation path must not litter the working directory. LOREM = ("Retrieval quality degrades non-linearly as the corpus grows past the " "point where the embedding model was calibrated, and the failure is " @@ -68,13 +69,24 @@ def html_doc(name, body, extra_css=""): def chrome_pdf(html_path, out_pdf): + """Render one HTML file with headless Chromium. Returns (ok, reason).""" + if not CHROME: + return False, "no Chromium found (set CHROME=/path/to/chrome)" url = "file:///" + os.path.abspath(html_path).replace("\\", "/") cmd = [CHROME, "--headless=new", "--disable-gpu", "--no-sandbox", "--no-pdf-header-footer", "--run-all-compositor-stages-before-draw", "--virtual-time-budget=4000", "--print-to-pdf=" + os.path.abspath(out_pdf), url] - subprocess.run(cmd, capture_output=True, timeout=180) - return os.path.exists(out_pdf) + try: + r = subprocess.run(cmd, capture_output=True, timeout=180) + except (OSError, subprocess.SubprocessError) as e: + # A CHROME that points at nothing is a broken machine, not a bare one: + # report it as a failure with the reason, never as a traceback. + return False, "could not run %s: %s" % (CHROME, e) + if os.path.exists(out_pdf): + return True, "" + return False, "chromium exited %d: %s" % ( + r.returncode, (r.stderr or b"").decode("utf-8", "replace").strip()[-200:]) # ------------------------------------------------------------------ documents @@ -384,7 +396,15 @@ def f1_fpdf(): def l1_libreoffice(): - """Word-native dialect: build a DOCX, let LibreOffice render it to PDF.""" + """Word-native dialect: build a DOCX, let LibreOffice render it to PDF. + + The profile path must be ABSOLUTE. `-env:UserInstallation` takes a file + URL, and `file:///` + a relative path resolves against the filesystem root: + soffice then fails to create its profile, exits 1 and writes nothing. This + function used to swallow that and return None, so the corpus came back with + 15 documents instead of 16 and every figure downstream was quietly computed + over a different corpus than the one on record. + """ import docx from docx.shared import Pt, Inches, RGBColor from docx.enum.text import WD_ALIGN_PARAGRAPH @@ -409,39 +429,107 @@ def l1_libreoffice(): t.cell(i, j).text = c d.add_paragraph(LOREM2).alignment = WD_ALIGN_PARAGRAPH.JUSTIFY d.save(tmp) - prof = os.path.join(OUT, "_loprof2") - subprocess.run([SOFFICE, "--headless", "--norestore", - "-env:UserInstallation=file:///" + prof.replace("\\", "/"), - "--convert-to", "pdf", "--outdir", OUT, tmp], - capture_output=True, timeout=300) + prof = os.path.abspath(os.path.join(OUT, "_loprof2")) + r = subprocess.run([SOFFICE, "--headless", "--norestore", + "-env:UserInstallation=file:///" + prof.replace("\\", "/"), + "--convert-to", "pdf", "--outdir", OUT, tmp], + capture_output=True, timeout=300) src = os.path.join(OUT, "_l1.pdf") if os.path.exists(src): shutil.move(src, out) - return out if os.path.exists(out) else None - - -if __name__ == "__main__": - made = [] - for fn in (c1_whitepaper, c2_paper2col, c3_tables, c4_i18n, c5_graphics, - c6_long, c7_code, c8_toc_links): + if os.path.exists(out): + return out + raise RuntimeError("soffice exited %d without writing a PDF: %s" % ( + r.returncode, (r.stderr or b"").decode("utf-8", "replace").strip()[-200:])) + + +CHROMIUM_DOCS = (c1_whitepaper, c2_paper2col, c3_tables, c4_i18n, c5_graphics, + c6_long, c7_code, c8_toc_links) +PURE_PYTHON_DOCS = (r1_reportlab, f1_fpdf) # no external tool needed +SOFFICE_DOCS = (l1_libreoffice,) + + +def main(strict=None): + """Generate what this machine can, and say plainly what it could not. + + A missing external tool is a SKIP (exit 0, listed): the ReportLab and fpdf2 + documents need nothing but Python and must still be produced. A tool that + is present and fails is an ERROR (exit 1): that is a broken machine, not a + thin one. The previous version made no such distinction -- it called + subprocess with CHROME=None and died on a bare TypeError before writing a + single file, which is the "gate that cannot run" failure mode this + repository has already been bitten by once (STATUS.md §5). + + `--strict` makes a skip fatal too, and CI passes it. The distinction is + right for a contributor on a thin machine and wrong for the environment of + record: measured in a bare container, this printed "SKIPPED 8 document(s)" + and "gate numbers are NOT comparable", then exited 0, and the gate went on + to score the 8 documents that did exist against a 16-document baseline. + Prose that the next step ignores is not a safeguard. + """ + if strict is None: + strict = "--strict" in sys.argv[1:] + os.makedirs(HTML, exist_ok=True) + print("capabilities: chromium=%s soffice=%s" % + (CHROME or "MISSING", SOFFICE or "MISSING")) + made, skipped, failed = [], [], [] + + for fn in CHROMIUM_DOCS: + name = fn.__name__ + if not CHROME: + skipped.append((name, "no Chromium (set CHROME=/path/to/chrome)")) + continue h = fn() - name = os.path.splitext(os.path.basename(h))[0] - pdf = os.path.join(OUT, name + ".pdf") - ok = chrome_pdf(h, pdf) - print((" OK " if ok else "FAIL ") + name) + pdf = os.path.join(OUT, os.path.splitext(os.path.basename(h))[0] + ".pdf") + ok, why = chrome_pdf(h, pdf) if ok: - made.append(pdf) - for fn in (r1_reportlab, f1_fpdf, l1_libreoffice): + made.append(pdf); print(" OK " + name) + else: + failed.append((name, why)); print(" FAIL " + name) + + for fn in PURE_PYTHON_DOCS + SOFFICE_DOCS: + name = fn.__name__ + if fn in SOFFICE_DOCS and not SOFFICE: + skipped.append((name, "no LibreOffice (set SOFFICE=/path/to/soffice)")) + continue try: p = fn() - if p and os.path.exists(p): - made.append(p); print(" OK " + os.path.basename(p)) + made.append(p); print(" OK " + os.path.basename(p)) except Exception as e: - print("FAIL %s: %s" % (fn.__name__, e)) - print("\n%d PDFs" % len(made)) - import fitz - for m in sorted(made): - d = fitz.open(m) - print(" %-28s %d pages producer=%s" % - (os.path.basename(m), d.page_count, d.metadata.get("producer"))) - d.close() + failed.append((name, "%s: %s" % (type(e).__name__, e))) + print(" FAIL " + name) + + print("\n%d PDFs in %s" % (len(made), os.path.abspath(OUT))) + try: + import fitz + for m in sorted(made): + d = fitz.open(m) + print(" %-28s %d pages producer=%s" % + (os.path.basename(m), d.page_count, d.metadata.get("producer"))) + d.close() + except ImportError: + pass # the listing is a courtesy, not the job + + if skipped: + print("\nSKIPPED %d document(s) -- the tool that makes them is not " + "installed:" % len(skipped)) + for name, why in skipped: + print(" %-20s %s" % (name, why)) + print("The corpus is incomplete, so gate numbers from it are NOT " + "comparable to the recorded baselines.") + if failed: + print("\nFAILED %d document(s) -- the tool IS present and did not " + "deliver:" % len(failed)) + for name, why in failed: + print(" %-20s %s" % (name, why)) + return 1 + if skipped and strict: + print("\n--strict: an incomplete corpus is a failure here. Provision " + "the missing tool (scripts/bootstrap.sh) or drop --strict and " + "accept that the numbers describe a different corpus.") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/golden/01_whitepaper_market.json b/testkit/golden/01_whitepaper_market.json index 002320a..2d8fdea 100644 --- a/testkit/golden/01_whitepaper_market.json +++ b/testkit/golden/01_whitepaper_market.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":2247,"draws":[["rect",[0.0,0.0,612.0,170.0],"#1e3a5f",null],["rect",[0.0,170.0,612.0,174.0],"#f59e0b",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["hline",[60.0,239.0,552.0,239.0],null,"#e5e7eb"],["rect",[57.0,434.0,555.0,480.0],"#eff6ff",null],["vline",[57.0,434.0,57.0,480.0],null,"#2563eb"]],"lines":[[[54.0,65.2,501.4,99.7],92.0,"6baac5944897","Helvetica-Bold",25.0,2],[[54.0,102.6,234.6,119.7],116.0,"116933731436","Helvetica",12.5,0],[[54.0,137.8,332.9,150.8],148.0,"56cde2dc90f1","Helvetica",9.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"0cd2664cb285","Helvetica",8.0,0],[[60.0,208.9,212.1,230.9],226.0,"3abcc2cf0440","Helvetica-Bold",16.0,2],[[60.0,248.2,552.0,262.6],259.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,263.2,552.0,277.6],274.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,278.2,552.0,292.6],289.5,"59c240692658","Helvetica",10.5,0],[[60.0,293.2,552.0,307.6],304.5,"00320e8c953c","Helvetica",10.5,0],[[60.0,308.2,552.0,322.6],319.5,"7c56e6a71d39","Helvetica",10.5,0],[[60.0,323.2,552.0,337.6],334.5,"fe07d78ac6b2","Helvetica",10.5,0],[[60.0,338.2,131.2,352.6],349.5,"5b2aaaa48c3f","Helvetica",10.5,0],[[60.0,361.2,552.0,375.6],372.5,"77087ed66d53","Helvetica",10.5,0],[[60.0,376.2,552.0,390.6],387.5,"637ca044c4c8","Helvetica",10.5,0],[[60.0,391.2,552.0,405.6],402.5,"1033c9232f0a","Helvetica",10.5,0],[[60.0,406.2,316.8,420.6],417.5,"438005e94aa0","Helvetica",10.5,0],[[69.0,442.2,545.0,456.1],453.0,"31a3d1564185","Helvetica-Bold",10.0,2],[[69.0,456.2,308.5,470.0],467.0,"382ca6927e07","Helvetica",10.0,0],[[60.0,496.9,193.4,518.9],514.0,"98eb4dfabd72","Helvetica-Bold",16.0,2],[[60.0,525.2,552.0,539.6],536.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,540.2,552.0,554.6],551.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,555.2,552.0,569.6],566.5,"59c240692658","Helvetica",10.5,0],[[60.0,570.2,552.0,584.6],581.5,"2dcf90a218dc","Helvetica",10.5,0],[[60.0,585.2,243.3,599.6],596.5,"84cdc161ffa6","Helvetica",10.5,0],[[60.0,613.1,220.5,630.3],626.5,"c33975c3410c","Helvetica-Bold",12.5,2],[[60.0,635.2,552.0,649.6],646.5,"ef00683c2526","Helvetica",10.5,0],[[60.0,650.2,552.0,664.6],661.5,"c3eb2ca9dba2","Helvetica",10.5,0],[[60.0,665.2,552.0,679.6],676.5,"4aae74c39e94","Helvetica",10.5,0],[[60.0,680.2,162.7,694.6],691.5,"7d1b38392f3d","Helvetica",10.5,0],[[60.0,703.1,64.2,719.6],716.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,703.2,547.5,717.6],714.5,"062759fe6c36","Helvetica",10.5,0]],"n_blocks":13,"n_draws":6,"n_images":0,"n_lines":32,"n_links":0,"n_spans":33,"size":[612.0,792.0]},{"chars":1370,"draws":[["rect",[0.0,0.0,612.0,30.0],"#1e3a5f",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["rect",[54.0,189.0,558.0,213.0],"#1e3a5f",null],["rect",[54.0,213.0,558.0,237.0],"#ffffff",null],["rect",[54.0,237.0,558.0,261.0],"#f8fafc",null],["rect",[54.0,261.0,558.0,285.0],"#ffffff",null],["rect",[54.0,285.0,558.0,309.0],"#f8fafc",null],["hline",[54.0,189.0,558.0,189.0],null,"#cbd5e1"],["hline",[54.0,309.0,558.0,309.0],null,"#cbd5e1"],["vline",[54.0,189.0,54.0,309.0],null,"#cbd5e1"],["vline",[558.0,189.0,558.0,309.0],null,"#cbd5e1"],["hline",[54.0,213.0,558.0,213.0],null,"#cbd5e1"],["hline",[54.0,237.0,558.0,237.0],null,"#cbd5e1"],["hline",[54.0,261.0,558.0,261.0],null,"#cbd5e1"],["hline",[54.0,285.0,558.0,285.0],null,"#cbd5e1"],["vline",[124.0,189.0,124.0,309.0],null,"#cbd5e1"],["vline",[254.0,189.0,254.0,309.0],null,"#cbd5e1"],["vline",[339.0,189.0,339.0,309.0],null,"#cbd5e1"],["vline",[419.0,189.0,419.0,309.0],null,"#cbd5e1"],["hline",[106.0,578.0,470.0,578.0],null,"#9ca3af"],["vline",[106.0,440.0,106.0,578.0],null,"#9ca3af"],["hline",[106.0,543.5,470.0,543.5],null,"#e5e7eb"],["hline",[106.0,509.0,470.0,509.0],null,"#e5e7eb"],["hline",[106.0,474.5,470.0,474.5],null,"#e5e7eb"],["hline",[106.0,440.0,470.0,440.0],null,"#e5e7eb"],["rect",[122.4,553.5,162.4,578.0],"#2563eb",null],["rect",[195.2,541.2,235.2,578.0],"#2563eb",null],["rect",[268.0,522.1,308.0,578.0],"#2563eb",null],["rect",[340.8,492.1,380.8,578.0],"#2563eb",null],["rect",[413.6,458.0,453.6,578.0],"#2563eb",null]],"lines":[[[54.0,9.9,235.8,21.6],19.0,"f8a88f9d26fe","Helvetica-Bold",8.5,2],[[466.8,9.9,558.0,21.5],19.0,"993b1fe1eb7f","Helvetica",8.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"66d39b62e4c2","Helvetica",8.0,0],[[60.0,59.1,64.2,75.6],72.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,59.2,473.7,73.6],70.5,"2035a2ec819c","Helvetica",10.5,0],[[60.0,82.1,64.2,98.6],95.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,82.2,457.9,96.6],93.5,"79e4334cb2db","Helvetica",10.5,0],[[60.0,124.1,210.8,141.3],137.5,"e21edaa94737","Helvetica-Bold",12.5,2],[[60.0,146.2,552.0,160.6],157.5,"ecdc4c5e0fb0","Helvetica",10.5,0],[[60.0,161.2,392.0,175.6],172.5,"3134540acdd7","Helvetica",10.5,0],[[62.0,194.4,78.5,206.8],204.0,"5bd44ebe63cd","Helvetica-Bold",9.0,2],[[132.0,194.4,198.0,206.8],204.0,"88b98a26617d","Helvetica-Bold",9.0,2],[[262.0,194.4,317.5,206.8],204.0,"f700c2d000a9","Helvetica-Bold",9.0,2],[[347.0,194.4,396.0,206.8],204.0,"60fd0d90fcb2","Helvetica-Bold",9.0,2],[[427.0,194.4,457.5,206.8],204.0,"5f03a109b61c","Helvetica-Bold",9.0,2],[[62.0,218.3,97.5,230.7],228.0,"5973df6188ff","Helvetica",9.0,0],[[132.0,218.3,205.0,230.7],228.0,"3f812f36e1f3","Helvetica",9.0,0],[[262.0,218.3,284.5,230.7],228.0,"4257c06895b6","Helvetica",9.0,0],[[347.0,218.3,376.5,230.7],228.0,"10a499e98015","Helvetica",9.0,0],[[427.0,218.3,498.0,230.7],228.0,"4bcd3a926bb7","Helvetica",9.0,0],[[62.0,242.3,98.5,254.7],252.0,"2dfa66079d9b","Helvetica",9.0,0],[[132.0,242.3,169.5,254.7],252.0,"03342dbb856b","Helvetica",9.0,0],[[262.0,242.3,284.5,254.7],252.0,"8e2315637128","Helvetica",9.0,0],[[347.0,242.3,376.5,254.7],252.0,"232f94659f70","Helvetica",9.0,0],[[427.0,242.3,484.5,254.7],252.0,"f57bece3ad41","Helvetica",9.0,0],[[62.0,266.3,85.0,278.7],276.0,"8bf1ef5668b1","Helvetica",9.0,0],[[132.0,266.3,219.5,278.7],276.0,"d6022e17c5c2","Helvetica",9.0,0],[[262.0,266.3,284.5,278.7],276.0,"d751b3cfe978","Helvetica",9.0,0],[[347.0,266.3,359.0,278.7],276.0,"d7bef4f23346","Helvetica",9.0,0],[[427.0,266.3,500.0,278.7],276.0,"9d6d8556fb86","Helvetica",9.0,0],[[62.0,290.3,80.5,302.7],300.0,"c5c5ba59d015","Helvetica",9.0,0],[[132.0,290.3,196.5,302.7],300.0,"b5bd2faac027","Helvetica",9.0,0],[[262.0,290.3,284.5,302.7],300.0,"d9a670e0695f","Helvetica",9.0,0],[[347.0,290.3,387.0,302.7],300.0,"a2db5af4b95c","Helvetica",9.0,0],[[427.0,290.3,492.0,302.7],300.0,"3aa3a4071e76","Helvetica",9.0,0],[[60.0,312.4,331.2,324.0],321.5,"24af50d96ead","Helvetica",8.5,0],[[60.0,344.9,164.0,366.9],362.0,"018aeea0b738","Helvetica-Bold",16.0,2],[[60.0,373.2,552.0,387.6],384.5,"6c310f32a35a","Helvetica",10.5,0],[[60.0,388.2,203.6,402.6],399.5,"1c82934b1d77","Helvetica",10.5,0],[[92.2,539.0,100.0,548.6],546.5,"f6e1126cedeb","Helvetica",7.0,0],[[92.2,504.5,100.0,514.1],512.0,"e1822db470e6","Helvetica",7.0,0],[[92.2,470.0,100.0,479.6],477.5,"450ddec8dd20","Helvetica",7.0,0],[[88.3,435.5,100.0,445.1],443.0,"dbc0f0048544","Helvetica",7.0,0],[[133.5,581.4,151.3,592.4],590.0,"e575dccc7114","Helvetica",8.0,0],[[138.0,540.9,146.8,551.9],549.5,"9e6a55b6b456","Helvetica-Bold",8.0,2],[[206.3,581.4,224.1,592.4],590.0,"445cd2fd3273","Helvetica",8.0,0],[[210.8,528.6,219.6,539.6],537.2,"bc33ea4e26e5","Helvetica-Bold",8.0,2],[[279.1,581.4,296.9,592.4],590.0,"7e79a3af2634","Helvetica",8.0,0],[[283.6,509.5,292.4,520.5],518.1,"761f22b2c159","Helvetica-Bold",8.0,2],[[351.9,581.4,369.7,592.4],590.0,"004be89dd9e0","Helvetica",8.0,0],[[356.4,479.5,365.2,490.5],488.1,"a17554a0d2b1","Helvetica-Bold",8.0,2],[[424.7,581.4,442.5,592.4],590.0,"aee655773d85","Helvetica",8.0,0],[[429.2,445.4,438.0,456.5],454.0,"b37f6ddcefad","Helvetica-Bold",8.0,2],[[106.0,421.8,359.4,434.9],432.0,"2832cccf5f16","Helvetica-Bold",9.5,2],[[60.0,630.1,143.4,647.3],643.5,"59a3ac83f950","Helvetica-Bold",12.5,2],[[60.0,652.2,552.0,666.6],663.5,"ef00683c2526","Helvetica",10.5,0],[[60.0,667.2,552.0,681.6],678.5,"c3eb2ca9dba2","Helvetica",10.5,0],[[60.0,682.2,552.0,696.6],693.5,"65aa48b3b50f","Helvetica",10.5,0],[[60.0,697.2,139.9,711.6],708.5,"590524885ef6","Helvetica",10.5,0]],"n_blocks":31,"n_draws":30,"n_images":0,"n_lines":60,"n_links":0,"n_spans":60,"size":[612.0,792.0]},{"chars":1624,"draws":[["rect",[0.0,0.0,612.0,30.0],"#1e3a5f",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["rect",[57.0,158.0,555.0,204.0],"#fef3c7",null],["vline",[57.0,158.0,57.0,204.0],null,"#f59e0b"],["hline",[60.0,306.8,246.7,306.8],null,"#2563eb"]],"lines":[[[54.0,9.9,235.8,21.6],19.0,"f8a88f9d26fe","Helvetica-Bold",8.5,2],[[466.8,9.9,558.0,21.5],19.0,"993b1fe1eb7f","Helvetica",8.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"01033ba8b734","Helvetica",8.0,0],[[60.0,59.1,66.7,75.6],72.0,"356a192b7913","Helvetica",12.0,0],[[78.0,59.2,311.4,73.6],70.5,"0b9d291ecc0a","Helvetica",10.5,0],[[60.0,82.1,66.7,98.6],95.0,"da4b9237bacc","Helvetica",12.0,0],[[78.0,82.2,309.7,96.6],93.5,"cc510cc71fb3","Helvetica",10.5,0],[[60.0,105.1,66.7,121.6],118.0,"77de68daecd8","Helvetica",12.0,0],[[78.0,105.2,347.0,119.6],116.5,"224c569b4fce","Helvetica",10.5,0],[[60.0,128.1,66.7,144.6],141.0,"1b6453892473","Helvetica",12.0,0],[[78.0,128.2,347.0,142.6],139.5,"98036bd22479","Helvetica",10.5,0],[[69.0,166.2,545.0,180.1],177.0,"529e43488d56","Helvetica-Bold",10.0,2],[[69.0,180.2,248.5,194.0],191.0,"1f7360f7441f","Helvetica",10.0,0],[[60.0,220.9,252.1,242.9],238.0,"c0f1b8584458","Helvetica-Bold",16.0,2],[[60.0,249.2,552.0,263.6],260.5,"77087ed66d53","Helvetica",10.5,0],[[60.0,264.2,552.0,278.6],275.5,"637ca044c4c8","Helvetica",10.5,0],[[60.0,279.2,112.5,293.6],290.5,"cf7dd1fd5070","Helvetica",10.5,0],[[130.7,279.2,140.6,293.6],290.5,"cdde5862ce5c","Helvetica",10.5,0],[[158.8,279.2,195.5,293.6],290.5,"45161b3a1365","Helvetica",10.5,0],[[213.7,279.2,256.3,293.6],290.5,"627dac8706cf","Helvetica",10.5,0],[[274.4,279.2,296.0,293.6],290.5,"16f2df8883b0","Helvetica",10.5,0],[[314.2,279.2,353.9,293.6],290.5,"8367aa06aeba","Helvetica",10.5,0],[[372.1,279.2,433.9,293.6],290.5,"65a9ec204e1d","Helvetica",10.5,0],[[452.1,279.2,462.6,293.6],290.5,"6afe7f4aed73","Helvetica",10.5,0],[[480.7,279.2,525.1,293.6],290.5,"dcf16d9d0564","Helvetica",10.5,0],[[543.2,279.2,552.0,293.6],290.5,"27e90dfa57c3","Helvetica",10.5,0],[[60.0,294.2,249.6,308.6],305.5,"531d499539c9","Helvetica",10.5,0],[[60.0,317.2,552.0,331.6],328.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,332.2,552.0,346.6],343.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,347.2,552.0,361.6],358.5,"59c240692658","Helvetica",10.5,0],[[60.0,362.2,552.0,376.6],373.5,"00320e8c953c","Helvetica",10.5,0],[[60.0,377.2,552.0,391.6],388.5,"7c56e6a71d39","Helvetica",10.5,0],[[60.0,392.2,141.1,406.6],403.5,"f218cede2045","Helvetica",10.5,0],[[60.0,424.9,164.9,446.9],442.0,"5c5798940b93","Helvetica-Bold",16.0,2],[[60.0,453.2,552.0,467.6],464.5,"4151dfe4727b","Helvetica",10.5,0],[[60.0,468.2,552.0,482.6],479.5,"829ec1dd24a9","Helvetica",10.5,0],[[60.0,483.2,471.4,497.6],494.5,"b79ca4c4fde9","Helvetica",10.5,0]],"n_blocks":12,"n_draws":5,"n_images":0,"n_lines":38,"n_links":1,"n_spans":40,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":2247,"draws":[["rect",[0.0,0.0,612.0,170.0],"#1e3a5f",null],["rect",[0.0,170.0,612.0,174.0],"#f59e0b",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["hline",[60.0,239.0,552.0,239.0],null,"#e5e7eb"],["rect",[57.0,434.0,555.0,480.0],"#eff6ff",null],["vline",[57.0,434.0,57.0,480.0],null,"#2563eb"]],"lines":[[[54.0,65.2,501.4,99.7],92.0,"6baac5944897","Helvetica-Bold",25.0,2],[[54.0,102.6,234.6,119.7],116.0,"116933731436","Helvetica",12.5,0],[[54.0,137.8,332.9,150.8],148.0,"56cde2dc90f1","Helvetica",9.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"0cd2664cb285","Helvetica",8.0,0],[[60.0,208.9,212.1,230.9],226.0,"3abcc2cf0440","Helvetica-Bold",16.0,2],[[60.0,248.2,552.0,262.6],259.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,263.2,552.0,277.6],274.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,278.2,552.0,292.6],289.5,"59c240692658","Helvetica",10.5,0],[[60.0,293.2,552.0,307.6],304.5,"00320e8c953c","Helvetica",10.5,0],[[60.0,308.2,552.0,322.6],319.5,"7c56e6a71d39","Helvetica",10.5,0],[[60.0,323.2,552.0,337.6],334.5,"fe07d78ac6b2","Helvetica",10.5,0],[[60.0,338.2,131.2,352.6],349.5,"5b2aaaa48c3f","Helvetica",10.5,0],[[60.0,361.2,552.0,375.6],372.5,"77087ed66d53","Helvetica",10.5,0],[[60.0,376.2,552.0,390.6],387.5,"637ca044c4c8","Helvetica",10.5,0],[[60.0,391.2,552.0,405.6],402.5,"1033c9232f0a","Helvetica",10.5,0],[[60.0,406.2,316.8,420.6],417.5,"438005e94aa0","Helvetica",10.5,0],[[69.0,442.2,545.0,456.1],453.0,"31a3d1564185","Helvetica-Bold",10.0,2],[[69.0,456.2,308.5,470.0],467.0,"382ca6927e07","Helvetica",10.0,0],[[60.0,496.9,193.4,518.9],514.0,"98eb4dfabd72","Helvetica-Bold",16.0,2],[[60.0,525.2,552.0,539.6],536.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,540.2,552.0,554.6],551.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,555.2,552.0,569.6],566.5,"59c240692658","Helvetica",10.5,0],[[60.0,570.2,552.0,584.6],581.5,"2dcf90a218dc","Helvetica",10.5,0],[[60.0,585.2,243.3,599.6],596.5,"84cdc161ffa6","Helvetica",10.5,0],[[60.0,613.1,220.5,630.3],626.5,"c33975c3410c","Helvetica-Bold",12.5,2],[[60.0,635.2,552.0,649.6],646.5,"ef00683c2526","Helvetica",10.5,0],[[60.0,650.2,552.0,664.6],661.5,"c3eb2ca9dba2","Helvetica",10.5,0],[[60.0,665.2,552.0,679.6],676.5,"4aae74c39e94","Helvetica",10.5,0],[[60.0,680.2,162.7,694.6],691.5,"7d1b38392f3d","Helvetica",10.5,0],[[60.0,703.1,64.2,719.6],716.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,703.2,547.5,717.6],714.5,"062759fe6c36","Helvetica",10.5,0]],"n_blocks":13,"n_draws":6,"n_images":0,"n_lines":32,"n_links":0,"n_spans":33,"size":[612.0,792.0]},{"chars":1370,"draws":[["rect",[0.0,0.0,612.0,30.0],"#1e3a5f",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["rect",[54.0,189.0,558.0,213.0],"#1e3a5f",null],["rect",[54.0,213.0,558.0,237.0],"#ffffff",null],["rect",[54.0,237.0,558.0,261.0],"#f8fafc",null],["rect",[54.0,261.0,558.0,285.0],"#ffffff",null],["rect",[54.0,285.0,558.0,309.0],"#f8fafc",null],["hline",[54.0,189.0,558.0,189.0],null,"#cbd5e1"],["hline",[54.0,309.0,558.0,309.0],null,"#cbd5e1"],["vline",[54.0,189.0,54.0,309.0],null,"#cbd5e1"],["vline",[558.0,189.0,558.0,309.0],null,"#cbd5e1"],["hline",[54.0,213.0,558.0,213.0],null,"#cbd5e1"],["hline",[54.0,237.0,558.0,237.0],null,"#cbd5e1"],["hline",[54.0,261.0,558.0,261.0],null,"#cbd5e1"],["hline",[54.0,285.0,558.0,285.0],null,"#cbd5e1"],["vline",[124.0,189.0,124.0,309.0],null,"#cbd5e1"],["vline",[254.0,189.0,254.0,309.0],null,"#cbd5e1"],["vline",[339.0,189.0,339.0,309.0],null,"#cbd5e1"],["vline",[419.0,189.0,419.0,309.0],null,"#cbd5e1"],["hline",[106.0,578.0,470.0,578.0],null,"#9ca3af"],["vline",[106.0,440.0,106.0,578.0],null,"#9ca3af"],["hline",[106.0,543.5,470.0,543.5],null,"#e5e7eb"],["hline",[106.0,509.0,470.0,509.0],null,"#e5e7eb"],["hline",[106.0,474.5,470.0,474.5],null,"#e5e7eb"],["hline",[106.0,440.0,470.0,440.0],null,"#e5e7eb"],["rect",[122.4,553.5,162.4,578.0],"#2563eb",null],["rect",[195.2,541.2,235.2,578.0],"#2563eb",null],["rect",[268.0,522.1,308.0,578.0],"#2563eb",null],["rect",[340.8,492.1,380.8,578.0],"#2563eb",null],["rect",[413.6,458.0,453.6,578.0],"#2563eb",null]],"lines":[[[54.0,9.9,235.8,21.6],19.0,"f8a88f9d26fe","Helvetica-Bold",8.5,2],[[466.8,9.9,558.0,21.5],19.0,"993b1fe1eb7f","Helvetica",8.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"66d39b62e4c2","Helvetica",8.0,0],[[60.0,59.1,64.2,75.6],72.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,59.2,473.7,73.6],70.5,"2035a2ec819c","Helvetica",10.5,0],[[60.0,82.1,64.2,98.6],95.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,82.2,457.9,96.6],93.5,"79e4334cb2db","Helvetica",10.5,0],[[60.0,124.1,210.8,141.3],137.5,"e21edaa94737","Helvetica-Bold",12.5,2],[[60.0,146.2,552.0,160.6],157.5,"ecdc4c5e0fb0","Helvetica",10.5,0],[[60.0,161.2,392.0,175.6],172.5,"3134540acdd7","Helvetica",10.5,0],[[62.0,194.4,78.5,206.8],204.0,"5bd44ebe63cd","Helvetica-Bold",9.0,2],[[132.0,194.4,198.0,206.8],204.0,"88b98a26617d","Helvetica-Bold",9.0,2],[[262.0,194.4,317.5,206.8],204.0,"f700c2d000a9","Helvetica-Bold",9.0,2],[[347.0,194.4,396.0,206.8],204.0,"60fd0d90fcb2","Helvetica-Bold",9.0,2],[[427.0,194.4,457.5,206.8],204.0,"5f03a109b61c","Helvetica-Bold",9.0,2],[[62.0,218.3,97.5,230.7],228.0,"5973df6188ff","Helvetica",9.0,0],[[132.0,218.3,205.0,230.7],228.0,"3f812f36e1f3","Helvetica",9.0,0],[[262.0,218.3,284.5,230.7],228.0,"4257c06895b6","Helvetica",9.0,0],[[347.0,218.3,376.5,230.7],228.0,"10a499e98015","Helvetica",9.0,0],[[427.0,218.3,498.0,230.7],228.0,"4bcd3a926bb7","Helvetica",9.0,0],[[62.0,242.3,98.5,254.7],252.0,"2dfa66079d9b","Helvetica",9.0,0],[[132.0,242.3,169.5,254.7],252.0,"03342dbb856b","Helvetica",9.0,0],[[262.0,242.3,284.5,254.7],252.0,"8e2315637128","Helvetica",9.0,0],[[347.0,242.3,376.5,254.7],252.0,"232f94659f70","Helvetica",9.0,0],[[427.0,242.3,484.5,254.7],252.0,"f57bece3ad41","Helvetica",9.0,0],[[62.0,266.3,85.0,278.7],276.0,"8bf1ef5668b1","Helvetica",9.0,0],[[132.0,266.3,219.5,278.7],276.0,"d6022e17c5c2","Helvetica",9.0,0],[[262.0,266.3,284.5,278.7],276.0,"d751b3cfe978","Helvetica",9.0,0],[[347.0,266.3,359.0,278.7],276.0,"d7bef4f23346","Helvetica",9.0,0],[[427.0,266.3,500.0,278.7],276.0,"9d6d8556fb86","Helvetica",9.0,0],[[62.0,290.3,80.5,302.7],300.0,"c5c5ba59d015","Helvetica",9.0,0],[[132.0,290.3,196.5,302.7],300.0,"b5bd2faac027","Helvetica",9.0,0],[[262.0,290.3,284.5,302.7],300.0,"d9a670e0695f","Helvetica",9.0,0],[[347.0,290.3,387.0,302.7],300.0,"a2db5af4b95c","Helvetica",9.0,0],[[427.0,290.3,492.0,302.7],300.0,"3aa3a4071e76","Helvetica",9.0,0],[[60.0,312.4,331.2,324.0],321.5,"24af50d96ead","Helvetica",8.5,0],[[60.0,344.9,164.0,366.9],362.0,"018aeea0b738","Helvetica-Bold",16.0,2],[[60.0,373.2,552.0,387.6],384.5,"6c310f32a35a","Helvetica",10.5,0],[[60.0,388.2,203.6,402.6],399.5,"1c82934b1d77","Helvetica",10.5,0],[[92.2,539.0,100.0,548.6],546.5,"f6e1126cedeb","Helvetica",7.0,0],[[92.2,504.5,100.0,514.1],512.0,"e1822db470e6","Helvetica",7.0,0],[[92.2,470.0,100.0,479.6],477.5,"450ddec8dd20","Helvetica",7.0,0],[[88.3,435.5,100.0,445.1],443.0,"dbc0f0048544","Helvetica",7.0,0],[[133.5,581.4,151.3,592.4],590.0,"e575dccc7114","Helvetica",8.0,0],[[138.0,540.9,146.8,551.9],549.5,"9e6a55b6b456","Helvetica-Bold",8.0,2],[[206.3,581.4,224.1,592.4],590.0,"445cd2fd3273","Helvetica",8.0,0],[[210.8,528.6,219.6,539.6],537.2,"bc33ea4e26e5","Helvetica-Bold",8.0,2],[[279.1,581.4,296.9,592.4],590.0,"7e79a3af2634","Helvetica",8.0,0],[[283.6,509.5,292.4,520.5],518.1,"761f22b2c159","Helvetica-Bold",8.0,2],[[351.9,581.4,369.7,592.4],590.0,"004be89dd9e0","Helvetica",8.0,0],[[356.4,479.5,365.2,490.5],488.1,"a17554a0d2b1","Helvetica-Bold",8.0,2],[[424.7,581.4,442.5,592.4],590.0,"aee655773d85","Helvetica",8.0,0],[[429.2,445.4,438.0,456.5],454.0,"b37f6ddcefad","Helvetica-Bold",8.0,2],[[106.0,421.8,359.4,434.9],432.0,"2832cccf5f16","Helvetica-Bold",9.5,2],[[60.0,630.1,143.4,647.3],643.5,"59a3ac83f950","Helvetica-Bold",12.5,2],[[60.0,652.2,552.0,666.6],663.5,"ef00683c2526","Helvetica",10.5,0],[[60.0,667.2,552.0,681.6],678.5,"c3eb2ca9dba2","Helvetica",10.5,0],[[60.0,682.2,552.0,696.6],693.5,"65aa48b3b50f","Helvetica",10.5,0],[[60.0,697.2,139.9,711.6],708.5,"590524885ef6","Helvetica",10.5,0]],"n_blocks":31,"n_draws":30,"n_images":0,"n_lines":60,"n_links":0,"n_spans":60,"size":[612.0,792.0]},{"chars":1624,"draws":[["rect",[0.0,0.0,612.0,30.0],"#1e3a5f",null],["hline",[54.0,746.0,558.0,746.0],null,"#d1d5db"],["rect",[57.0,158.0,555.0,204.0],"#fef3c7",null],["vline",[57.0,158.0,57.0,204.0],null,"#f59e0b"],["hline",[60.0,306.8,246.7,306.8],null,"#2563eb"]],"lines":[[[54.0,9.9,235.8,21.6],19.0,"f8a88f9d26fe","Helvetica-Bold",8.5,2],[[466.8,9.9,558.0,21.5],19.0,"993b1fe1eb7f","Helvetica",8.5,0],[[54.0,749.4,201.7,760.4],758.0,"4f19ed365f89","Helvetica",8.0,0],[[532.6,749.4,558.0,760.4],758.0,"01033ba8b734","Helvetica",8.0,0],[[60.0,59.1,66.7,75.6],72.0,"356a192b7913","Helvetica",12.0,0],[[78.0,59.2,311.4,73.6],70.5,"0b9d291ecc0a","Helvetica",10.5,0],[[60.0,82.1,66.7,98.6],95.0,"da4b9237bacc","Helvetica",12.0,0],[[78.0,82.2,309.7,96.6],93.5,"cc510cc71fb3","Helvetica",10.5,0],[[60.0,105.1,66.7,121.6],118.0,"77de68daecd8","Helvetica",12.0,0],[[78.0,105.2,347.0,119.6],116.5,"224c569b4fce","Helvetica",10.5,0],[[60.0,128.1,66.7,144.6],141.0,"1b6453892473","Helvetica",12.0,0],[[78.0,128.2,347.0,142.6],139.5,"98036bd22479","Helvetica",10.5,0],[[69.0,166.2,545.0,180.1],177.0,"529e43488d56","Helvetica-Bold",10.0,2],[[69.0,180.2,248.5,194.0],191.0,"1f7360f7441f","Helvetica",10.0,0],[[60.0,220.9,252.1,242.9],238.0,"c0f1b8584458","Helvetica-Bold",16.0,2],[[60.0,249.2,552.0,263.6],260.5,"77087ed66d53","Helvetica",10.5,0],[[60.0,264.2,552.0,278.6],275.5,"637ca044c4c8","Helvetica",10.5,0],[[60.0,279.2,112.5,293.6],290.5,"cf7dd1fd5070","Helvetica",10.5,0],[[130.7,279.2,140.6,293.6],290.5,"cdde5862ce5c","Helvetica",10.5,0],[[158.8,279.2,195.5,293.6],290.5,"45161b3a1365","Helvetica",10.5,0],[[213.7,279.2,256.3,293.6],290.5,"627dac8706cf","Helvetica",10.5,0],[[274.4,279.2,296.0,293.6],290.5,"16f2df8883b0","Helvetica",10.5,0],[[314.2,279.2,353.9,293.6],290.5,"8367aa06aeba","Helvetica",10.5,0],[[372.1,279.2,433.9,293.6],290.5,"65a9ec204e1d","Helvetica",10.5,0],[[452.1,279.2,462.6,293.6],290.5,"6afe7f4aed73","Helvetica",10.5,0],[[480.7,279.2,525.1,293.6],290.5,"dcf16d9d0564","Helvetica",10.5,0],[[543.2,279.2,552.0,293.6],290.5,"27e90dfa57c3","Helvetica",10.5,0],[[60.0,294.2,249.6,308.6],305.5,"531d499539c9","Helvetica",10.5,0],[[60.0,317.2,552.0,331.6],328.5,"22a9076c4ee0","Helvetica",10.5,0],[[60.0,332.2,552.0,346.6],343.5,"616bc5c817a6","Helvetica",10.5,0],[[60.0,347.2,552.0,361.6],358.5,"59c240692658","Helvetica",10.5,0],[[60.0,362.2,552.0,376.6],373.5,"00320e8c953c","Helvetica",10.5,0],[[60.0,377.2,552.0,391.6],388.5,"7c56e6a71d39","Helvetica",10.5,0],[[60.0,392.2,141.1,406.6],403.5,"f218cede2045","Helvetica",10.5,0],[[60.0,424.9,164.9,446.9],442.0,"5c5798940b93","Helvetica-Bold",16.0,2],[[60.0,453.2,552.0,467.6],464.5,"4151dfe4727b","Helvetica",10.5,0],[[60.0,468.2,552.0,482.6],479.5,"829ec1dd24a9","Helvetica",10.5,0],[[60.0,483.2,471.4,497.6],494.5,"b79ca4c4fde9","Helvetica",10.5,0]],"n_blocks":12,"n_draws":5,"n_images":0,"n_lines":38,"n_links":1,"n_spans":40,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/02_research_paper.json b/testkit/golden/02_research_paper.json index c3fa77b..eb14d21 100644 --- a/testkit/golden/02_research_paper.json +++ b/testkit/golden/02_research_paper.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":2440,"draws":[["hline",[370.0,342.0,534.0,342.0],null,"#9ca3af"],["vline",[370.0,244.0,370.0,342.0],null,"#9ca3af"],["hline",[370.0,317.5,534.0,317.5],null,"#e5e7eb"],["hline",[370.0,293.0,534.0,293.0],null,"#e5e7eb"],["hline",[370.0,268.5,534.0,268.5],null,"#e5e7eb"],["hline",[370.0,244.0,534.0,244.0],null,"#e5e7eb"],["rect",[379.2,297.1,401.8,342.0],"#0f766e",null],["rect",[420.2,280.1,442.8,342.0],"#0f766e",null],["rect",[461.2,287.7,483.8,342.0],"#0f766e",null],["rect",[502.2,256.8,524.8,342.0],"#0f766e",null],["hline",[321.0,462.8,551.0,462.8],null,"#111111"],["hline",[321.0,480.8,551.0,480.8],null,"#111111"],["hline",[321.0,534.8,551.0,534.8],null,"#111111"]],"lines":[[[303.8,746.5,308.2,758.5],756.0,"356a192b7913","Times-Roman",9.0,0],[[91.4,63.3,520.6,86.8],81.0,"5fb56b252b1a","Times-Bold",17.0,2],[[279.6,84.3,332.4,107.8],102.0,"f3798f81c7b6","Times-Bold",17.0,2],[[208.1,115.4,403.9,130.1],127.0,"7d10ba8d0cba","Times-Roman",11.0,0],[[227.5,132.5,384.5,144.1],141.5,"8d2a4474d4ca","Times-Italic",9.5,1],[[64.0,221.5,106.8,237.4],233.5,"d79da395b5d7","Times-Bold",11.5,2],[[64.0,239.5,288.0,251.5],249.0,"0135dc6a34cf","Times-Roman",9.0,0],[[64.0,251.0,288.0,263.0],260.5,"470f37073c7f","Times-Roman",9.0,0],[[64.0,262.5,288.0,274.5],272.0,"712466eb24d2","Times-Roman",9.0,0],[[64.0,274.0,288.0,286.0],283.5,"25a9a3e083c3","Times-Roman",9.0,0],[[64.0,285.5,288.0,297.5],295.0,"2b6bc4539dd3","Times-Roman",9.0,0],[[64.0,297.0,288.0,309.0],306.5,"279083149d16","Times-Roman",9.0,0],[[64.0,308.5,288.0,320.5],318.0,"55fba4e9d148","Times-Roman",9.0,0],[[64.0,320.0,288.0,332.0],329.5,"3d5ff6eacb10","Times-Roman",9.0,0],[[64.0,331.5,139.5,343.5],341.0,"2512b2d472e0","Times-Roman",9.0,0],[[64.0,353.0,138.1,368.9],365.0,"ef5c75f8e615","Times-Bold",11.5,2],[[64.0,371.0,288.0,383.7],381.0,"645f69f54302","Times-Roman",9.5,0],[[64.0,383.2,288.0,395.9],393.2,"a6c93be17fda","Times-Roman",9.5,0],[[64.0,395.4,288.0,408.1],405.4,"96bf93b22024","Times-Roman",9.5,0],[[64.0,407.6,288.0,420.3],417.6,"fa7281512d9f","Times-Roman",9.5,0],[[64.0,419.8,288.0,432.5],429.8,"e7aa16e66fe4","Times-Roman",9.5,0],[[64.0,432.0,222.3,444.7],442.0,"9aaf60d64884","Times-Roman",9.5,0],[[64.0,449.2,288.0,461.9],459.2,"7664df3aca8b","Times-Roman",9.5,0],[[64.0,461.4,288.0,474.1],471.4,"bd7cff00bec2","Times-Roman",9.5,0],[[64.0,473.6,288.0,486.3],483.6,"9cd2af3ddc35","Times-Roman",9.5,0],[[64.0,485.8,177.2,498.5],495.8,"872c81949b3a","Times-Roman",9.5,0],[[64.0,508.0,113.8,523.9],520.0,"f5f7fb17b6da","Times-Bold",11.5,2],[[64.0,526.0,288.0,538.7],536.0,"0cc9b5b7e6cd","Times-Roman",9.5,0],[[64.0,538.2,288.0,550.9],548.2,"47a86d67b031","Times-Roman",9.5,0],[[64.0,550.4,288.0,563.1],560.4,"cbfef881ea4f","Times-Roman",9.5,0],[[64.0,562.6,288.0,575.3],572.6,"15604c9b91c6","Times-Roman",9.5,0],[[64.0,574.8,288.0,587.5],584.8,"1990fd16f99d","Times-Roman",9.5,0],[[64.0,587.0,203.3,599.7],597.0,"d4174ecdb992","Times-Roman",9.5,0],[[64.0,604.2,288.0,616.9],614.2,"d9ad36dc9ba3","Times-Roman",9.5,0],[[64.0,616.4,288.0,629.1],626.4,"2f8714ee0324","Times-Roman",9.5,0],[[64.0,628.6,143.1,641.3],638.6,"ded0e00ccab9","Times-Roman",9.5,0],[[356.2,313.0,364.0,322.6],320.5,"80e28a51cbc2","Helvetica",7.0,0],[[352.3,288.5,364.0,298.1],296.0,"a1422e6a1686","Helvetica",7.0,0],[[352.3,264.0,364.0,273.6],271.5,"fd93751649ac","Helvetica",7.0,0],[[352.3,239.5,364.0,249.1],247.0,"3d5bdf107de5","Helvetica",7.0,0],[[387.8,345.4,393.2,356.4],354.0,"02aa629c8b16","Helvetica",8.0,0],[[383.8,284.6,397.2,295.6],293.1,"310b86e0b62b","Helvetica-Bold",8.0,2],[[428.2,345.4,434.8,356.4],354.0,"c63ae6dd4fc9","Helvetica",8.0,0],[[424.8,267.5,438.2,278.6],276.1,"56ad4d4deaec","Helvetica-Bold",8.0,2],[[470.3,345.4,474.7,356.4],354.0,"d160e0986aca","Helvetica",8.0,0],[[465.8,275.2,479.2,286.2],283.7,"8bd7954c40c1","Helvetica-Bold",8.0,2],[[505.3,345.4,521.7,356.4],354.0,"5271593ca406","Helvetica",8.0,0],[[506.8,244.2,520.2,255.2],252.8,"3a2dc677d8e8","Helvetica-Bold",8.0,2],[[370.0,225.8,507.8,238.9],236.0,"ec1ce377395e","Helvetica-Bold",9.5,2],[[324.0,387.5,371.3,403.4],399.5,"928a067d807a","Times-Bold",11.5,2],[[324.0,405.5,548.0,418.2],415.5,"87e31f3c0ff9","Times-Roman",9.5,0],[[324.0,417.7,548.0,430.4],427.7,"09daac4cca57","Times-Roman",9.5,0],[[324.0,429.9,548.0,442.6],439.9,"3c20e90ecba1","Times-Roman",9.5,0],[[324.0,442.1,379.7,454.8],452.1,"4ed5c6c51621","Times-Roman",9.5,0],[[327.0,465.4,363.8,477.2],474.3,"81e47fe9b8ff","Times-Bold",8.5,2],[[405.2,465.4,440.8,477.2],474.3,"ac4b8555a6a9","Times-Bold",8.5,2],[[465.0,465.4,482.0,477.2],474.3,"5271593ca406","Times-Bold",8.5,2],[[511.0,465.7,516.2,476.8],474.3,"cd7b1f2bca92","Symbol",8.5,0],[[327.0,483.3,345.9,494.7],492.3,"097d071561bc","Times-Roman",8.5,0],[[405.2,483.3,424.9,494.7],492.3,"a91617c3a5a5","Times-Roman",8.5,0],[[465.0,483.3,484.7,494.7],492.3,"178a62919694","Times-Roman",8.5,0],[[511.0,483.3,527.1,494.7],492.3,"2a44d422d0e4","Times-Roman",8.5,0],[[327.0,501.3,344.9,512.7],510.3,"adac69379a62","Times-Roman",8.5,0],[[405.2,501.3,424.9,512.7],510.3,"999c73ef8663","Times-Roman",8.5,0],[[465.0,501.3,484.7,512.7],510.3,"b8e3e77b80d5","Times-Roman",8.5,0],[[511.0,501.3,531.4,512.7],510.3,"eb8cec8a0242","Times-Roman",8.5,0],[[327.0,519.3,349.2,530.7],528.3,"c0dcb865ad81","Times-Roman",8.5,0],[[405.2,519.3,424.9,530.7],528.3,"85f9b5046aeb","Times-Roman",8.5,0],[[465.0,519.3,484.7,530.7],528.3,"033089a2c7aa","Times-Roman",8.5,0],[[511.0,519.3,531.4,530.7],528.3,"d9a55bbd1066","Times-Roman",8.5,0],[[354.7,538.2,517.3,548.0],545.8,"07ced58ce1e1","Times-Italic",8.0,1],[[324.0,557.3,404.8,573.2],569.3,"22e5cef5d3db","Times-Bold",11.5,2],[[324.0,575.3,370.7,588.0],585.3,"4ddf7cc5b45f","Times-Roman",9.5,0],[[378.5,575.3,415.7,588.0],585.3,"7809ba8a38d7","Times-Roman",9.5,0],[[423.6,575.3,449.2,588.0],585.3,"4900504d5031","Times-Roman",9.5,0],[[457.0,575.3,468.9,588.0],585.3,"03fcaa9166c8","Times-Roman",9.5,0],[[476.8,575.3,548.0,588.0],585.3,"59a9182cfecf","Times-Roman",9.5,0],[[324.0,587.5,548.0,600.2],597.5,"a28c9c016e73","Times-Roman",9.5,0],[[324.0,599.7,548.0,612.4],609.7,"9abbaf6771c8","Times-Roman",9.5,0],[[324.0,611.9,548.0,624.6],621.9,"0194db296abf","Times-Roman",9.5,0],[[324.0,624.1,504.5,636.8],634.1,"f2e14c9a8ab7","Times-Roman",9.5,0],[[324.0,646.3,390.5,662.2],658.3,"f343f80ba6ff","Times-Bold",11.5,2],[[324.0,664.3,548.0,677.0],674.3,"6899dd6cbd3c","Times-Roman",9.5,0],[[324.0,676.5,548.0,689.2],686.5,"38f7b21002f8","Times-Roman",9.5,0],[[324.0,688.7,537.8,701.4],698.7,"1ebf620be75b","Times-Roman",9.5,0],[[324.0,710.9,377.6,726.8],722.9,"5d20d0fee3b9","Times-Bold",11.5,2]],"n_blocks":38,"n_draws":13,"n_images":0,"n_lines":86,"n_links":0,"n_spans":86,"size":[612.0,792.0]},{"chars":243,"draws":[],"lines":[[[303.8,746.5,308.2,758.5],756.0,"da4b9237bacc","Times-Roman",9.0,0],[[64.0,63.5,287.8,74.9],72.5,"6d86c17fec3c","Times-Roman",8.5,0],[[76.0,74.0,115.0,85.4],83.0,"7e1ef74ad489","Times-Roman",8.5,0],[[64.0,87.5,281.4,98.9],96.5,"eecc497351a9","Times-Roman",8.5,0],[[76.0,98.0,142.6,109.4],107.0,"5a94a9a16da2","Times-Roman",8.5,0],[[64.0,111.5,253.8,122.9],120.5,"430d85e47052","Times-Roman",8.5,0],[[76.0,122.0,178.7,133.4],131.0,"1ce32c373ca8","Times-Roman",8.5,0]],"n_blocks":7,"n_draws":0,"n_images":0,"n_lines":7,"n_links":0,"n_spans":7,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":2440,"draws":[["hline",[370.0,342.0,534.0,342.0],null,"#9ca3af"],["vline",[370.0,244.0,370.0,342.0],null,"#9ca3af"],["hline",[370.0,317.5,534.0,317.5],null,"#e5e7eb"],["hline",[370.0,293.0,534.0,293.0],null,"#e5e7eb"],["hline",[370.0,268.5,534.0,268.5],null,"#e5e7eb"],["hline",[370.0,244.0,534.0,244.0],null,"#e5e7eb"],["rect",[379.2,297.1,401.8,342.0],"#0f766e",null],["rect",[420.2,280.1,442.8,342.0],"#0f766e",null],["rect",[461.2,287.7,483.8,342.0],"#0f766e",null],["rect",[502.2,256.8,524.8,342.0],"#0f766e",null],["hline",[321.0,462.8,551.0,462.8],null,"#111111"],["hline",[321.0,480.8,551.0,480.8],null,"#111111"],["hline",[321.0,534.8,551.0,534.8],null,"#111111"]],"lines":[[[303.8,746.5,308.2,758.5],756.0,"356a192b7913","Times-Roman",9.0,0],[[91.4,63.3,520.6,86.8],81.0,"5fb56b252b1a","Times-Bold",17.0,2],[[279.6,84.3,332.4,107.8],102.0,"f3798f81c7b6","Times-Bold",17.0,2],[[208.1,115.4,403.9,130.1],127.0,"7d10ba8d0cba","Times-Roman",11.0,0],[[227.5,132.5,384.5,144.1],141.5,"8d2a4474d4ca","Times-Italic",9.5,1],[[64.0,221.5,106.8,237.4],233.5,"d79da395b5d7","Times-Bold",11.5,2],[[64.0,239.5,288.0,251.5],249.0,"0135dc6a34cf","Times-Roman",9.0,0],[[64.0,251.0,288.0,263.0],260.5,"470f37073c7f","Times-Roman",9.0,0],[[64.0,262.5,288.0,274.5],272.0,"712466eb24d2","Times-Roman",9.0,0],[[64.0,274.0,288.0,286.0],283.5,"25a9a3e083c3","Times-Roman",9.0,0],[[64.0,285.5,288.0,297.5],295.0,"2b6bc4539dd3","Times-Roman",9.0,0],[[64.0,297.0,288.0,309.0],306.5,"279083149d16","Times-Roman",9.0,0],[[64.0,308.5,288.0,320.5],318.0,"55fba4e9d148","Times-Roman",9.0,0],[[64.0,320.0,288.0,332.0],329.5,"3d5ff6eacb10","Times-Roman",9.0,0],[[64.0,331.5,139.5,343.5],341.0,"2512b2d472e0","Times-Roman",9.0,0],[[64.0,353.0,138.1,368.9],365.0,"ef5c75f8e615","Times-Bold",11.5,2],[[64.0,371.0,288.0,383.7],381.0,"645f69f54302","Times-Roman",9.5,0],[[64.0,383.2,288.0,395.9],393.2,"a6c93be17fda","Times-Roman",9.5,0],[[64.0,395.4,288.0,408.1],405.4,"96bf93b22024","Times-Roman",9.5,0],[[64.0,407.6,288.0,420.3],417.6,"fa7281512d9f","Times-Roman",9.5,0],[[64.0,419.8,288.0,432.5],429.8,"e7aa16e66fe4","Times-Roman",9.5,0],[[64.0,432.0,222.3,444.7],442.0,"9aaf60d64884","Times-Roman",9.5,0],[[64.0,449.2,288.0,461.9],459.2,"7664df3aca8b","Times-Roman",9.5,0],[[64.0,461.4,288.0,474.1],471.4,"bd7cff00bec2","Times-Roman",9.5,0],[[64.0,473.6,288.0,486.3],483.6,"9cd2af3ddc35","Times-Roman",9.5,0],[[64.0,485.8,177.2,498.5],495.8,"872c81949b3a","Times-Roman",9.5,0],[[64.0,508.0,113.8,523.9],520.0,"f5f7fb17b6da","Times-Bold",11.5,2],[[64.0,526.0,288.0,538.7],536.0,"0cc9b5b7e6cd","Times-Roman",9.5,0],[[64.0,538.2,288.0,550.9],548.2,"47a86d67b031","Times-Roman",9.5,0],[[64.0,550.4,288.0,563.1],560.4,"cbfef881ea4f","Times-Roman",9.5,0],[[64.0,562.6,288.0,575.3],572.6,"15604c9b91c6","Times-Roman",9.5,0],[[64.0,574.8,288.0,587.5],584.8,"1990fd16f99d","Times-Roman",9.5,0],[[64.0,587.0,203.3,599.7],597.0,"d4174ecdb992","Times-Roman",9.5,0],[[64.0,604.2,288.0,616.9],614.2,"d9ad36dc9ba3","Times-Roman",9.5,0],[[64.0,616.4,288.0,629.1],626.4,"2f8714ee0324","Times-Roman",9.5,0],[[64.0,628.6,143.1,641.3],638.6,"ded0e00ccab9","Times-Roman",9.5,0],[[356.2,313.0,364.0,322.6],320.5,"80e28a51cbc2","Helvetica",7.0,0],[[352.3,288.5,364.0,298.1],296.0,"a1422e6a1686","Helvetica",7.0,0],[[352.3,264.0,364.0,273.6],271.5,"fd93751649ac","Helvetica",7.0,0],[[352.3,239.5,364.0,249.1],247.0,"3d5bdf107de5","Helvetica",7.0,0],[[387.8,345.4,393.2,356.4],354.0,"02aa629c8b16","Helvetica",8.0,0],[[383.8,284.6,397.2,295.6],293.1,"310b86e0b62b","Helvetica-Bold",8.0,2],[[428.2,345.4,434.8,356.4],354.0,"c63ae6dd4fc9","Helvetica",8.0,0],[[424.8,267.5,438.2,278.6],276.1,"56ad4d4deaec","Helvetica-Bold",8.0,2],[[470.3,345.4,474.7,356.4],354.0,"d160e0986aca","Helvetica",8.0,0],[[465.8,275.2,479.2,286.2],283.7,"8bd7954c40c1","Helvetica-Bold",8.0,2],[[505.3,345.4,521.7,356.4],354.0,"5271593ca406","Helvetica",8.0,0],[[506.8,244.2,520.2,255.2],252.8,"3a2dc677d8e8","Helvetica-Bold",8.0,2],[[370.0,225.8,507.8,238.9],236.0,"ec1ce377395e","Helvetica-Bold",9.5,2],[[324.0,387.5,371.3,403.4],399.5,"928a067d807a","Times-Bold",11.5,2],[[324.0,405.5,548.0,418.2],415.5,"87e31f3c0ff9","Times-Roman",9.5,0],[[324.0,417.7,548.0,430.4],427.7,"09daac4cca57","Times-Roman",9.5,0],[[324.0,429.9,548.0,442.6],439.9,"3c20e90ecba1","Times-Roman",9.5,0],[[324.0,442.1,379.7,454.8],452.1,"4ed5c6c51621","Times-Roman",9.5,0],[[327.0,465.4,363.8,477.2],474.3,"81e47fe9b8ff","Times-Bold",8.5,2],[[405.2,465.4,440.8,477.2],474.3,"ac4b8555a6a9","Times-Bold",8.5,2],[[465.0,465.4,482.0,477.2],474.3,"5271593ca406","Times-Bold",8.5,2],[[511.0,465.7,516.2,476.8],474.3,"cd7b1f2bca92","Symbol",8.5,0],[[327.0,483.3,345.9,494.7],492.3,"097d071561bc","Times-Roman",8.5,0],[[405.2,483.3,424.9,494.7],492.3,"a91617c3a5a5","Times-Roman",8.5,0],[[465.0,483.3,484.7,494.7],492.3,"178a62919694","Times-Roman",8.5,0],[[511.0,483.3,527.1,494.7],492.3,"2a44d422d0e4","Times-Roman",8.5,0],[[327.0,501.3,344.9,512.7],510.3,"adac69379a62","Times-Roman",8.5,0],[[405.2,501.3,424.9,512.7],510.3,"999c73ef8663","Times-Roman",8.5,0],[[465.0,501.3,484.7,512.7],510.3,"b8e3e77b80d5","Times-Roman",8.5,0],[[511.0,501.3,531.4,512.7],510.3,"eb8cec8a0242","Times-Roman",8.5,0],[[327.0,519.3,349.2,530.7],528.3,"c0dcb865ad81","Times-Roman",8.5,0],[[405.2,519.3,424.9,530.7],528.3,"85f9b5046aeb","Times-Roman",8.5,0],[[465.0,519.3,484.7,530.7],528.3,"033089a2c7aa","Times-Roman",8.5,0],[[511.0,519.3,531.4,530.7],528.3,"d9a55bbd1066","Times-Roman",8.5,0],[[354.7,538.2,517.3,548.0],545.8,"07ced58ce1e1","Times-Italic",8.0,1],[[324.0,557.3,404.8,573.2],569.3,"22e5cef5d3db","Times-Bold",11.5,2],[[324.0,575.3,370.7,588.0],585.3,"4ddf7cc5b45f","Times-Roman",9.5,0],[[378.5,575.3,415.7,588.0],585.3,"7809ba8a38d7","Times-Roman",9.5,0],[[423.6,575.3,449.2,588.0],585.3,"4900504d5031","Times-Roman",9.5,0],[[457.0,575.3,468.9,588.0],585.3,"03fcaa9166c8","Times-Roman",9.5,0],[[476.8,575.3,548.0,588.0],585.3,"59a9182cfecf","Times-Roman",9.5,0],[[324.0,587.5,548.0,600.2],597.5,"a28c9c016e73","Times-Roman",9.5,0],[[324.0,599.7,548.0,612.4],609.7,"9abbaf6771c8","Times-Roman",9.5,0],[[324.0,611.9,548.0,624.6],621.9,"0194db296abf","Times-Roman",9.5,0],[[324.0,624.1,504.5,636.8],634.1,"f2e14c9a8ab7","Times-Roman",9.5,0],[[324.0,646.3,390.5,662.2],658.3,"f343f80ba6ff","Times-Bold",11.5,2],[[324.0,664.3,548.0,677.0],674.3,"6899dd6cbd3c","Times-Roman",9.5,0],[[324.0,676.5,548.0,689.2],686.5,"38f7b21002f8","Times-Roman",9.5,0],[[324.0,688.7,537.8,701.4],698.7,"1ebf620be75b","Times-Roman",9.5,0],[[324.0,710.9,377.6,726.8],722.9,"5d20d0fee3b9","Times-Bold",11.5,2]],"n_blocks":38,"n_draws":13,"n_images":0,"n_lines":86,"n_links":0,"n_spans":86,"size":[612.0,792.0]},{"chars":243,"draws":[],"lines":[[[303.8,746.5,308.2,758.5],756.0,"da4b9237bacc","Times-Roman",9.0,0],[[64.0,63.5,287.8,74.9],72.5,"6d86c17fec3c","Times-Roman",8.5,0],[[76.0,74.0,115.0,85.4],83.0,"7e1ef74ad489","Times-Roman",8.5,0],[[64.0,87.5,281.4,98.9],96.5,"eecc497351a9","Times-Roman",8.5,0],[[76.0,98.0,142.6,109.4],107.0,"5a94a9a16da2","Times-Roman",8.5,0],[[64.0,111.5,253.8,122.9],120.5,"430d85e47052","Times-Roman",8.5,0],[[76.0,122.0,178.7,133.4],131.0,"1ce32c373ca8","Times-Roman",8.5,0]],"n_blocks":7,"n_draws":0,"n_images":0,"n_lines":7,"n_links":0,"n_spans":7,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/03_tech_report_code.json b/testkit/golden/03_tech_report_code.json index e1773fb..a3a8111 100644 --- a/testkit/golden/03_tech_report_code.json +++ b/testkit/golden/03_tech_report_code.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":1222,"draws":[["hline",[54.0,40.0,558.0,40.0],null,"#0f766e"],["hline",[60.0,138.2,552.0,138.2],null,"#0f766e"],["rect",[54.0,217.2,558.0,267.7],"#f3f4f6",null],["hline",[54.0,217.2,558.0,217.2],null,"#d1d5db"],["hline",[54.0,267.7,558.0,267.7],null,"#d1d5db"],["vline",[54.0,217.2,54.0,267.7],null,"#d1d5db"],["vline",[558.0,217.2,558.0,267.7],null,"#d1d5db"],["rect",[54.0,354.7,558.0,485.7],"#f3f4f6",null],["hline",[54.0,354.7,558.0,354.7],null,"#d1d5db"],["hline",[54.0,485.7,558.0,485.7],null,"#d1d5db"],["vline",[54.0,354.7,54.0,485.7],null,"#d1d5db"],["vline",[558.0,354.7,558.0,485.7],null,"#d1d5db"],["rect",[57.0,493.7,555.0,538.7],"#fef3c7",null],["vline",[57.0,493.7,57.0,538.7],null,"#f59e0b"],["rect",[54.0,602.7,558.0,624.7],"#0f766e",null],["rect",[54.0,624.7,558.0,646.7],"#ffffff",null],["rect",[54.0,646.7,558.0,668.7],"#f0fdfa",null],["rect",[54.0,668.7,558.0,690.7],"#ffffff",null],["rect",[54.0,690.7,558.0,712.7],"#f0fdfa",null],["hline",[54.0,602.7,558.0,602.7],null,"#99f6e4"],["vline",[54.0,602.7,54.0,712.7],null,"#99f6e4"],["vline",[558.0,602.7,558.0,712.7],null,"#99f6e4"],["hline",[54.0,712.7,558.0,712.7],null,"#99f6e4"],["hline",[54.0,624.7,558.0,624.7],null,"#99f6e4"],["hline",[54.0,646.7,558.0,646.7],null,"#99f6e4"],["hline",[54.0,668.7,558.0,668.7],null,"#99f6e4"],["hline",[54.0,690.7,558.0,690.7],null,"#99f6e4"],["vline",[146.0,602.7,146.0,712.7],null,"#99f6e4"],["vline",[216.0,602.7,216.0,712.7],null,"#99f6e4"],["vline",[316.0,602.7,316.0,712.7],null,"#99f6e4"]],"lines":[[[54.0,24.4,120.0,36.8],34.0,"802c6d521b9f","Helvetica-Bold",9.0,2],[[488.5,24.3,558.0,36.7],34.0,"e0719a8c6df0","Helvetica",9.0,0],[[293.6,751.4,318.4,762.4],760.0,"a0085032e25a","Helvetica",8.0,0],[[60.0,76.5,398.4,105.4],99.0,"9e2f85a41ff4","Helvetica-Bold",21.0,2],[[60.0,107.2,418.9,122.3],119.0,"cd6b20ec3ce0","Helvetica",11.0,0],[[60.0,153.2,155.9,173.8],169.2,"3dbca386b3bf","Helvetica-Bold",15.0,2],[[60.0,179.4,538.0,193.8],190.7,"e66d093311f2","Helvetica",10.5,0],[[60.0,194.4,196.6,208.8],205.7,"81d48da158e9","Helvetica",10.5,0],[[64.0,225.8,186.4,236.4],233.7,"3bed44c8aa28","Courier",8.5,0],[[64.0,237.3,155.8,247.9],245.2,"c4e22e2a0db3","Courier",8.5,0],[[64.0,248.8,217.0,259.4],256.7,"504bd405d226","Courier",8.5,0],[[60.0,290.7,150.9,311.3],306.7,"8c525ab9fbf6","Helvetica-Bold",15.0,2],[[60.0,316.9,525.1,331.3],328.2,"5a8cd59bb93e","Helvetica",10.5,0],[[60.0,331.9,138.2,346.3],343.2,"1f9311b78475","Helvetica",10.5,0],[[64.0,363.3,252.7,373.9],371.2,"cde5a2c7537a","Courier",8.5,0],[[64.0,386.3,166.0,396.9],394.2,"8a700f5dcb8f","Courier",8.5,0],[[64.0,397.8,262.9,408.4],405.7,"fc14a92979b6","Courier",8.5,0],[[64.0,409.3,359.8,419.9],417.2,"95c9209d83c3","Courier",8.5,0],[[64.0,420.8,69.1,431.4],428.7,"e7064f0b80f6","Courier",8.5,0],[[64.0,443.8,140.5,454.4],451.7,"d92b91a5f332","Courier",8.5,0],[[64.0,455.3,166.0,465.9],463.2,"3cb3f25b2e55","Courier",8.5,0],[[64.0,466.8,257.8,477.4],474.7,"51864e9a04e6","Courier",8.5,0],[[69.0,502.0,531.8,515.1],512.2,"2a31a2f97fee","Helvetica-Bold",9.5,2],[[69.0,515.5,235.3,528.5],525.7,"d193a13a7946","Helvetica",9.5,0],[[60.0,553.7,251.7,574.3],569.7,"029649442fad","Helvetica-Bold",15.0,2],[[60.0,579.9,550.2,594.3],591.2,"c61f31c04cf6","Helvetica",10.5,0],[[61.0,607.1,102.6,618.8],616.2,"f699f295e5ae","Helvetica-Bold",8.5,2],[[153.0,607.1,172.8,618.8],616.2,"3deb74565196","Helvetica-Bold",8.5,2],[[223.0,607.1,251.8,618.8],616.2,"808d7dca8a74","Helvetica-Bold",8.5,2],[[323.0,607.1,369.8,618.8],616.2,"55f8ebc805e6","Helvetica-Bold",8.5,2],[[61.0,630.3,91.6,640.9],638.2,"9f00fad98bad","Courier",8.5,0],[[153.0,630.3,183.6,640.9],638.2,"bb9cf1418089","Courier",8.5,0],[[223.0,630.3,289.3,640.9],638.2,"a400e6003b89","Courier",8.5,0],[[323.0,629.1,462.8,640.7],638.2,"27ff14435d5a","Helvetica",8.5,0],[[61.0,652.3,101.8,662.9],660.2,"3536f72ac0ec","Courier",8.5,0],[[153.0,652.3,198.9,662.9],660.2,"1a05d3c96fdc","Courier",8.5,0],[[223.0,652.3,279.1,662.9],660.2,"722d30dfce0e","Courier",8.5,0],[[323.0,651.1,436.4,662.7],660.2,"f3f4d1802f0c","Helvetica",8.5,0],[[61.0,674.3,81.4,684.9],682.2,"6f2b83563239","Courier",8.5,0],[[153.0,674.3,173.4,684.9],682.2,"e53e8d5300c8","Courier",8.5,0],[[223.0,674.3,253.6,684.9],682.2,"476d9ec701e2","Courier",8.5,0],[[323.0,673.1,424.6,684.7],682.2,"90802863e515","Helvetica",8.5,0],[[61.0,696.3,117.1,706.9],704.2,"85dd687a69f6","Courier",8.5,0],[[153.0,696.3,178.5,706.9],704.2,"685e80366130","Courier",8.5,0],[[223.0,696.3,238.3,706.9],704.2,"e8dc057d3346","Courier",8.5,0],[[323.0,695.1,425.0,706.7],704.2,"3146b0146f07","Helvetica",8.5,0]],"n_blocks":20,"n_draws":30,"n_images":0,"n_lines":46,"n_links":0,"n_spans":47,"size":[612.0,792.0]},{"chars":838,"draws":[["hline",[54.0,40.0,558.0,40.0],null,"#0f766e"],["rect",[54.0,78.0,558.0,100.0],"#ffffff",null],["vline",[54.0,78.0,54.0,100.0],null,"#99f6e4"],["vline",[558.0,78.0,558.0,100.0],null,"#99f6e4"],["hline",[54.0,100.0,558.0,100.0],null,"#99f6e4"],["hline",[54.0,78.0,558.0,78.0],null,"#99f6e4"],["vline",[146.0,78.0,146.0,100.0],null,"#99f6e4"],["vline",[216.0,78.0,216.0,100.0],null,"#99f6e4"],["vline",[316.0,78.0,316.0,100.0],null,"#99f6e4"],["rect",[54.0,198.0,558.0,283.0],"#f3f4f6",null],["hline",[54.0,198.0,558.0,198.0],null,"#d1d5db"],["hline",[54.0,283.0,558.0,283.0],null,"#d1d5db"],["vline",[54.0,198.0,54.0,283.0],null,"#d1d5db"],["vline",[558.0,198.0,558.0,283.0],null,"#d1d5db"],["rect",[57.0,291.0,555.0,322.5],"#fee2e2",null],["vline",[57.0,291.0,57.0,322.5],null,"#dc2626"]],"lines":[[[54.0,24.4,120.0,36.8],34.0,"802c6d521b9f","Helvetica-Bold",9.0,2],[[488.5,24.3,558.0,36.7],34.0,"e0719a8c6df0","Helvetica",9.0,0],[[293.6,751.4,318.4,762.4],760.0,"e78296a9ea92","Helvetica",8.0,0],[[61.0,83.6,106.9,94.2],91.5,"85cd01a28662","Courier",8.5,0],[[153.0,83.6,173.4,94.2],91.5,"5039d155a71c","Courier",8.5,0],[[223.0,83.6,243.4,94.2],91.5,"88b33e4e12f7","Courier",8.5,0],[[323.0,82.4,429.3,94.0],91.5,"6e8dddcef068","Helvetica",8.5,0],[[60.0,103.4,184.7,115.0],112.5,"2358a3d96202","Helvetica",8.5,0],[[60.0,133.9,199.2,154.6],150.0,"b3e27c92b9c8","Helvetica-Bold",15.0,2],[[60.0,160.2,545.6,174.6],171.5,"60d46aae319e","Helvetica",10.5,0],[[60.0,175.2,199.6,189.8],186.5,"0e114dbd29ed","Courier",10.5,0],[[64.0,206.6,227.2,217.2],214.5,"2c1d96f7d723","Courier",8.5,0],[[64.0,218.1,140.5,228.7],226.0,"eb0a5d6927c6","Courier",8.5,0],[[64.0,229.6,375.1,240.2],237.5,"0f298fbb4938","Courier",8.5,0],[[64.0,241.1,283.3,251.7],249.0,"a75844b64edf","Courier",8.5,0],[[64.0,252.6,247.6,263.2],260.5,"7205efff1aea","Courier",8.5,0],[[64.0,264.1,298.6,274.7],272.0,"e200dc4031d7","Courier",8.5,0],[[69.0,299.3,540.2,312.4],309.5,"b330b0d3f2c6","Helvetica-Bold",9.5,2],[[60.0,332.7,164.2,348.5],345.0,"09baaad6ec38","Helvetica-Bold",11.5,2],[[60.0,352.6,66.7,369.1],365.5,"356a192b7913","Helvetica",12.0,0],[[78.0,352.7,307.9,367.1],364.0,"97c7b2698d13","Helvetica",10.5,0],[[60.0,374.6,66.7,391.1],387.5,"da4b9237bacc","Helvetica",12.0,0],[[78.0,374.7,343.5,389.1],386.0,"c060755a6e41","Helvetica",10.5,0],[[60.0,396.6,66.7,413.1],409.5,"77de68daecd8","Helvetica",12.0,0],[[78.0,396.7,359.3,411.1],408.0,"8e9866dc4d24","Helvetica",10.5,0],[[60.0,418.6,66.7,435.1],431.5,"1b6453892473","Helvetica",12.0,0],[[78.0,418.7,292.2,433.1],430.0,"ef57da4d314f","Helvetica",10.5,0]],"n_blocks":13,"n_draws":16,"n_images":0,"n_lines":27,"n_links":0,"n_spans":29,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":1222,"draws":[["hline",[54.0,40.0,558.0,40.0],null,"#0f766e"],["hline",[60.0,138.2,552.0,138.2],null,"#0f766e"],["rect",[54.0,217.2,558.0,267.7],"#f3f4f6",null],["hline",[54.0,217.2,558.0,217.2],null,"#d1d5db"],["hline",[54.0,267.7,558.0,267.7],null,"#d1d5db"],["vline",[54.0,217.2,54.0,267.7],null,"#d1d5db"],["vline",[558.0,217.2,558.0,267.7],null,"#d1d5db"],["rect",[54.0,354.7,558.0,485.7],"#f3f4f6",null],["hline",[54.0,354.7,558.0,354.7],null,"#d1d5db"],["hline",[54.0,485.7,558.0,485.7],null,"#d1d5db"],["vline",[54.0,354.7,54.0,485.7],null,"#d1d5db"],["vline",[558.0,354.7,558.0,485.7],null,"#d1d5db"],["rect",[57.0,493.7,555.0,538.7],"#fef3c7",null],["vline",[57.0,493.7,57.0,538.7],null,"#f59e0b"],["rect",[54.0,602.7,558.0,624.7],"#0f766e",null],["rect",[54.0,624.7,558.0,646.7],"#ffffff",null],["rect",[54.0,646.7,558.0,668.7],"#f0fdfa",null],["rect",[54.0,668.7,558.0,690.7],"#ffffff",null],["rect",[54.0,690.7,558.0,712.7],"#f0fdfa",null],["hline",[54.0,602.7,558.0,602.7],null,"#99f6e4"],["vline",[54.0,602.7,54.0,712.7],null,"#99f6e4"],["vline",[558.0,602.7,558.0,712.7],null,"#99f6e4"],["hline",[54.0,712.7,558.0,712.7],null,"#99f6e4"],["hline",[54.0,624.7,558.0,624.7],null,"#99f6e4"],["hline",[54.0,646.7,558.0,646.7],null,"#99f6e4"],["hline",[54.0,668.7,558.0,668.7],null,"#99f6e4"],["hline",[54.0,690.7,558.0,690.7],null,"#99f6e4"],["vline",[146.0,602.7,146.0,712.7],null,"#99f6e4"],["vline",[216.0,602.7,216.0,712.7],null,"#99f6e4"],["vline",[316.0,602.7,316.0,712.7],null,"#99f6e4"]],"lines":[[[54.0,24.4,120.0,36.8],34.0,"802c6d521b9f","Helvetica-Bold",9.0,2],[[488.5,24.3,558.0,36.7],34.0,"e0719a8c6df0","Helvetica",9.0,0],[[293.6,751.4,318.4,762.4],760.0,"a0085032e25a","Helvetica",8.0,0],[[60.0,76.5,398.4,105.4],99.0,"9e2f85a41ff4","Helvetica-Bold",21.0,2],[[60.0,107.2,418.9,122.3],119.0,"cd6b20ec3ce0","Helvetica",11.0,0],[[60.0,153.2,155.9,173.8],169.2,"3dbca386b3bf","Helvetica-Bold",15.0,2],[[60.0,179.4,538.0,193.8],190.7,"e66d093311f2","Helvetica",10.5,0],[[60.0,194.4,196.6,208.8],205.7,"81d48da158e9","Helvetica",10.5,0],[[64.0,225.8,186.4,236.4],233.7,"3bed44c8aa28","Courier",8.5,0],[[64.0,237.3,155.8,247.9],245.2,"c4e22e2a0db3","Courier",8.5,0],[[64.0,248.8,217.0,259.4],256.7,"504bd405d226","Courier",8.5,0],[[60.0,290.7,150.9,311.3],306.7,"8c525ab9fbf6","Helvetica-Bold",15.0,2],[[60.0,316.9,525.1,331.3],328.2,"5a8cd59bb93e","Helvetica",10.5,0],[[60.0,331.9,138.2,346.3],343.2,"1f9311b78475","Helvetica",10.5,0],[[64.0,363.3,252.7,373.9],371.2,"cde5a2c7537a","Courier",8.5,0],[[64.0,386.3,166.0,396.9],394.2,"8a700f5dcb8f","Courier",8.5,0],[[64.0,397.8,262.9,408.4],405.7,"fc14a92979b6","Courier",8.5,0],[[64.0,409.3,359.8,419.9],417.2,"95c9209d83c3","Courier",8.5,0],[[64.0,420.8,69.1,431.4],428.7,"e7064f0b80f6","Courier",8.5,0],[[64.0,443.8,140.5,454.4],451.7,"d92b91a5f332","Courier",8.5,0],[[64.0,455.3,166.0,465.9],463.2,"3cb3f25b2e55","Courier",8.5,0],[[64.0,466.8,257.8,477.4],474.7,"51864e9a04e6","Courier",8.5,0],[[69.0,502.0,531.8,515.1],512.2,"2a31a2f97fee","Helvetica-Bold",9.5,2],[[69.0,515.5,235.3,528.5],525.7,"d193a13a7946","Helvetica",9.5,0],[[60.0,553.7,251.7,574.3],569.7,"029649442fad","Helvetica-Bold",15.0,2],[[60.0,579.9,550.2,594.3],591.2,"c61f31c04cf6","Helvetica",10.5,0],[[61.0,607.1,102.6,618.8],616.2,"f699f295e5ae","Helvetica-Bold",8.5,2],[[153.0,607.1,172.8,618.8],616.2,"3deb74565196","Helvetica-Bold",8.5,2],[[223.0,607.1,251.8,618.8],616.2,"808d7dca8a74","Helvetica-Bold",8.5,2],[[323.0,607.1,369.8,618.8],616.2,"55f8ebc805e6","Helvetica-Bold",8.5,2],[[61.0,630.3,91.6,640.9],638.2,"9f00fad98bad","Courier",8.5,0],[[153.0,630.3,183.6,640.9],638.2,"bb9cf1418089","Courier",8.5,0],[[223.0,630.3,289.3,640.9],638.2,"a400e6003b89","Courier",8.5,0],[[323.0,629.1,462.8,640.7],638.2,"27ff14435d5a","Helvetica",8.5,0],[[61.0,652.3,101.8,662.9],660.2,"3536f72ac0ec","Courier",8.5,0],[[153.0,652.3,198.9,662.9],660.2,"1a05d3c96fdc","Courier",8.5,0],[[223.0,652.3,279.1,662.9],660.2,"722d30dfce0e","Courier",8.5,0],[[323.0,651.1,436.4,662.7],660.2,"f3f4d1802f0c","Helvetica",8.5,0],[[61.0,674.3,81.4,684.9],682.2,"6f2b83563239","Courier",8.5,0],[[153.0,674.3,173.4,684.9],682.2,"e53e8d5300c8","Courier",8.5,0],[[223.0,674.3,253.6,684.9],682.2,"476d9ec701e2","Courier",8.5,0],[[323.0,673.1,424.6,684.7],682.2,"90802863e515","Helvetica",8.5,0],[[61.0,696.3,117.1,706.9],704.2,"85dd687a69f6","Courier",8.5,0],[[153.0,696.3,178.5,706.9],704.2,"685e80366130","Courier",8.5,0],[[223.0,696.3,238.3,706.9],704.2,"e8dc057d3346","Courier",8.5,0],[[323.0,695.1,425.0,706.7],704.2,"3146b0146f07","Helvetica",8.5,0]],"n_blocks":20,"n_draws":30,"n_images":0,"n_lines":46,"n_links":0,"n_spans":47,"size":[612.0,792.0]},{"chars":838,"draws":[["hline",[54.0,40.0,558.0,40.0],null,"#0f766e"],["rect",[54.0,78.0,558.0,100.0],"#ffffff",null],["vline",[54.0,78.0,54.0,100.0],null,"#99f6e4"],["vline",[558.0,78.0,558.0,100.0],null,"#99f6e4"],["hline",[54.0,100.0,558.0,100.0],null,"#99f6e4"],["hline",[54.0,78.0,558.0,78.0],null,"#99f6e4"],["vline",[146.0,78.0,146.0,100.0],null,"#99f6e4"],["vline",[216.0,78.0,216.0,100.0],null,"#99f6e4"],["vline",[316.0,78.0,316.0,100.0],null,"#99f6e4"],["rect",[54.0,198.0,558.0,283.0],"#f3f4f6",null],["hline",[54.0,198.0,558.0,198.0],null,"#d1d5db"],["hline",[54.0,283.0,558.0,283.0],null,"#d1d5db"],["vline",[54.0,198.0,54.0,283.0],null,"#d1d5db"],["vline",[558.0,198.0,558.0,283.0],null,"#d1d5db"],["rect",[57.0,291.0,555.0,322.5],"#fee2e2",null],["vline",[57.0,291.0,57.0,322.5],null,"#dc2626"]],"lines":[[[54.0,24.4,120.0,36.8],34.0,"802c6d521b9f","Helvetica-Bold",9.0,2],[[488.5,24.3,558.0,36.7],34.0,"e0719a8c6df0","Helvetica",9.0,0],[[293.6,751.4,318.4,762.4],760.0,"e78296a9ea92","Helvetica",8.0,0],[[61.0,83.6,106.9,94.2],91.5,"85cd01a28662","Courier",8.5,0],[[153.0,83.6,173.4,94.2],91.5,"5039d155a71c","Courier",8.5,0],[[223.0,83.6,243.4,94.2],91.5,"88b33e4e12f7","Courier",8.5,0],[[323.0,82.4,429.3,94.0],91.5,"6e8dddcef068","Helvetica",8.5,0],[[60.0,103.4,184.7,115.0],112.5,"2358a3d96202","Helvetica",8.5,0],[[60.0,133.9,199.2,154.6],150.0,"b3e27c92b9c8","Helvetica-Bold",15.0,2],[[60.0,160.2,545.6,174.6],171.5,"60d46aae319e","Helvetica",10.5,0],[[60.0,175.2,199.6,189.8],186.5,"0e114dbd29ed","Courier",10.5,0],[[64.0,206.6,227.2,217.2],214.5,"2c1d96f7d723","Courier",8.5,0],[[64.0,218.1,140.5,228.7],226.0,"eb0a5d6927c6","Courier",8.5,0],[[64.0,229.6,375.1,240.2],237.5,"0f298fbb4938","Courier",8.5,0],[[64.0,241.1,283.3,251.7],249.0,"a75844b64edf","Courier",8.5,0],[[64.0,252.6,247.6,263.2],260.5,"7205efff1aea","Courier",8.5,0],[[64.0,264.1,298.6,274.7],272.0,"e200dc4031d7","Courier",8.5,0],[[69.0,299.3,540.2,312.4],309.5,"b330b0d3f2c6","Helvetica-Bold",9.5,2],[[60.0,332.7,164.2,348.5],345.0,"09baaad6ec38","Helvetica-Bold",11.5,2],[[60.0,352.6,66.7,369.1],365.5,"356a192b7913","Helvetica",12.0,0],[[78.0,352.7,307.9,367.1],364.0,"97c7b2698d13","Helvetica",10.5,0],[[60.0,374.6,66.7,391.1],387.5,"da4b9237bacc","Helvetica",12.0,0],[[78.0,374.7,343.5,389.1],386.0,"c060755a6e41","Helvetica",10.5,0],[[60.0,396.6,66.7,413.1],409.5,"77de68daecd8","Helvetica",12.0,0],[[78.0,396.7,359.3,411.1],408.0,"8e9866dc4d24","Helvetica",10.5,0],[[60.0,418.6,66.7,435.1],431.5,"1b6453892473","Helvetica",12.0,0],[[78.0,418.7,292.2,433.1],430.0,"ef57da4d314f","Helvetica",10.5,0]],"n_blocks":13,"n_draws":16,"n_images":0,"n_lines":27,"n_links":0,"n_spans":29,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/04_exec_brief.json b/testkit/golden/04_exec_brief.json index ad11a56..13ad8f4 100644 --- a/testkit/golden/04_exec_brief.json +++ b/testkit/golden/04_exec_brief.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":909,"draws":[["rect",[0.0,0.0,612.0,130.0],"#312e81",null],["rect",[0.0,130.0,612.0,138.0],"#4f46e5",null],["rect",[66.0,242.0,226.0,323.0],"#4f46e5",null],["rect",[226.0,242.0,386.0,323.0],"#7c3aed",null],["rect",[386.0,242.0,546.0,323.0],"#6d28d9",null],["vline",[57.0,383.0,57.0,446.0],null,"#7c3aed"],["hline",[100.0,700.0,470.0,700.0],null,"#9ca3af"],["vline",[100.0,566.0,100.0,700.0],null,"#9ca3af"],["complex",[100.0,588.3,470.0,676.9],null,"#4f46e5"],["curve",[97.4,674.3,102.6,679.5],"#4f46e5",null],["curve",[150.3,660.8,155.5,666.0],"#4f46e5",null],["curve",[203.1,664.7,208.3,669.9],"#4f46e5",null],["curve",[256.0,643.5,261.2,648.7],"#4f46e5",null],["curve",[308.8,628.1,314.0,633.3],"#4f46e5",null],["curve",[361.7,633.9,366.9,639.1],"#4f46e5",null],["curve",[414.5,606.9,419.7,612.1],"#4f46e5",null],["curve",[467.4,585.7,472.6,590.9],"#4f46e5",null]],"lines":[[[54.0,49.4,395.3,81.1],74.0,"5dcfa5715b16","Helvetica-Bold",23.0,2],[[54.0,86.2,274.1,101.3],98.0,"aae43b043648","Helvetica",11.0,0],[[54.0,749.4,149.1,760.4],758.0,"5a5ac0bcdd3d","Helvetica",8.0,0],[[542.4,749.4,558.0,760.4],758.0,"72d1b7f3d6c4","Helvetica",8.0,0],[[60.0,170.9,157.3,192.3],187.5,"3cd90abc0f70","Helvetica-Bold",15.5,2],[[60.0,197.2,552.0,211.6],208.5,"ce01cd993e61","Helvetica",10.5,0],[[60.0,212.2,382.2,226.6],223.5,"08126eb2b5db","Helvetica",10.5,0],[[122.0,254.3,170.0,287.4],280.0,"5bc250463169","Helvetica-Bold",24.0,2],[[94.5,286.4,197.5,298.0],295.5,"1657fc691ef7","Helvetica",8.5,0],[[126.2,297.4,165.8,309.0],306.5,"90a8834de763","Helvetica",8.5,0],[[282.3,259.8,329.7,292.9],285.5,"bb9ad7c39a1f","Helvetica-Bold",24.0,2],[[246.7,291.9,365.3,303.5],301.0,"4251f095cc92","Helvetica",8.5,0],[[442.0,259.8,490.0,292.9],285.5,"4e27935b7594","Helvetica-Bold",24.0,2],[[397.0,291.9,535.0,303.5],301.0,"e69a8df35407","Helvetica",8.5,0],[[60.0,336.2,552.0,350.6],347.5,"5aa081c160f8","Helvetica",10.5,0],[[60.0,351.2,501.8,365.6],362.5,"c8ea045f9d29","Helvetica",10.5,0],[[71.0,386.1,514.9,403.7],400.0,"f15dfe75140e","Helvetica-Oblique",13.0,1],[[71.0,404.1,152.2,421.7],418.0,"1717781aa3fe","Helvetica-Oblique",13.0,1],[[71.0,426.3,264.6,438.7],436.0,"0b4421f2f22c","Helvetica",9.0,0],[[60.0,474.9,228.0,496.3],491.5,"9fa99b4fdb3a","Helvetica-Bold",15.5,2],[[60.0,501.2,552.0,515.6],512.5,"2c9b7fd3842e","Helvetica",10.5,0],[[60.0,516.2,103.8,530.6],527.5,"607e09e1fb75","Helvetica",10.5,0],[[95.0,703.9,105.0,714.2],712.0,"db5623b4d098","Helvetica",7.5,0],[[147.9,703.9,157.9,714.2],712.0,"d37d0956d096","Helvetica",7.5,0],[[200.7,703.9,210.7,714.2],712.0,"e1d6b67ef5b1","Helvetica",7.5,0],[[253.6,703.9,263.6,714.2],712.0,"7b87e6472ed7","Helvetica",7.5,0],[[306.4,703.9,316.4,714.2],712.0,"db5623b4d098","Helvetica",7.5,0],[[359.3,703.9,369.3,714.2],712.0,"d37d0956d096","Helvetica",7.5,0],[[412.1,703.9,422.1,714.2],712.0,"e1d6b67ef5b1","Helvetica",7.5,0],[[465.0,703.9,475.0,714.2],712.0,"7b87e6472ed7","Helvetica",7.5,0],[[100.0,550.4,280.0,562.8],560.0,"10459a218323","Helvetica-Bold",9.0,2]],"n_blocks":18,"n_draws":17,"n_images":0,"n_lines":31,"n_links":0,"n_spans":31,"size":[612.0,792.0]},{"chars":310,"draws":[["hline",[248.5,198.8,413.7,198.8],null,"#4f46e5"]],"lines":[[[54.0,749.4,149.1,760.4],758.0,"5a5ac0bcdd3d","Helvetica",8.0,0],[[542.4,749.4,558.0,760.4],758.0,"64cb6682a054","Helvetica",8.0,0],[[60.0,86.9,267.6,108.3],103.5,"2791bf232794","Helvetica-Bold",15.5,2],[[60.0,113.1,64.2,129.6],126.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,113.2,364.6,127.6],124.5,"46a42694a8d6","Helvetica",10.5,0],[[60.0,136.1,64.2,152.6],149.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,136.2,363.4,150.6],147.5,"e0dffb44a2f0","Helvetica",10.5,0],[[60.0,159.1,64.2,175.6],172.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,159.2,346.5,173.6],170.5,"c9ecf27c3220","Helvetica",10.5,0],[[60.0,186.2,416.6,200.6],197.5,"14f97be5d64e","Helvetica",10.5,0]],"n_blocks":6,"n_draws":1,"n_images":0,"n_lines":10,"n_links":1,"n_spans":12,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":909,"draws":[["rect",[0.0,0.0,612.0,130.0],"#312e81",null],["rect",[0.0,130.0,612.0,138.0],"#4f46e5",null],["rect",[66.0,242.0,226.0,323.0],"#4f46e5",null],["rect",[226.0,242.0,386.0,323.0],"#7c3aed",null],["rect",[386.0,242.0,546.0,323.0],"#6d28d9",null],["vline",[57.0,383.0,57.0,446.0],null,"#7c3aed"],["hline",[100.0,700.0,470.0,700.0],null,"#9ca3af"],["vline",[100.0,566.0,100.0,700.0],null,"#9ca3af"],["complex",[100.0,588.3,470.0,676.9],null,"#4f46e5"],["curve",[97.4,674.3,102.6,679.5],"#4f46e5",null],["curve",[150.3,660.8,155.5,666.0],"#4f46e5",null],["curve",[203.1,664.7,208.3,669.9],"#4f46e5",null],["curve",[256.0,643.5,261.2,648.7],"#4f46e5",null],["curve",[308.8,628.1,314.0,633.3],"#4f46e5",null],["curve",[361.7,633.9,366.9,639.1],"#4f46e5",null],["curve",[414.5,606.9,419.7,612.1],"#4f46e5",null],["curve",[467.4,585.7,472.6,590.9],"#4f46e5",null]],"lines":[[[54.0,49.4,395.3,81.1],74.0,"5dcfa5715b16","Helvetica-Bold",23.0,2],[[54.0,86.2,274.1,101.3],98.0,"aae43b043648","Helvetica",11.0,0],[[54.0,749.4,149.1,760.4],758.0,"5a5ac0bcdd3d","Helvetica",8.0,0],[[542.4,749.4,558.0,760.4],758.0,"72d1b7f3d6c4","Helvetica",8.0,0],[[60.0,170.9,157.3,192.3],187.5,"3cd90abc0f70","Helvetica-Bold",15.5,2],[[60.0,197.2,552.0,211.6],208.5,"ce01cd993e61","Helvetica",10.5,0],[[60.0,212.2,382.2,226.6],223.5,"08126eb2b5db","Helvetica",10.5,0],[[122.0,254.3,170.0,287.4],280.0,"5bc250463169","Helvetica-Bold",24.0,2],[[94.5,286.4,197.5,298.0],295.5,"1657fc691ef7","Helvetica",8.5,0],[[126.2,297.4,165.8,309.0],306.5,"90a8834de763","Helvetica",8.5,0],[[282.3,259.8,329.7,292.9],285.5,"bb9ad7c39a1f","Helvetica-Bold",24.0,2],[[246.7,291.9,365.3,303.5],301.0,"4251f095cc92","Helvetica",8.5,0],[[442.0,259.8,490.0,292.9],285.5,"4e27935b7594","Helvetica-Bold",24.0,2],[[397.0,291.9,535.0,303.5],301.0,"e69a8df35407","Helvetica",8.5,0],[[60.0,336.2,552.0,350.6],347.5,"5aa081c160f8","Helvetica",10.5,0],[[60.0,351.2,501.8,365.6],362.5,"c8ea045f9d29","Helvetica",10.5,0],[[71.0,386.1,514.9,403.7],400.0,"f15dfe75140e","Helvetica-Oblique",13.0,1],[[71.0,404.1,152.2,421.7],418.0,"1717781aa3fe","Helvetica-Oblique",13.0,1],[[71.0,426.3,264.6,438.7],436.0,"0b4421f2f22c","Helvetica",9.0,0],[[60.0,474.9,228.0,496.3],491.5,"9fa99b4fdb3a","Helvetica-Bold",15.5,2],[[60.0,501.2,552.0,515.6],512.5,"2c9b7fd3842e","Helvetica",10.5,0],[[60.0,516.2,103.8,530.6],527.5,"607e09e1fb75","Helvetica",10.5,0],[[95.0,703.9,105.0,714.2],712.0,"db5623b4d098","Helvetica",7.5,0],[[147.9,703.9,157.9,714.2],712.0,"d37d0956d096","Helvetica",7.5,0],[[200.7,703.9,210.7,714.2],712.0,"e1d6b67ef5b1","Helvetica",7.5,0],[[253.6,703.9,263.6,714.2],712.0,"7b87e6472ed7","Helvetica",7.5,0],[[306.4,703.9,316.4,714.2],712.0,"db5623b4d098","Helvetica",7.5,0],[[359.3,703.9,369.3,714.2],712.0,"d37d0956d096","Helvetica",7.5,0],[[412.1,703.9,422.1,714.2],712.0,"e1d6b67ef5b1","Helvetica",7.5,0],[[465.0,703.9,475.0,714.2],712.0,"7b87e6472ed7","Helvetica",7.5,0],[[100.0,550.4,280.0,562.8],560.0,"10459a218323","Helvetica-Bold",9.0,2]],"n_blocks":18,"n_draws":17,"n_images":0,"n_lines":31,"n_links":0,"n_spans":31,"size":[612.0,792.0]},{"chars":310,"draws":[["hline",[248.5,198.8,413.7,198.8],null,"#4f46e5"]],"lines":[[[54.0,749.4,149.1,760.4],758.0,"5a5ac0bcdd3d","Helvetica",8.0,0],[[542.4,749.4,558.0,760.4],758.0,"64cb6682a054","Helvetica",8.0,0],[[60.0,86.9,267.6,108.3],103.5,"2791bf232794","Helvetica-Bold",15.5,2],[[60.0,113.1,64.2,129.6],126.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,113.2,364.6,127.6],124.5,"46a42694a8d6","Helvetica",10.5,0],[[60.0,136.1,64.2,152.6],149.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,136.2,363.4,150.6],147.5,"e0dffb44a2f0","Helvetica",10.5,0],[[60.0,159.1,64.2,175.6],172.0,"ecf727ea048d","Helvetica",12.0,0],[[78.0,159.2,346.5,173.6],170.5,"c9ecf27c3220","Helvetica",10.5,0],[[60.0,186.2,416.6,200.6],197.5,"14f97be5d64e","Helvetica",10.5,0]],"n_blocks":6,"n_draws":1,"n_images":0,"n_lines":10,"n_links":1,"n_spans":12,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/05_memo.json b/testkit/golden/05_memo.json index 1ded4ee..441861c 100644 --- a/testkit/golden/05_memo.json +++ b/testkit/golden/05_memo.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":501,"draws":[["hline",[78.0,168.8,534.0,168.8],null,"#d1d5db"]],"lines":[[[78.0,77.0,204.0,96.3],92.0,"7445cfebc446","Helvetica-Bold",14.0,2],[[78.0,105.2,155.8,119.0],116.0,"db1c625699b9","Helvetica",10.0,0],[[78.0,122.2,158.0,136.0],133.0,"1d429d91cca2","Helvetica",10.0,0],[[78.0,139.2,164.2,153.0],150.0,"0d6e8be56212","Helvetica",10.0,0],[[78.0,180.0,526.2,194.4],191.3,"ad01c44420b1","Helvetica",10.5,0],[[78.0,195.0,501.7,209.4],206.3,"0c3255f5a1c5","Helvetica",10.5,0],[[78.0,218.0,245.5,232.4],229.3,"9669198a4302","Helvetica",10.5,0],[[78.0,240.9,82.2,257.4],253.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,241.0,316.0,255.4],252.3,"50fe01bed540","Helvetica",10.5,0],[[78.0,263.9,82.2,280.4],276.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,264.0,302.0,278.4],275.3,"7dc76eb36a0b","Helvetica",10.5,0],[[78.0,286.9,82.2,303.4],299.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,287.0,243.1,301.4],298.3,"cd8510d7c6ea","Helvetica",10.5,0],[[78.0,310.0,530.3,324.4],321.3,"02b6d14c7c39","Helvetica",10.5,0]],"n_blocks":10,"n_draws":1,"n_images":0,"n_lines":14,"n_links":0,"n_spans":14,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":501,"draws":[["hline",[78.0,168.8,534.0,168.8],null,"#d1d5db"]],"lines":[[[78.0,77.0,204.0,96.3],92.0,"7445cfebc446","Helvetica-Bold",14.0,2],[[78.0,105.2,155.8,119.0],116.0,"db1c625699b9","Helvetica",10.0,0],[[78.0,122.2,158.0,136.0],133.0,"1d429d91cca2","Helvetica",10.0,0],[[78.0,139.2,164.2,153.0],150.0,"0d6e8be56212","Helvetica",10.0,0],[[78.0,180.0,526.2,194.4],191.3,"ad01c44420b1","Helvetica",10.5,0],[[78.0,195.0,501.7,209.4],206.3,"0c3255f5a1c5","Helvetica",10.5,0],[[78.0,218.0,245.5,232.4],229.3,"9669198a4302","Helvetica",10.5,0],[[78.0,240.9,82.2,257.4],253.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,241.0,316.0,255.4],252.3,"50fe01bed540","Helvetica",10.5,0],[[78.0,263.9,82.2,280.4],276.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,264.0,302.0,278.4],275.3,"7dc76eb36a0b","Helvetica",10.5,0],[[78.0,286.9,82.2,303.4],299.8,"ecf727ea048d","Helvetica",12.0,0],[[96.0,287.0,243.1,301.4],298.3,"cd8510d7c6ea","Helvetica",10.5,0],[[78.0,310.0,530.3,324.4],321.3,"02b6d14c7c39","Helvetica",10.5,0]],"n_blocks":10,"n_draws":1,"n_images":0,"n_lines":14,"n_links":0,"n_spans":14,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/f1_fpdf_brief.json b/testkit/golden/f1_fpdf_brief.json index 84ef037..9ff9c45 100644 --- a/testkit/golden/f1_fpdf_brief.json +++ b/testkit/golden/f1_fpdf_brief.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":759,"draws":[["rect",[28.4,132.3,138.4,148.3],null,"#000000"],["rect",[138.4,132.3,248.4,148.3],null,"#000000"],["rect",[248.4,132.3,358.4,148.3],null,"#000000"],["rect",[28.4,148.3,138.4,164.3],null,"#000000"],["rect",[138.4,148.3,248.4,164.3],null,"#000000"],["rect",[248.4,148.3,358.4,164.3],null,"#000000"],["rect",[28.4,164.3,138.4,180.3],null,"#000000"],["rect",[138.4,164.3,248.4,180.3],null,"#000000"],["rect",[248.4,164.3,358.4,180.3],null,"#000000"],["rect",[28.4,180.3,138.4,196.3],null,"#000000"],["rect",[138.4,180.3,248.4,196.3],null,"#000000"],["rect",[248.4,180.3,358.4,196.3],null,"#000000"]],"lines":[[[31.2,26.5,172.2,51.3],45.8,"493a60e54c08","Helvetica-Bold",18.0,2],[[31.2,51.4,580.8,65.5],62.5,"684c48ecf7f9","Times-Roman",10.5,0],[[31.2,65.4,580.8,79.5],76.5,"25e55136eab9","Times-Roman",10.5,0],[[31.2,79.4,580.8,93.5],90.5,"67070070e06c","Times-Roman",10.5,0],[[31.2,93.4,354.9,107.5],104.5,"bafc15e53056","Times-Roman",10.5,0],[[31.2,114.1,99.2,130.6],127.0,"fd7a8aac9308","Helvetica-Bold",12.0,2],[[31.2,133.4,59.7,145.7],143.0,"0f217179940c","Helvetica",9.0,0],[[141.2,133.4,166.7,145.7],143.0,"bae7d5be7082","Helvetica",9.0,0],[[251.2,133.4,277.7,145.7],143.0,"89ff31225c5f","Helvetica",9.0,0],[[31.2,149.4,43.7,161.7],159.0,"3feda0153eee","Helvetica",9.0,0],[[141.2,149.4,164.2,161.7],159.0,"bc74f4f071a5","Helvetica",9.0,0],[[251.2,149.4,282.7,161.7],159.0,"d6685fc69dd3","Helvetica",9.0,0],[[31.2,165.4,56.7,177.7],175.0,"269e6e46e9db","Helvetica",9.0,0],[[141.2,165.4,166.7,177.7],175.0,"ef4bfd1e6d27","Helvetica",9.0,0],[[251.2,165.4,282.7,177.7],175.0,"16a8af0ac00c","Helvetica",9.0,0],[[31.2,181.4,55.7,193.7],191.0,"2b7880713449","Helvetica",9.0,0],[[141.2,181.4,164.2,193.7],191.0,"bc74f4f071a5","Helvetica",9.0,0],[[251.2,181.4,283.2,193.7],191.0,"96888d5c791b","Helvetica",9.0,0],[[31.2,203.4,580.8,217.5],214.5,"67070070e06c","Times-Roman",10.5,0],[[31.2,217.4,354.9,231.5],228.5,"bafc15e53056","Times-Roman",10.5,0]],"n_blocks":8,"n_draws":12,"n_images":0,"n_lines":20,"n_links":0,"n_spans":20,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":759,"draws":[["rect",[28.4,132.3,138.4,148.3],null,"#000000"],["rect",[138.4,132.3,248.4,148.3],null,"#000000"],["rect",[248.4,132.3,358.4,148.3],null,"#000000"],["rect",[28.4,148.3,138.4,164.3],null,"#000000"],["rect",[138.4,148.3,248.4,164.3],null,"#000000"],["rect",[248.4,148.3,358.4,164.3],null,"#000000"],["rect",[28.4,164.3,138.4,180.3],null,"#000000"],["rect",[138.4,164.3,248.4,180.3],null,"#000000"],["rect",[248.4,164.3,358.4,180.3],null,"#000000"],["rect",[28.4,180.3,138.4,196.3],null,"#000000"],["rect",[138.4,180.3,248.4,196.3],null,"#000000"],["rect",[248.4,180.3,358.4,196.3],null,"#000000"]],"lines":[[[31.2,26.5,172.2,51.3],45.8,"493a60e54c08","Helvetica-Bold",18.0,2],[[31.2,51.4,580.8,65.5],62.5,"684c48ecf7f9","Times-Roman",10.5,0],[[31.2,65.4,580.8,79.5],76.5,"25e55136eab9","Times-Roman",10.5,0],[[31.2,79.4,580.8,93.5],90.5,"67070070e06c","Times-Roman",10.5,0],[[31.2,93.4,354.9,107.5],104.5,"bafc15e53056","Times-Roman",10.5,0],[[31.2,114.1,99.2,130.6],127.0,"fd7a8aac9308","Helvetica-Bold",12.0,2],[[31.2,133.4,59.7,145.7],143.0,"0f217179940c","Helvetica",9.0,0],[[141.2,133.4,166.7,145.7],143.0,"bae7d5be7082","Helvetica",9.0,0],[[251.2,133.4,277.7,145.7],143.0,"89ff31225c5f","Helvetica",9.0,0],[[31.2,149.4,43.7,161.7],159.0,"3feda0153eee","Helvetica",9.0,0],[[141.2,149.4,164.2,161.7],159.0,"bc74f4f071a5","Helvetica",9.0,0],[[251.2,149.4,282.7,161.7],159.0,"d6685fc69dd3","Helvetica",9.0,0],[[31.2,165.4,56.7,177.7],175.0,"269e6e46e9db","Helvetica",9.0,0],[[141.2,165.4,166.7,177.7],175.0,"ef4bfd1e6d27","Helvetica",9.0,0],[[251.2,165.4,282.7,177.7],175.0,"16a8af0ac00c","Helvetica",9.0,0],[[31.2,181.4,55.7,193.7],191.0,"2b7880713449","Helvetica",9.0,0],[[141.2,181.4,164.2,193.7],191.0,"bc74f4f071a5","Helvetica",9.0,0],[[251.2,181.4,283.2,193.7],191.0,"96888d5c791b","Helvetica",9.0,0],[[31.2,203.4,580.8,217.5],214.5,"67070070e06c","Times-Roman",10.5,0],[[31.2,217.4,354.9,231.5],228.5,"bafc15e53056","Times-Roman",10.5,0]],"n_blocks":8,"n_draws":12,"n_images":0,"n_lines":20,"n_links":0,"n_spans":20,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden/r1_reportlab_report.json b/testkit/golden/r1_reportlab_report.json index f05e280..5661b22 100644 --- a/testkit/golden/r1_reportlab_report.json +++ b/testkit/golden/r1_reportlab_report.json @@ -1 +1 @@ -{"backend":"pymupdf","pages":[{"chars":1211,"draws":[["rect",[111.0,342.6,501.0,360.6],"#123a5e",null],["rect",[111.0,360.6,501.0,378.6],"#ffffff",null],["rect",[111.0,378.6,501.0,396.6],"#f2f5f8",null],["rect",[111.0,396.6,501.0,414.6],"#ffffff",null],["hline",[111.0,342.6,501.0,342.6],null,"#c9d3dd"],["hline",[111.0,414.6,501.0,414.6],null,"#c9d3dd"],["vline",[111.0,342.6,111.0,414.6],null,"#c9d3dd"],["vline",[501.0,342.6,501.0,414.6],null,"#c9d3dd"],["hline",[111.0,360.6,501.0,360.6],null,"#c9d3dd"],["hline",[111.0,378.6,501.0,378.6],null,"#c9d3dd"],["hline",[111.0,396.6,501.0,396.6],null,"#c9d3dd"],["vline",[201.0,342.6,201.0,414.6],null,"#c9d3dd"],["vline",[311.0,342.6,311.0,414.6],null,"#c9d3dd"],["vline",[411.0,342.6,411.0,414.6],null,"#c9d3dd"]],"lines":[[[70.8,62.3,292.9,87.1],81.6,"d2816682c720","Helvetica-Bold",18.0,2],[[70.8,96.7,161.1,114.6],110.6,"034824a67fcb","Helvetica-Bold",13.0,2],[[70.8,121.0,541.2,135.1],132.1,"59a123486b81","Times-Roman",10.5,0],[[70.8,135.5,541.2,149.6],146.6,"896f9a43e7a7","Times-Roman",10.5,0],[[70.8,150.0,541.2,164.1],161.1,"0a6262d12866","Times-Roman",10.5,0],[[70.8,164.5,541.2,178.6],175.6,"2d7718ffcc3b","Times-Roman",10.5,0],[[70.8,179.0,178.1,193.1],190.1,"ac38c3dfad58","Times-Roman",10.5,0],[[70.8,205.2,139.4,223.1],219.1,"1356caacb628","Helvetica-Bold",13.0,2],[[70.8,229.5,541.2,243.6],240.6,"aa3f9a55882b","Times-Roman",10.5,0],[[70.8,244.0,479.4,258.1],255.1,"fbf4ab57dd63","Times-Roman",10.5,0],[[70.8,266.2,75.0,282.7],279.1,"ecf727ea048d","Helvetica",12.0,0],[[88.8,266.5,233.8,280.6],277.6,"0279a96eb938","Times-Roman",10.5,0],[[70.8,288.7,75.0,305.2],301.6,"ecf727ea048d","Helvetica",12.0,0],[[88.8,289.0,283.6,303.1],300.1,"35c7b7ad9032","Times-Roman",10.5,0],[[70.8,311.2,75.0,327.7],324.1,"ecf727ea048d","Helvetica",12.0,0],[[88.8,311.5,301.7,325.6],322.6,"56998157b521","Times-Roman",10.5,0],[[117.0,345.0,142.5,357.4],354.6,"712da99c9c7f","Helvetica-Bold",9.0,2],[[207.0,345.0,240.0,357.4],354.6,"c4d8fa69b55a","Helvetica-Bold",9.0,2],[[317.0,345.0,349.5,357.4],354.6,"441bda6cd856","Helvetica-Bold",9.0,2],[[417.0,345.0,463.5,357.4],354.6,"5f5a042355ba","Helvetica-Bold",9.0,2],[[117.0,362.9,134.5,375.3],372.6,"10c69dacbcac","Helvetica",9.0,0],[[265.0,362.9,305.0,375.3],372.6,"a851550cdfa4","Helvetica",9.0,0],[[372.5,362.9,405.0,375.3],372.6,"c5d5b422446e","Helvetica",9.0,0],[[480.0,362.9,495.0,375.3],372.6,"4dea1daedbe9","Helvetica",9.0,0],[[117.0,380.9,134.5,393.3],390.6,"2a7ab28611c8","Helvetica",9.0,0],[[265.0,380.9,305.0,393.3],390.6,"7452c7c8588d","Helvetica",9.0,0],[[365.0,380.9,405.0,393.3],390.6,"a470d6c301ef","Helvetica",9.0,0],[[480.0,380.9,495.0,393.3],390.6,"01592d51db5a","Helvetica",9.0,0],[[117.0,398.9,134.5,411.3],408.6,"c72dd2f011c6","Helvetica",9.0,0],[[265.0,398.9,305.0,411.3],408.6,"b3eb86862c20","Helvetica",9.0,0],[[377.5,398.9,405.0,411.3],408.6,"8fcf662353a5","Helvetica",9.0,0],[[480.0,398.9,495.0,411.3],408.6,"8bd7954c40c1","Helvetica",9.0,0],[[70.8,435.7,151.0,453.6],449.6,"ddafdd979564","Helvetica-Bold",13.0,2],[[70.8,460.0,541.2,474.1],471.1,"59a123486b81","Times-Roman",10.5,0],[[70.8,474.5,541.2,488.6],485.6,"896f9a43e7a7","Times-Roman",10.5,0],[[70.8,489.0,208.4,503.1],500.1,"ce1b07cdec09","Times-Roman",10.5,0]],"n_blocks":14,"n_draws":14,"n_images":0,"n_lines":36,"n_links":0,"n_spans":36,"size":[612.0,792.0]}]} \ No newline at end of file +{"backend":"pymupdf","manifest":{"fpdf2":"2.8.7","platform":"Linux","pymupdf":"1.28.0","python":"3.12","reportlab":"5.0.0"},"pages":[{"chars":1211,"draws":[["rect",[111.0,342.6,501.0,360.6],"#123a5e",null],["rect",[111.0,360.6,501.0,378.6],"#ffffff",null],["rect",[111.0,378.6,501.0,396.6],"#f2f5f8",null],["rect",[111.0,396.6,501.0,414.6],"#ffffff",null],["hline",[111.0,342.6,501.0,342.6],null,"#c9d3dd"],["hline",[111.0,414.6,501.0,414.6],null,"#c9d3dd"],["vline",[111.0,342.6,111.0,414.6],null,"#c9d3dd"],["vline",[501.0,342.6,501.0,414.6],null,"#c9d3dd"],["hline",[111.0,360.6,501.0,360.6],null,"#c9d3dd"],["hline",[111.0,378.6,501.0,378.6],null,"#c9d3dd"],["hline",[111.0,396.6,501.0,396.6],null,"#c9d3dd"],["vline",[201.0,342.6,201.0,414.6],null,"#c9d3dd"],["vline",[311.0,342.6,311.0,414.6],null,"#c9d3dd"],["vline",[411.0,342.6,411.0,414.6],null,"#c9d3dd"]],"lines":[[[70.8,62.3,292.9,87.1],81.6,"d2816682c720","Helvetica-Bold",18.0,2],[[70.8,96.7,161.1,114.6],110.6,"034824a67fcb","Helvetica-Bold",13.0,2],[[70.8,121.0,541.2,135.1],132.1,"59a123486b81","Times-Roman",10.5,0],[[70.8,135.5,541.2,149.6],146.6,"896f9a43e7a7","Times-Roman",10.5,0],[[70.8,150.0,541.2,164.1],161.1,"0a6262d12866","Times-Roman",10.5,0],[[70.8,164.5,541.2,178.6],175.6,"2d7718ffcc3b","Times-Roman",10.5,0],[[70.8,179.0,178.1,193.1],190.1,"ac38c3dfad58","Times-Roman",10.5,0],[[70.8,205.2,139.4,223.1],219.1,"1356caacb628","Helvetica-Bold",13.0,2],[[70.8,229.5,541.2,243.6],240.6,"aa3f9a55882b","Times-Roman",10.5,0],[[70.8,244.0,479.4,258.1],255.1,"fbf4ab57dd63","Times-Roman",10.5,0],[[70.8,266.2,75.0,282.7],279.1,"ecf727ea048d","Helvetica",12.0,0],[[88.8,266.5,233.8,280.6],277.6,"0279a96eb938","Times-Roman",10.5,0],[[70.8,288.7,75.0,305.2],301.6,"ecf727ea048d","Helvetica",12.0,0],[[88.8,289.0,283.6,303.1],300.1,"35c7b7ad9032","Times-Roman",10.5,0],[[70.8,311.2,75.0,327.7],324.1,"ecf727ea048d","Helvetica",12.0,0],[[88.8,311.5,301.7,325.6],322.6,"56998157b521","Times-Roman",10.5,0],[[117.0,345.0,142.5,357.4],354.6,"712da99c9c7f","Helvetica-Bold",9.0,2],[[207.0,345.0,240.0,357.4],354.6,"c4d8fa69b55a","Helvetica-Bold",9.0,2],[[317.0,345.0,349.5,357.4],354.6,"441bda6cd856","Helvetica-Bold",9.0,2],[[417.0,345.0,463.5,357.4],354.6,"5f5a042355ba","Helvetica-Bold",9.0,2],[[117.0,362.9,134.5,375.3],372.6,"10c69dacbcac","Helvetica",9.0,0],[[265.0,362.9,305.0,375.3],372.6,"a851550cdfa4","Helvetica",9.0,0],[[372.5,362.9,405.0,375.3],372.6,"c5d5b422446e","Helvetica",9.0,0],[[480.0,362.9,495.0,375.3],372.6,"4dea1daedbe9","Helvetica",9.0,0],[[117.0,380.9,134.5,393.3],390.6,"2a7ab28611c8","Helvetica",9.0,0],[[265.0,380.9,305.0,393.3],390.6,"7452c7c8588d","Helvetica",9.0,0],[[365.0,380.9,405.0,393.3],390.6,"a470d6c301ef","Helvetica",9.0,0],[[480.0,380.9,495.0,393.3],390.6,"01592d51db5a","Helvetica",9.0,0],[[117.0,398.9,134.5,411.3],408.6,"c72dd2f011c6","Helvetica",9.0,0],[[265.0,398.9,305.0,411.3],408.6,"b3eb86862c20","Helvetica",9.0,0],[[377.5,398.9,405.0,411.3],408.6,"8fcf662353a5","Helvetica",9.0,0],[[480.0,398.9,495.0,411.3],408.6,"8bd7954c40c1","Helvetica",9.0,0],[[70.8,435.7,151.0,453.6],449.6,"ddafdd979564","Helvetica-Bold",13.0,2],[[70.8,460.0,541.2,474.1],471.1,"59a123486b81","Times-Roman",10.5,0],[[70.8,474.5,541.2,488.6],485.6,"896f9a43e7a7","Times-Roman",10.5,0],[[70.8,489.0,208.4,503.1],500.1,"ce1b07cdec09","Times-Roman",10.5,0]],"n_blocks":14,"n_draws":14,"n_images":0,"n_lines":36,"n_links":0,"n_spans":36,"size":[612.0,792.0]}]} \ No newline at end of file diff --git a/testkit/golden_ir.py b/testkit/golden_ir.py index b9663ad..2f150da 100644 --- a/testkit/golden_ir.py +++ b/testkit/golden_ir.py @@ -16,7 +16,18 @@ That is what inference actually consumes, and it keeps the goldens small enough to live in git and to diff by eye. - python testkit/golden_ir.py freeze # write goldens +**The canonical freeze environment is CI Linux** (`.github/workflows/gate.yml`: +ubuntu-24.04 + fonts-liberation), for the same reason CI is the number of +record everywhere else in this project. A golden is a comparison between two +runs, and the corpus is *regenerated* on every machine: ReportLab and fpdf2 are +deterministic in their layout, but the library versions that produce the PDFs +and the PyMuPDF version that reads them are not fixed by this repository. So +each golden carries a manifest of the versions it was frozen with, and `verify` +says so when the running environment differs -- otherwise environment drift +arrives looking exactly like parser breakage, and the natural response to that +is to "fix" a parser that was never broken. + + python testkit/golden_ir.py freeze # write goldens (+ env manifest) python testkit/golden_ir.py verify # compare, exit non-zero on drift """ import argparse @@ -42,6 +53,38 @@ def _h(s): return hashlib.sha1(s.encode("utf-8", "replace")).hexdigest()[:12] +# Everything that can legitimately change a golden without the parser changing. +# pymupdf reads the PDFs; reportlab and fpdf2 WRITE them, so their versions are +# part of the input, not of the environment around it. +MANIFEST_PKGS = ("pymupdf", "reportlab", "fpdf2") + + +def manifest(): + import platform + try: + from importlib.metadata import version, PackageNotFoundError + except ImportError: # py<3.8 + return {"platform": platform.system()} + out = {} + for pkg in MANIFEST_PKGS: + try: + out[pkg] = version(pkg) + except PackageNotFoundError: + out[pkg] = "absent" + out["python"] = "%d.%d" % sys.version_info[:2] + out["platform"] = platform.system() + return out + + +def manifest_delta(frozen, running): + """Keys whose value differs, as 'key frozen->running'. Empty if identical.""" + if not frozen: + return ["(golden carries no manifest -- frozen before this was recorded)"] + return ["%s %s->%s" % (k, frozen.get(k, "?"), running.get(k, "?")) + for k in sorted(set(frozen) | set(running)) + if frozen.get(k) != running.get(k)] + + def digest(path): ir = parse_pdf(path, keep_image_data=False) pages = [] @@ -71,7 +114,7 @@ def digest(path): "lines": lines, "draws": draws, }) - return {"backend": "pymupdf", "pages": pages} + return {"backend": "pymupdf", "manifest": manifest(), "pages": pages} # The corpus is REGENERATED on each machine, so a golden is only meaningful @@ -127,6 +170,8 @@ def verify(): if not pdfs: print("no corpus; run the generators first") return 2 + running = manifest() + deltas = set() bad = 0 for p in pdfs: name = os.path.splitext(os.path.basename(p))[0] @@ -137,6 +182,7 @@ def verify(): continue want = json.load(open(gp)) got = digest(p) + deltas.update(manifest_delta(want.get("manifest"), running)) if len(want["pages"]) != len(got["pages"]): print(" %-26s page count %d -> %d" % (name[:26], len(want["pages"]), len(got["pages"]))) @@ -160,6 +206,14 @@ def verify(): else: print(" %-26s ok" % name[:26]) print("\n%d/%d documents match the golden IR" % (len(pdfs) - bad, len(pdfs))) + if deltas: + print("\nNOTE: this environment is not the one the goldens were frozen " + "in:\n " + "\n ".join(sorted(deltas))) + print(" Running: " + ", ".join("%s=%s" % kv for kv in sorted(running.items()))) + if bad: + print(" Drift above may be attributable to that and NOT to the " + "parser. Re-freeze only on the canonical environment " + "(CI Linux), never to make a failure go away.") return 1 if bad else 0 diff --git a/testkit/harness.py b/testkit/harness.py index a2eefbf..60caf84 100644 --- a/testkit/harness.py +++ b/testkit/harness.py @@ -135,6 +135,39 @@ def gram_cov(src, out, k=3): # ------------------------------------------------------------- geometry side +# Chinese, Japanese and Korean are written without spaces, so a whitespace +# tokeniser returns one "word" per rendered LINE -- up to 32 characters on the +# corpus i18n page. Re-wrap that line one character earlier and the token no +# longer matches anything, although every character survived: c4_i18n scored +# doc_recall 0.8298 on Linux and passed on Windows purely because the two +# renderers broke the line in different places. Measured before the fix: 16 of +# 94 source tokens unmatched, all 16 Hangul/CJK/Kana, zero Latin, zero Arabic, +# zero Hebrew (Arabic and Hebrew DO use spaces and never had the problem). +# +# So runs in these scripts are tokenised per character, with the run's box +# divided evenly across them. It is not a leniency: it counts the same content +# the writer emitted, in the unit that script actually has. +_CONTINUA = ((0x3040, 0x30FF), # Hiragana + Katakana + (0x3400, 0x4DBF), # CJK ext A + (0x4E00, 0x9FFF), # CJK unified + (0xAC00, 0xD7AF), # Hangul syllables + (0xF900, 0xFAFF)) # CJK compatibility + + +def _is_continua(ch): + o = ord(ch) + return any(lo <= o <= hi for lo, hi in _CONTINUA) + + +def _split_continua(text, x0, y0, x1, y1): + """One token per character when the run has no word boundaries of its own.""" + if not any(_is_continua(c) for c in text): + return [(text, x0, y0, x1, y1)] + step = (x1 - x0) / max(1, len(text)) + return [(c, x0 + i * step, y0, x0 + (i + 1) * step, y1) + for i, c in enumerate(text) if not c.isspace()] + + def page_words(pdf_path): """[(page_idx, text, x0, y0, x1, y1)] in reading order per page.""" doc = fitz.open(pdf_path) @@ -142,7 +175,10 @@ def page_words(pdf_path): for p in doc: ws = p.get_text("words") # x0,y0,x1,y1,word,block,line,wordno ws.sort(key=lambda w: (round(w[1], 1), w[0])) - pages.append([(w[4], w[0], w[1], w[2], w[3]) for w in ws]) + out = [] + for w in ws: + out.extend(_split_continua(w[4], w[0], w[1], w[2], w[3])) + pages.append(out) doc.close() return pages diff --git a/testkit/margin_probe.py b/testkit/margin_probe.py new file mode 100644 index 0000000..37c2519 --- /dev/null +++ b/testkit/margin_probe.py @@ -0,0 +1,112 @@ +"""Does the page's vertical origin agree between backends, and would it? + +Ruling law 18 gate 3. `infer()` derives `DocLayout.margin_t` from line-box +TOPS -- the one vertical quantity the two parsers legitimately disagree about, +because it comes from font-metric tables they do not share (M2.d escalation +packet). Baselines, which they report identically, carry the same information. + +This reports, per document and per backend: + + margin_t as shipped, from line-box tops + baseline-anchored what it becomes when the topmost text contributes + `baseline - (leading - 0.21*size)` instead of its box top -- + exactdoc's own published paragraph-top convention, the same + formula the writer uses to place the paragraph + +and the pdfium-minus-pymupdf disagreement under each. The fix's own claim is +that the second column agrees sub-0.1pt across backends; if it does not, the +convention has not been removed and nothing should be rendered. + + python testkit/margin_probe.py +""" +import argparse +import glob +import os +import sys + +import _paths # noqa: F401 + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +DESCENT = 0.21 # exactdoc's published descent convention +SINGLE_LEAD = 1.16 # infer()'s own single-line leading multiple + + +def _line_top_baseline_anchored(block_lines, ln): + """`baseline - (leading - 0.21*size)`, the writer's paragraph-top formula.""" + size = max((s.size for s in ln.spans), default=10.0) + bases = [l.baseline for l in block_lines] + diffs = sorted(b2 - b1 for b1, b2 in zip(bases, bases[1:]) if b2 > b1) + lead = diffs[len(diffs) // 2] if diffs else max(size * SINGLE_LEAD, 4.0) + return ln.baseline - (lead - DESCENT * size) + + +def margins(path, parse, why=False): + """(shipped, baseline-anchored) margin_t, unrounded, + what set the minimum.""" + from exactdoc.dialect import normalize + ir = normalize(parse(path, keep_image_data=False)) + shipped, anchored = [], [] + for p in ir.pages: + for b in p.blocks: + for ln in b.lines: + shipped.append((ln.bbox[1], "text p%d |%s|" + % (p.number, ln.text[:26]))) + anchored.append((_line_top_baseline_anchored(b.lines, ln), + "text p%d |%s|" % (p.number, ln.text[:26]))) + for d in p.drawings: + # drawings have no baseline; their boxes are geometry both backends + # now agree on (M2.b), so they contribute unchanged to both columns + shipped.append((d.bbox[1], "draw p%d %s" % (p.number, d.shape))) + anchored.append((d.bbox[1], "draw p%d %s" % (p.number, d.shape))) + if not shipped: + return None, None, "", "" + s = min(shipped) + a = min(anchored) + return s[0], a[0], s[1], a[1] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("names", nargs="*") + ap.add_argument("--why", action="store_true") + a = ap.parse_args() + from exactdoc.parse import parse_pdf as mu + from exactdoc.parse_pdfium import parse_pdf as pf + + srcs = sorted(glob.glob(os.path.join(ROOT, "corpus", "pdfs", "*.pdf"))) + srcs += sorted(glob.glob(os.path.join(ROOT, "testkit", "adv", "*.pdf"))) + if a.names: + srcs = [s for s in srcs if any(k in os.path.basename(s) for k in a.names)] + + print("%-24s %-17s %-17s %-9s %s" + % ("document", "shipped mu/pf", "anchored mu/pf", "d shipped", "d anchored")) + worst_ship = worst_anch = 0.0 + for s in srcs: + name = os.path.splitext(os.path.basename(s))[0] + try: + ms, ma, msw, maw = margins(s, mu) + ps, pa, psw, paw = margins(s, pf) + except Exception as e: + print("%-24s FAILED %s" % (name[:24], e)) + continue + if ms is None or ps is None: + continue + ds, da = abs(ps - ms), abs(pa - ma) + worst_ship = max(worst_ship, ds) + worst_anch = max(worst_anch, da) + flag = "" if da <= 0.1 else " <-- still disagrees" + print("%-24s %-17s %-17s %-9.2f %.3f%s" + % (name[:24], "%.1f / %.1f" % (ms, ps), "%.1f / %.1f" % (ma, pa), + ds, da, flag)) + if a.why and da > 0.1: + print(" pymupdf min set by: %s" % maw) + print(" pdfium min set by: %s" % paw) + print("\nworst backend disagreement: shipped %.2fpt -> baseline-anchored %.2fpt" + % (worst_ship, worst_anch)) + print("law 18 gate 3 requires the anchored column to agree within 0.1pt") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/testkit/parity_policy.json b/testkit/parity_policy.json new file mode 100644 index 0000000..034a6fc --- /dev/null +++ b/testkit/parity_policy.json @@ -0,0 +1,60 @@ +{ + "_note": "The backend-swap acceptance policy, as data the gate executes rather than prose a reader is trusted to apply. The test used to say 'swap is acceptable when regressions == 0' and exit on that count, while ROADMAP \u00a73.2 and STATUS D2 said two named documents were formally accepted divergences -- so the executable rule and the ratified rule disagreed, and CI resolved the disagreement by marking the step continue-on-error, which retired the gate altogether. Record the floors with `backend_parity.py --update-policy` on the canonical environment.", + "accepted_shortfalls": { + "01_whitepaper_market.pdf": { + "defect": "D2", + "floors": { + "live_text_cov": 0.9573, + "page_err": 0, + "within2pt": 0.5333, + "word_recall": 0.9677 + }, + "reason": "font-metric convention difference: PyMuPDF's base-14 above/below-baseline ratios reach margin_t and displace every word on the page by a constant. Reproducing them means vendoring MuPDF's table into the permissive tree.", + "reference_at_record": { + "live_text_cov": 0.9595, + "page_err": 0, + "within2pt": 0.7194, + "word_recall": 0.9677 + } + }, + "02_research_paper.pdf": { + "defect": "D2", + "floors": { + "live_text_cov": 0.9736, + "page_err": 0, + "within2pt": 0.5685, + "word_recall": 0.9586 + }, + "reason": "same cause as 01_whitepaper_market: measured margin_t 63.30 (PyMuPDF) against 64.90 (PDFium), a constant 1.5pt displacement visible as two identical dy distributions offset by exactly that.", + "reference_at_record": { + "live_text_cov": 0.9736, + "page_err": 0, + "within2pt": 0.7614, + "word_recall": 0.9586 + } + }, + "_note": "Documents where the candidate is measurably worse and that is RATIFIED, with the cause attributed and proven unreachable from a permissive parser (STATUS D2: infer() derives the page's vertical origin from line-box tops, PyMuPDF reads it from its own base-14 metric table, PDFium exposes one vertical font metric and the parser already uses it). Acceptance is bounded by numeric floors: worsening past them fails, and clearing the divergence entirely fails as stale." + }, + "candidate_backend": "pdfium", + "expected_divergence": { + "_note": "Documents where the two backends disagree about what CORRECT means and the candidate was verified to be the right one. Nothing goes in here without rendered evidence, because the harness measures agreement with the incumbent and would otherwise demand that a bug be reproduced.", + "c4_i18n.pdf": { + "reason": "RTL: PDFium reports glyphs in visual order and this backend reorders them to logical, which is what a DOCX must carry. PyMuPDF returns visual order, so its DOCX renders Arabic backwards.", + "verified": "rendered source and both outputs side by side; the PyMuPDF output reads the same words in reverse" + }, + "c5_graphics.pdf": { + "metric_caveat": "live_text_cov scored PyMuPDF higher (0.71 vs 0.68) because invisible white text still counts as live. Text coverage cannot see contrast.", + "reason": "The page opens with a gradient band carrying white text. PyMuPDF does not report the gradient, so the band's text is emitted white on white and is invisible. PDFium reports the pattern flattened to grey, so the band survives and its text is legible.", + "verified": "rendered: PyMuPDF's output has no band, PDFium's has a grey one with the heading readable" + } + }, + "margins": { + "_note": "How much worse the candidate may be on a dimension before the document counts as a regression. Compared in priority order -- page error, then live text, then page-level placement, then fine placement -- because that is the order in which a reader notices, and a flat 'any drop is a regression' rule once called c5_graphics worse for a 0.03 live-text dip while it gained a correct page count and 0.45 of placement.", + "live_text_cov": 0.05, + "page_err": 0, + "within2pt": 0.08, + "word_recall": 0.05 + }, + "reference_backend": "pymupdf", + "schema": 1 +} diff --git a/testkit/residual.py b/testkit/residual.py new file mode 100644 index 0000000..e706424 --- /dev/null +++ b/testkit/residual.py @@ -0,0 +1,145 @@ +"""Is the remaining placement error systematic, or is it scatter? + +The question this answers is the one that decides whether the permissive-parser +port can finish at all. Three times on this branch a change made the pdfium IR +provably match PyMuPDF on the thing that was blamed -- block boundaries 201/201, +path bboxes coordinate-identical, a code region classified identically -- and +the rendered score barely moved. Either the residual is systematic (a per-page +offset or a steady accumulation, which a second pass can remove, and which means +convention-matching still pays) or it is irreducible scatter spread over every +word (which means it cannot be chased document by document). + +For a source/render pair this reports, pooled over pages: + + med |dx| A -> B A = median |dx| as measured + B = median |dx| after removing each page's CONSTANT dx + med |dy| A -> B A = median |dy| as measured + B = median |dy| after removing each page's AFFINE trend + (offset + accumulation down the page) + within2pt -> CEILING + within2pt as measured, then what it would be if both + systematic parts were removed perfectly -- the best any + anchoring fix could do + +Both arrows are raw -> after-fit, never p50 -> p90. A number that grows across +the arrow means the fit is fighting the data, not that the error got worse. + + python testkit/residual.py src.pdf rendered.pdf + python testkit/residual.py src.pdf rendered.pdf --hist # per-line dy shape + +The ceiling is the number to read. If pdfium's ceiling reaches PyMuPDF's actual +score, the gap is systematic and the port is a matter of finding the anchor. If +the ceiling sits well below it, the error is scatter and no amount of +convention-matching closes it. + +--hist answers a question no summary statistic can: is the vertical error a few +lines displaced by a whole leading (a wrap or line-count difference) or every +line off by a fraction of a point (an anchoring model difference)? Those have +opposite fixes and identical medians. +""" +import os +import sys + +import numpy as np + +import _paths # noqa: F401 +import harness + + +def analyse(src, rendered, keep=False): + sw, ow = harness.page_words(src), harness.page_words(rendered) + rows = [] + for i, sp in enumerate(sw): + if i >= len(ow): + break + by = {} + for j, o in enumerate(ow[i]): + by.setdefault(o[0], []).append(j) + cands = [] + for si, s in enumerate(sp): + for oj in by.get(s[0], ()): + o = ow[i][oj] + cands.append((abs(o[2] - s[2]) * 3 + abs(o[1] - s[1]), si, oj)) + cands.sort() + us, uo, xs, ys, dxs, dys = set(), set(), [], [], [], [] + for d, si, oj in cands: + if si in us or oj in uo: + continue + us.add(si) + uo.add(oj) + xs.append(sp[si][1]) + ys.append(sp[si][2]) + dxs.append(ow[i][oj][1] - sp[si][1]) + dys.append(ow[i][oj][2] - sp[si][2]) + if len(ys) < 8: + continue + xs, ys = np.array(xs), np.array(ys) + dxs, dys = np.array(dxs), np.array(dys) + # y: per-page affine trend (offset + accumulation down the page) + slope, inter = np.polyfit(ys, dys, 1) + ry = dys - (slope * ys + inter) + # x: per-page constant only. A slope in x would be a scale error, which + # is not what a second pass can fix by shifting. + rx = dxs - np.median(dxs) + rows.append((dxs, dys, rx, ry)) + if not rows: + return None + dxs = np.concatenate([r[0] for r in rows]) + dys = np.concatenate([r[1] for r in rows]) + rx = np.concatenate([r[2] for r in rows]) + ry = np.concatenate([r[3] for r in rows]) + return { + "n": len(dxs), + "dx": float(np.median(np.abs(dxs))), + "dy": float(np.median(np.abs(dys))), + "rx": float(np.median(np.abs(rx))), + "ry": float(np.median(np.abs(ry))), + "within2": float(np.mean(np.hypot(dxs, dys) <= 2.0)), + "ceiling": float(np.mean(np.hypot(rx, ry) <= 2.0)), + "dys": dys if keep else None, + } + + +def histogram(dys, leading=None, width=52): + """Where the vertical error actually sits, in 0.5pt buckets.""" + import collections + buckets = collections.Counter(round(float(d) * 2) / 2.0 for d in dys) + if not buckets: + return + top = max(buckets.values()) + print(" per-word dy distribution (0.5pt buckets, %d words):" % len(dys)) + for k in sorted(buckets): + if abs(k) > 20: + continue + n = buckets[k] + bar = "#" * max(1, int(width * n / top)) + note = "" + if leading and abs(abs(k) - leading) < 1.0: + note = " <-- one leading (%.1fpt)" % leading + print(" %+6.1f %-5d %s%s" % (k, n, bar, note)) + far = sum(n for k, n in buckets.items() if abs(k) > 20) + if far: + print(" beyond +-20pt: %d words" % far) + + +def main(): + hist = "--hist" in sys.argv + args = [a for a in sys.argv[1:] if not a.startswith("--")] + r = analyse(args[0], args[1], keep=hist) + if not r: + print("too few matched words") + return 2 + print("%s (%d matched words)" % (os.path.basename(args[0]), r["n"])) + print(" median |dx| %.2f -> %.2f after removing the per-page constant" + % (r["dx"], r["rx"])) + print(" median |dy| %.2f -> %.2f after removing the per-page affine trend" + % (r["dy"], r["ry"])) + print(" within2pt %.3f -> CEILING %.3f" % (r["within2"], r["ceiling"])) + if hist: + lead = float(os.environ.get("LEADING", "0")) or None + histogram(r["dys"], leading=lead) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testkit/runall.py b/testkit/runall.py index b43bcb7..a9f4037 100644 --- a/testkit/runall.py +++ b/testkit/runall.py @@ -1,44 +1,96 @@ -"""Convert every PDF in the given directories and score with the harness. - - python testkit/runall.py testkit/adv my_samples exactdoc_v1.1/corpus/pdfs - -Writes batch/results.json plus side-by-side comparison PNGs per document. -Exit code is non-zero if any document regresses past the gate thresholds, -so this doubles as a CI check. +"""Convert the manifest corpus, score it with the harness, and gate on it. + + python testkit/runall.py # both lanes, the default + python testkit/runall.py --lane product # one lane + python testkit/runall.py --absolute # release-qualification gate + GATE_BASELINE=update python testkit/runall.py # re-record the numbers + +Two lanes always run, because `refine()` tunes the layout against the same +renderer the gate then measures with: a refined-only number can improve because +the loop memorised the oracle rather than because the converter got better. The +`raw` lane is the uncontaminated control, the `product` lane is what ships +(`exactdoc.options.PRODUCT`), and **the exit code gates on both**. It used to +gate on the refined lane alone, so a raw-lane regression could not fail the +build -- which meant the control lane, the one whose whole purpose is to be +untainted, was the one nobody had to answer for. + +The decision itself is `testkit/gate.py`, tested independently in +`tests/test_gate_mutations.py`. This file's job is to produce numbers and hand +them over; it makes no policy of its own. + +Writes, per lane, into testkit/batch/lane_/: + results.json every harness result + verdict.json what the gate decided and why +and folds both, plus the environment, into testkit/batch/evidence.json. """ -import os, sys, json, time, glob, traceback +import argparse +import glob +import json +import os +import sys +import time +import traceback import _paths # noqa: F401 +import evidence +import gate import harness ROOT = os.path.dirname(os.path.abspath(__file__)) +PROJECT = os.path.dirname(ROOT) OUT = os.path.join(ROOT, "batch") +EVIDENCE = os.path.join(OUT, "evidence.json") -# CI gate: a conversion must clear all of these. -GATE = {"page_match": True, "live_text_cov": 0.95, "doc_recall": 0.95, - "word_recall": 0.90} - - -def main(dirs, out=OUT, gate=True, refine_rounds=None): - if refine_rounds is None: - env = os.environ.get("REFINE", "0") - refine_rounds = int(env) if env.isdigit() else 0 - os.makedirs(out, exist_ok=True) - pdfs = [] - for d in dirs: - pdfs += sorted(glob.glob(os.path.join(d, "*.pdf"))) - if not pdfs: - print("no PDFs found in", dirs) - return 2 +# ------------------------------------------------------------------- the corpus +def resolve_corpus(manifest, dirs=None): + """-> (paths, problems). Manifest-driven, with the directories cross-checked. + + Globbing a directory answers "what is here", which is not the question. The + question is "is this the corpus the baseline was recorded against", and only + a manifest can answer it. The glob is still run, to catch a document that is + present but unexpected. + """ + problems, paths, seen = [], [], {} + for doc_id, spec in sorted(manifest.get("documents", {}).items()): + p = os.path.join(PROJECT, spec["path"], doc_id) + if not os.path.exists(p): + problems.append(("missing", doc_id, + "expected at %s -- run the generator named in the " + "manifest (%s)" % (spec["path"], spec.get("generator", "?")))) + continue + if doc_id in seen: + problems.append(("duplicate", doc_id, + "two manifest entries share a basename; outputs " + "and result rows would overwrite each other")) + continue + seen[doc_id] = p + paths.append(p) + for d in sorted(set(s["path"] for s in manifest.get("documents", {}).values()) + if not dirs else dirs): + for p in sorted(glob.glob(os.path.join(PROJECT, d, "*.pdf"))): + if os.path.basename(p) not in seen: + problems.append(("unexpected", os.path.basename(p), + "present in %s but not in the manifest" % d)) + return paths, problems + + +# ------------------------------------------------------------------- one lane +def run_lane(lane, paths, options, out_dir, baseline=None, manifest=None, + absolute=False, save_images=True): + """Convert + score + gate one lane. Returns (results, verdict).""" from exactdoc.convert import convert + + os.makedirs(out_dir, exist_ok=True) results, converted = [], [] - for p in pdfs: + print("\n================ lane: %s (%s) ================" + % (lane, options.profile_id())) + for p in paths: name = os.path.splitext(os.path.basename(p))[0] - docx = os.path.join(out, name + ".docx") + docx = os.path.join(out_dir, name + ".docx") t0 = time.time() try: - convert(p, docx, refine_rounds=refine_rounds) + convert(p, docx, options=options) converted.append((p, docx, round(time.time() - t0, 2))) except Exception as e: results.append({"src": os.path.basename(p), @@ -48,16 +100,16 @@ def main(dirs, out=OUT, gate=True, refine_rounds=None): print("\n-- LibreOffice batch render --") rmap = harness.batch_docx_to_pdf([d for _, d, _ in converted], - os.path.join(out, "rendered")) + os.path.join(out_dir, "rendered")) print("rendered %d/%d" % (sum(1 for v in rmap.values() if v), len(rmap))) print("\n-- scoring --") for p, docx, secs in converted: name = os.path.splitext(os.path.basename(p))[0] try: - r = harness.evaluate(p, docx, os.path.join(out, "rendered"), - save_images=True, - img_dir=os.path.join(out, "cmp_" + name)) + r = harness.evaluate(p, docx, os.path.join(out_dir, "rendered"), + save_images=save_images, + img_dir=os.path.join(out_dir, "cmp_" + name)) r["convert_s"] = secs results.append(r) print(harness.brief(r)) @@ -66,65 +118,103 @@ def main(dirs, out=OUT, gate=True, refine_rounds=None): "eval_error": "%s: %s" % (type(e).__name__, e)}) print("EVAL FAIL %-28s %s" % (name, e)) - with open(os.path.join(out, "results.json"), "w") as f: + with open(os.path.join(out_dir, "results.json"), "w") as f: json.dump(results, f, indent=1) + verdict = gate.check(lane, results, manifest=manifest, baseline=baseline, + absolute=absolute) + with open(os.path.join(out_dir, "verdict.json"), "w") as f: + json.dump(verdict.as_dict(), f, indent=1) + print("\n" + verdict.report()) + return results, verdict + + +# ----------------------------------------------------------------------- main +def main(argv=None): + from exactdoc.options import LANES + + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("dirs", nargs="*", default=None, + help="(compatibility) extra directories to cross-check for " + "unexpected documents; the corpus itself comes from " + "the manifest") + ap.add_argument("--lane", choices=sorted(LANES) + ["both"], default="both") + ap.add_argument("--absolute", action="store_true", + help="also apply the release-qualification thresholds") + ap.add_argument("--backend", default=None, + help="override the profile's backend for every lane") + ap.add_argument("--no-images", action="store_true", + help="skip the side-by-side comparison PNGs") + ap.add_argument("--out", default=OUT) + ap.add_argument("--evidence", default=None, + help="evidence JSON to merge into (default /evidence.json)") + a = ap.parse_args(argv) + updating = os.environ.get("GATE_BASELINE") == "update" + + manifest = gate.load_manifest() + if manifest is None: + print("no corpus manifest at %s -- the gate cannot know which documents " + "it is supposed to have measured." % gate.MANIFEST_PATH) + return 2 + paths, problems = resolve_corpus(manifest, a.dirs or None) + for kind, doc, why in problems: + print("CORPUS %-11s %-28s %s" % (kind, doc[:28], why)) + if not paths: + print("no corpus documents resolved; run the generators first") + return 2 - fails = [] - for r in results: - if "convert_error" in r or "eval_error" in r: - fails.append((r["src"], "did not convert/score")) - continue - for k, thr in GATE.items(): - v = r.get(k) - if v is None: - continue - if (thr is True and v is not True) or (thr is not True and v < thr): - fails.append((r["src"], "%s=%s (want %s)" % (k, v, thr))) - print("\n%d/%d documents pass the gate" % (len(results) - len({f[0] for f in fails}), - len(results))) - for s, why in fails: - print(" FAIL %-34s %s" % (s[:34], why)) - print("\nwrote", os.path.join(out, "results.json")) - return 1 if (gate and fails) else 0 - - -def lanes(dirs): - """Run the gate twice -- refine OFF and refine ON -- and report both. - - refine() tunes the layout against the same renderer the gate then measures - with, so a refined-only number can improve because the loop memorised the - oracle rather than because the converter got better. The no-refine lane is - the uncontaminated number; the refined lane is the product default. Both - are always printed side by side, and the exit code gates on the refined - lane (what users get) while regressions in the raw lane stay visible. - """ - import statistics as st - results = {} - for tag, rr in (("norefine", 0), ("refine", 3)): - print("\n================ lane: %s ================" % tag) - out = os.path.join(OUT, "lane_" + tag) - code = main(dirs, out=out, gate=True, refine_rounds=rr) - with open(os.path.join(out, "results.json")) as f: - results[tag] = (code, json.load(f)) - print("\n================ lane comparison ================") - print("%-10s %-9s %-11s %-9s %-9s" % - ("lane", "pagematch", "within2pt", "livetext", "dy50med")) - for tag in ("norefine", "refine"): - _, rows = results[tag] - ok = [r for r in rows if "convert_error" not in r and "eval_error" not in r] - if not ok: - print("%-10s (no results)" % tag) - continue - print("%-10s %d/%-7d %-11.3f %-9.4f %-9.2f" % ( - tag, sum(1 for r in ok if r.get("page_match")), len(ok), - st.mean(r.get("within2pt", 0) for r in ok), - st.mean(r.get("live_text_cov", 0) for r in ok), - st.median([r.get("dy_p50", 0) for r in ok]))) - return results["refine"][0] + env = evidence.environment() + if not env["canonical"]: + print("\nNOTE: %s is not the canonical environment. CI Linux is the " + "number of record; local runs render with different fonts and may " + "legitimately differ inside tolerance." % env["os"]) + + lanes = sorted(LANES) if a.lane == "both" else [a.lane] + verdicts, lane_evidence = {}, {} + for lane in lanes: + options = LANES[lane] + if a.backend: + options = options.replace(backend=a.backend) + out_dir = os.path.join(a.out, "lane_" + lane) + baseline = None if updating else gate.load_lane(lane) + results, verdict = run_lane( + lane, paths, options, out_dir, baseline=baseline, manifest=manifest, + absolute=a.absolute, save_images=not a.no_images) + verdicts[lane] = verdict + rec = gate.record(lane, results) + lane_evidence[lane] = {"profile": options.as_dict(), + "profile_id": options.profile_id(), + "documents": rec["documents"], + "aggregate": rec["aggregate"], + "verdict": verdict.as_dict(), + "results": results} + if updating: + gate.save_lane(lane, rec, environment=env) + print("recorded numeric baseline for lane %r" % lane) + + ev_path = a.evidence or os.path.join(a.out, "evidence.json") + shipped = LANES.get("product") + profile = dict(shipped.as_dict(), profile_id=shipped.profile_id()) \ + if "product" in lanes else None + evidence.merge(ev_path, git=evidence.git_state(), environment=env, + profile=profile, + corpus={"manifest_documents": len(manifest.get("documents", {})), + "resolved": len(paths), + "problems": [{"kind": k, "document": d, "detail": w} + for k, d, w in problems]}, + lanes=lane_evidence) + print("\n-- evidence --\n%s" % evidence.summarise( + json.load(open(ev_path)))) + print("\nwrote %s" % ev_path) + + if problems: + print("\nThe corpus did not match the manifest. Numbers from an " + "incomplete or unexpected corpus are not comparable to the " + "baseline, so this is a failure and not a warning.") + return 1 + if updating: + return 0 + return 0 if all(v.ok for v in verdicts.values()) else 1 if __name__ == "__main__": - args = sys.argv[1:] or [os.path.join(ROOT, "adv")] - if os.environ.get("REFINE", "") == "lanes": - sys.exit(lanes(args)) - sys.exit(main(args)) + sys.exit(main()) diff --git a/tests/test_corpus_degradation.py b/tests/test_corpus_degradation.py new file mode 100644 index 0000000..35c789b --- /dev/null +++ b/tests/test_corpus_degradation.py @@ -0,0 +1,87 @@ +"""A missing oracle must degrade into a skip list, never a traceback. + +The corpus generator drives two external tools (headless Chromium, LibreOffice) +and three pure-Python producers. When Chromium was absent it called +subprocess.run([None, ...]) and died on a bare TypeError before writing a single +file -- including the eight documents that need no Chromium at all. An executor +who cannot generate a corpus cannot run the gate, and this repository has +already learned once that a gate which cannot run looks exactly like a gate that +passes (STATUS.md §5). + +So the contract is: + + tool missing -> skip, say which documents were skipped, exit 0 + tool present but failing -> error, say what it printed, exit 1 + + python -m pytest tests/ -q (or: python tests/test_corpus_degradation.py) +""" +import contextlib +import io +import os +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "testkit")) + +import gen_corpus as g # noqa: E402 + + +def _run(out, chrome, soffice): + """Run the generator with the given tool availability. Returns (code, log).""" + old = (g.OUT, g.HTML, g.CHROME, g.SOFFICE) + g.OUT, g.HTML = out, os.path.join(out, "_html") + g.CHROME, g.SOFFICE = chrome, soffice + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + code = g.main() + finally: + g.OUT, g.HTML, g.CHROME, g.SOFFICE = old + return code, buf.getvalue() + + +def test_no_external_tools_still_generates_the_python_documents(): + with tempfile.TemporaryDirectory() as td: + code, log = _run(td, chrome=None, soffice=None) + + assert code == 0, "a bare machine is a skip, not a failure:\n" + log + assert "SKIPPED" in log, "the skip list is the whole point:\n" + log + + made = sorted(f for f in os.listdir(td) if f.endswith(".pdf")) + assert made == ["f1_fpdf_brief.pdf", "r1_reportlab_report.pdf"], ( + "the documents that need no external tool must still be produced, " + "got %s\n%s" % (made, log)) + + # Every document that could not be made is named, with the reason. + for doc in ("c1_whitepaper", "c8_toc_links", "l1_libreoffice"): + assert doc in log, "%s was dropped without being named:\n%s" % (doc, log) + assert "CHROME=" in log and "SOFFICE=" in log, ( + "a skip must say how to fix itself:\n" + log) + + # An incomplete corpus must not be quietly comparable to the baselines. + assert "NOT comparable" in log, log + + +def test_a_present_but_broken_tool_is_an_error_not_a_skip(): + with tempfile.TemporaryDirectory() as td: + code, log = _run(td, chrome=os.path.join(td, "not-a-real-chrome"), + soffice=None) + assert code == 1, "a tool that is present and fails is a broken " \ + "machine, not a thin one:\n" + log + assert "FAILED" in log, log + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_"): + continue + try: + fn() + print(" ok %s" % name) + except AssertionError as e: + failures += 1 + print(" FAIL %s\n%s" % (name, e)) + print("\n%d test(s) failed" % failures) + sys.exit(1 if failures else 0) diff --git a/tests/test_gate_mutations.py b/tests/test_gate_mutations.py new file mode 100644 index 0000000..96b2487 --- /dev/null +++ b/tests/test_gate_mutations.py @@ -0,0 +1,445 @@ +"""Mutation tests: every way the gate used to report a false green must be red. + +A gate is a claim about what cannot get past it, and this project has already +paid twice for believing such a claim unverified -- once when the parity harness +omitted `within2pt` and reported 0 regressions on a swap that cost 0.510 -> 0.291, +and once when a gate that could not run at all (an undeclared `pypdfium2`) looked +exactly like a gate that passed. + +So each test below starts from a *healthy* result set, breaks exactly one thing, +and asserts the verdict turns red for the expected reason. If a check is ever +weakened or deleted, one of these fails. They need no corpus, no LibreOffice and +no PDF: the decision under test is a pure function of already-measured numbers, +which is the reason it was extracted into `testkit/gate.py` in the first place. + + python tests/test_gate_mutations.py +""" +import copy +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))), "testkit")) + +import gate # noqa: E402 + +FAILED = [] + + +def check(name, cond, detail=""): + print(" %-4s %s%s" % ("ok" if cond else "FAIL", name, + "" if cond else " <-- " + detail)) + if not cond: + FAILED.append(name) + + +# ------------------------------------------------------------------- fixtures +# Two documents is enough to exercise every rule, and small enough that a +# failure names the cause instead of requiring a bisect. `good` passes every +# threshold; `known` is a recorded shortfall carrying a defect ID, modelled on +# 04_exec_brief, whose live-text coverage has never reached 0.95. +def result(src, pages=(3, 3), live=0.99, doc=0.99, word=0.97, w2=0.60, + dy=0.50, raster=0.01): + return {"src": src, "src_pages": pages[0], "out_pages": pages[1], + "page_match": pages[0] == pages[1], "live_text_cov": live, + "doc_recall": doc, "word_recall": word, "within2pt": w2, + "dy_p50": dy, "dy_p90": dy * 3, "raster_frac": raster, + "mean_ssim": 0.80, "mean_iou": 0.70, "renderer": "libreoffice", + "src_words": 900, "n_media": 0} + + +def healthy(): + return [result("good.pdf"), + result("known.pdf", live=0.941, doc=0.944)] + + +MANIFEST = {"documents": {"good.pdf": {"path": "corpus/pdfs", "src_pages": 3}, + "known.pdf": {"path": "corpus/pdfs", "src_pages": 3}}} + + +def baseline_for(results): + rec = gate.record("product", results) + rec["shortfall_defects"] = {"known.pdf": "D-test"} + return rec + + +def verdict(results, baseline=None, manifest=None, absolute=False): + return gate.check("product", results, + manifest=manifest if manifest is not None else MANIFEST, + baseline=baseline if baseline is not None + else baseline_for(healthy()), + absolute=absolute) + + +def kinds(v): + return set(v.kinds()) + + +# ----------------------------------------------------------------------- tests +def test_healthy_passes(): + v = verdict(healthy()) + check("a healthy lane passes", v.ok, v.report()) + + +def test_removed_document(): + """`gen_corpus.py` exits 0 after skipping 8 of 16 documents.""" + r = [x for x in healthy() if x["src"] != "known.pdf"] + v = verdict(r) + check("removing a corpus document fails", "missing" in kinds(v), v.report()) + + +def test_unexpected_document(): + r = healthy() + [result("stranger.pdf")] + v = verdict(r) + check("an unmanifested document fails", "unexpected" in kinds(v), v.report()) + + +def test_duplicate_document(): + """Two inputs with one basename overwrite each other's output and row.""" + r = healthy() + [result("good.pdf", w2=0.10)] + v = verdict(r) + check("a duplicated document fails", "duplicate" in kinds(v), v.report()) + + +def test_identity_change(): + """A generator change that alters a document re-bases every number.""" + r = healthy() + r[0]["src_pages"] = 4 + r[0]["out_pages"] = 4 + v = verdict(r) + check("a changed source page count fails", "identity" in kinds(v), v.report()) + + +def test_render_error(): + """harness.evaluate() returns {'error': ...}; nothing used to look.""" + r = healthy() + r[0] = {"src": "good.pdf", "error": "LibreOffice produced no PDF", + "live_text_cov": 0.99} + v = verdict(r) + check("a render error fails", "error" in kinds(v), v.report()) + + +def test_convert_error(): + r = healthy() + r[0] = {"src": "good.pdf", "convert_error": "ValueError: document closed"} + v = verdict(r) + check("a conversion error fails", "error" in kinds(v), v.report()) + + +def test_missing_metric(): + """An absent metric used to be skipped: `if v is None: continue`.""" + for metric in ("within2pt", "dy_p50", "live_text_cov", "word_recall"): + r = healthy() + del r[0][metric] + v = verdict(r) + check("deleting %s fails" % metric, + "no-metric" in kinds(v) or "unrecorded" in kinds(v), v.report()) + + +def test_known_failure_sliding_further(): + """The old baseline stored metric NAMES: 0.941 could fall to 0.10 unseen.""" + r = healthy() + r[1]["live_text_cov"] = 0.10 + v = verdict(r) + check("a known failure sliding to 0.10 fails", "regression" in kinds(v), + v.report()) + + +def test_known_failure_inside_tolerance(): + r = healthy() + r[1]["live_text_cov"] = 0.941 - gate.METRICS["live_text_cov"]["tol"] / 2 + v = verdict(r) + check("a known failure inside tolerance still passes", v.ok, v.report()) + + +def test_page_error_magnitude(): + """page_match is a boolean: 1 page over and 40 over looked identical.""" + base = healthy() + base[1]["out_pages"] = 4 # already failing page_err + bl = baseline_for(base) + bl["shortfall_defects"] = {"known.pdf": "D-test"} + worse = copy.deepcopy(base) + worse[1]["out_pages"] = 40 + v = gate.check("product", worse, manifest=MANIFEST, baseline=bl) + check("a page error growing 1 -> 37 fails", "regression" in kinds(v), + v.report()) + + +def test_stale_record(): + """A recorded shortfall that now passes silently re-admits the regression.""" + r = healthy() + r[1]["live_text_cov"] = 0.97 + r[1]["doc_recall"] = 0.98 + v = verdict(r) + check("a stale record fails", "stale" in kinds(v), v.report()) + + +def test_undocumented_shortfall(): + """A recorded shortfall with no defect ID is an unexplained number.""" + bl = gate.record("product", healthy()) # no shortfall_defects + v = verdict(healthy(), baseline=bl) + check("a shortfall with no defect ID fails", "undocumented" in kinds(v), + v.report()) + + +def test_new_threshold_failure(): + r = healthy() + r[0]["word_recall"] = 0.50 + v = verdict(r) + check("a new threshold failure fails", + "threshold" in kinds(v) and "regression" in kinds(v), v.report()) + + +def test_unrecorded_document(): + bl = gate.record("product", [healthy()[0]]) + bl["shortfall_defects"] = {"known.pdf": "D-test"} + v = verdict(healthy(), baseline=bl) + check("a document with no numeric baseline fails", + "unrecorded" in kinds(v), v.report()) + + +def test_unrecorded_metric(): + bl = baseline_for(healthy()) + del bl["documents"]["good.pdf"]["within2pt"] + v = verdict(healthy(), baseline=bl) + check("a metric with no recorded value fails", "unrecorded" in kinds(v), + v.report()) + + +def test_aggregate_regression(): + """Per-document moves can each stay in tolerance and still move the mean.""" + bl = baseline_for(healthy()) + r = healthy() + for x in r: + x["within2pt"] -= 0.045 # under the per-doc tolerance + v = gate.check("product", r, manifest=MANIFEST, baseline=bl) + check("an aggregate-only regression fails", "regression" in kinds(v), + v.report()) + + +def test_absolute_mode_flags_known_shortfall(): + v = verdict(healthy(), absolute=True) + check("release mode refuses a known shortfall", + "unqualified" in kinds(v), v.report()) + check("regression mode accepts the same shortfall", verdict(healthy()).ok) + + +def test_both_lanes_gate(): + """`REFINE=lanes` returned only the refined lane's status.""" + import runall + src = open(runall.__file__).read() + check("the runner gates on every lane it ran", + "all(v.ok for v in verdicts.values())" in src, + "runall.main() must not return one lane's status") + + +def test_shipped_default_is_the_measured_default(): + """The API, the CLI and the product lane must be one configuration.""" + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) + from exactdoc.cli import build_parser + from exactdoc.options import LANES, PRODUCT + defaults = {a.dest: a.default for a in build_parser()._actions} + check("CLI refine default == PRODUCT", + defaults["refine"] == PRODUCT.refine_rounds, + "CLI %r vs profile %r" % (defaults["refine"], PRODUCT.refine_rounds)) + check("CLI target default == PRODUCT", defaults["target"] == PRODUCT.target) + check("CLI backend default == PRODUCT", defaults["backend"] == PRODUCT.backend) + check("CLI dpi default == PRODUCT", defaults["dpi"] == PRODUCT.dpi) + check("the product lane is the shipped profile", + LANES["product"] is PRODUCT) + check("the raw lane is refine-free", LANES["raw"].refine_rounds == 0) + + +# ------------------------------------------------------- the parity policy +# `backend_parity.adjudicate()` is pure for the same reason `gate.check()` is, +# and it needs the same treatment: the policy it applies used to live in a +# docstring while the code exited on a different rule entirely. +def parity_fixture(): + """Reference and candidate results, plus a policy that accepts one doc.""" + ref = {"good.pdf": result("good.pdf", w2=0.60), + "accepted.pdf": result("accepted.pdf", w2=0.72), + "diverges.pdf": result("diverges.pdf", live=0.71)} + cand = {"good.pdf": result("good.pdf", w2=0.60), + "accepted.pdf": result("accepted.pdf", w2=0.53), + "diverges.pdf": result("diverges.pdf", live=0.68)} + policy = { + "reference_backend": "pymupdf", "candidate_backend": "pdfium", + "margins": {"page_err": 0, "live_text_cov": 0.05, + "word_recall": 0.05, "within2pt": 0.08}, + "expected_divergence": {"diverges.pdf": {"reason": "verified visually"}}, + "accepted_shortfalls": {"accepted.pdf": { + "defect": "D2", + "floors": {"within2pt": 0.53, "page_err": 0, "live_text_cov": 0.99, + "word_recall": 0.97}}}, + } + return ref, cand, policy + + +def parity_kinds(ref, cand, policy, subset=False): + import backend_parity + _, summary = backend_parity.adjudicate(ref, cand, policy, subset=subset) + return summary, set(f["kind"] for f in summary["failures"]) + + +def test_parity_healthy_passes(): + ref, cand, policy = parity_fixture() + summary, kinds_ = parity_kinds(ref, cand, policy) + check("the ratified policy passes", summary["ok"], str(summary["failures"])) + check("the accepted shortfall is not counted a regression", + summary["regressions"] == 0, str(summary)) + check("the expected divergence is not counted a regression", + summary["expected_div"] == 1, str(summary)) + + +def test_parity_accepted_shortfall_worsening(): + """An unbounded acceptance is an acceptance of anything.""" + ref, cand, policy = parity_fixture() + cand["accepted.pdf"]["within2pt"] = 0.20 + summary, kinds_ = parity_kinds(ref, cand, policy) + check("an accepted shortfall falling past its floor fails", + "below-floor" in kinds_, str(summary["failures"])) + + +def test_parity_stale_acceptance(): + """A document accepted as worse that is no longer worse hides the next one.""" + ref, cand, policy = parity_fixture() + cand["accepted.pdf"]["within2pt"] = 0.72 + summary, kinds_ = parity_kinds(ref, cand, policy) + check("a stale acceptance fails", "stale" in kinds_, + str(summary["failures"])) + + +def test_parity_unbounded_acceptance(): + ref, cand, policy = parity_fixture() + policy["accepted_shortfalls"]["accepted.pdf"]["floors"] = None + summary, kinds_ = parity_kinds(ref, cand, policy) + check("an acceptance with no numeric floors fails", "unrecorded" in kinds_, + str(summary["failures"])) + + +def test_parity_new_regression(): + ref, cand, policy = parity_fixture() + cand["good.pdf"]["within2pt"] = 0.20 + summary, kinds_ = parity_kinds(ref, cand, policy) + check("a new regression fails", "regression" in kinds_ and + summary["regressions"] == 1, str(summary["failures"])) + + +def test_parity_missing_document(): + ref, cand, policy = parity_fixture() + del cand["good.pdf"] + summary, kinds_ = parity_kinds(ref, cand, policy) + check("a document scored under one backend only fails", + "missing" in kinds_, str(summary["failures"])) + + +def test_parity_subset_cannot_pass(): + ref, cand, policy = parity_fixture() + summary, _ = parity_kinds(ref, cand, policy, subset=True) + check("a --only subset can never report the swap acceptable", + not summary["ok"], str(summary)) + + +def test_committed_parity_policy_is_wellformed(): + import backend_parity + policy = backend_parity.load_policy() + accepted = {k: v for k, v in policy.get("accepted_shortfalls", {}).items() + if not k.startswith("_")} + check("the policy names its two backends", + policy.get("reference_backend") and policy.get("candidate_backend")) + for doc_id, spec in sorted(accepted.items()): + check("accepted %s carries a defect ID" % doc_id, bool(spec.get("defect"))) + check("accepted %s carries numeric floors" % doc_id, + isinstance(spec.get("floors"), dict) and spec["floors"], + "floors=%r -- record them with --update-policy" % spec.get("floors")) + for doc_id, spec in sorted(policy.get("expected_divergence", {}).items()): + if doc_id.startswith("_"): + continue + check("divergence %s carries rendered evidence" % doc_id, + bool(spec.get("verified"))) + + +def test_evidence_merge_never_empties_a_section(): + """The artifact is the single source of a release claim. Nothing may blank it. + + Measured on a full green run: the final `evidence.py --out` step, whose job is + to fill in the environment, passed the empty template's `parity: None` over + the verdict the previous step had recorded, and the run finished with an + evidence file that had forgotten its own parity result. + """ + import tempfile + import evidence + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, "evidence.json") + evidence.merge(p, parity={"ok": True, "regressions": 0}, + lanes={"product": {"verdict": {"ok": True}}}, + corpus={"resolved": 16}) + evidence.merge(p, parity=None, corpus=None, lanes={}, + environment={"os": "linux"}) + with open(p) as f: + doc = json.load(f) + check("a None section does not overwrite a recorded one", + doc.get("parity", {}).get("ok") is True, json.dumps(doc.get("parity"))) + check("an empty lanes dict does not drop recorded lanes", + "product" in (doc.get("lanes") or {}), str(doc.get("lanes"))) + check("the corpus section survives", (doc.get("corpus") or {}).get("resolved") == 16) + check("a later section still merges in", doc["environment"]["os"] == "linux") + + +def test_relative_tolerance(): + """dy_p50 spans 0.04pt to 101pt; one absolute slack cannot serve both.""" + small = gate.tolerance(gate.METRICS["dy_p50"], 0.6) + large = gate.tolerance(gate.METRICS["dy_p50"], 101.0) + check("the absolute floor governs a small drift", abs(small - 0.5) < 1e-9, + str(small)) + check("the proportional term governs a large drift", large > 10.0, str(large)) + check("a fraction metric stays absolute", + gate.tolerance(gate.METRICS["within2pt"], 0.9) == 0.05) + + +def test_committed_baseline_is_wellformed(): + """The committed record must satisfy the schema the gate reads.""" + doc = gate.load() + if not doc: + check("a baseline is committed", False, "no gate_baseline.json") + return + check("baseline is schema 2", doc.get("schema") == 2, str(doc.get("schema"))) + from exactdoc.options import LANES + for lane in LANES: + entry = doc.get("lanes", {}).get(lane) + check("lane %r is recorded" % lane, bool(entry), + "lanes present: %s" % sorted(doc.get("lanes", {}))) + if not entry: + continue + docs = entry.get("documents", {}) + manifest = gate.load_manifest() or {"documents": {}} + check("lane %r records every manifest document" % lane, + set(docs) == set(manifest["documents"]), + "record %d, manifest %d" % (len(docs), len(manifest["documents"]))) + for doc_id, metrics in sorted(docs.items()): + missing = [m for m in gate.METRICS if m not in metrics] + check("lane %r %s records every gated metric" % (lane, doc_id), + not missing, "missing %s" % missing) + for name, value in sorted(metrics.items()): + spec = gate.METRICS.get(name) + if spec and not gate.clears(spec["dir"], value, spec["threshold"]): + check("lane %r %s shortfall has a defect ID" % (lane, doc_id), + doc_id in entry.get("shortfall_defects", {}), + "%s=%s below %s" % (name, value, spec["threshold"])) + + +def main(): + tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + print("gate mutation tests (%d)" % len(tests)) + for t in tests: + print("\n%s" % t.__name__) + t() + print("\n%s" % ("all clear" if not FAILED else + "%d FAILED: %s" % (len(FAILED), ", ".join(FAILED)))) + return 1 if FAILED else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uv.lock b/uv.lock index 72af03d..39b1dfd 100644 --- a/uv.lock +++ b/uv.lock @@ -475,7 +475,7 @@ wheels = [ [[package]] name = "exactdoc" -version = "0.2.0" +version = "0.1.0a1" source = { editable = "." } dependencies = [ { name = "lxml" }, @@ -498,6 +498,9 @@ gdocs = [ { name = "google-auth-oauthlib", version = "1.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "google-auth-oauthlib", version = "1.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +pdfium = [ + { name = "pypdfium2" }, +] test = [ { name = "fpdf2", version = "2.8.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "fpdf2", version = "2.8.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -514,10 +517,11 @@ requires-dist = [ { name = "numpy", specifier = ">=1.24" }, { name = "pillow", specifier = ">=10.0" }, { name = "pymupdf", specifier = ">=1.23" }, + { name = "pypdfium2", marker = "extra == 'pdfium'", specifier = ">=4.25" }, { name = "python-docx", specifier = ">=1.1" }, { name = "reportlab", marker = "extra == 'test'", specifier = ">=4.0" }, ] -provides-extras = ["test", "gdocs"] +provides-extras = ["test", "pdfium", "gdocs"] [[package]] name = "fonttools" @@ -1699,6 +1703,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781 }, ] +[[package]] +name = "pypdfium2" +version = "5.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/42/0b51bdf50ccf13f3deb3209ca996179a49761dc191748469cf0de55b0055/pypdfium2-5.12.1.tar.gz", hash = "sha256:d0e0648fb2e28f50efcd1ec0a5a18ced9f4d66b2c227fae9b603f0a883b2d13f", size = 274428 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/7e/bd8df53b1131c582f6646372047b49162032bd01628d49b9a60cd94d2181/pypdfium2-5.12.1-py3-none-android_23_arm64_v8a.whl", hash = "sha256:05bab9b1ba2de7fc299ae2af25cb9c8a0543bc8bb893e879fe8c9ba8310e9ce4", size = 3392276 }, + { url = "https://files.pythonhosted.org/packages/b2/13/a2b71e17b0439d2af78c817a381e3557371c6c56098581da8485aef65ea6/pypdfium2-5.12.1-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:d4ee061e566a6422b660cdddaaa799a2d1cbf2f016921bcaf24d61426d01d942", size = 2848776 }, + { url = "https://files.pythonhosted.org/packages/e2/a8/d7a61700db3022792b28bce5264be90a7d2e7104998362af6b93766f50b2/pypdfium2-5.12.1-py3-none-macosx_13_0_arm64.whl", hash = "sha256:66a9ed40d70a5d728cd42148fecb9d7a0917c6161d6bb67c844093a4ed1df089", size = 3480243 }, + { url = "https://files.pythonhosted.org/packages/01/2c/d7a38fad74b6da0947cf8763aee0f8e6c9d3c12fc8e137aa615f7f8ae76c/pypdfium2-5.12.1-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:847378a5ab41332998b2621b21bab2e96dc8c3eff36a08bce26695b964163983", size = 3643490 }, + { url = "https://files.pythonhosted.org/packages/8f/29/aca739676323558595fcf8cdc8d7939d2b25aaaa6f538e829f1fee938cdd/pypdfium2-5.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eabf028ad8e7bc7811c9acf3a72718c180569b624b844d2c6cc974609784275", size = 3649734 }, + { url = "https://files.pythonhosted.org/packages/a4/e0/e4ecc05f4f1a11d11c8d684a24b2fc8be8207f341be985dac301caa4f6aa/pypdfium2-5.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7857cfa6642ec5a09db12ff8f5cf6b6494585b5e3a605399fddc4fb862837b63", size = 3380828 }, + { url = "https://files.pythonhosted.org/packages/17/1b/c94c9d486791276e736350917a11fe2cf3acba2c6c7f03f9ac0d51f8952a/pypdfium2-5.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05bfa20a08a96584253bbe38b60e13f81a037eac31c5579e607ec1480ad25dbf", size = 3777202 }, + { url = "https://files.pythonhosted.org/packages/14/aa/7f81f0c035fc32850dfab9daf78530814f215be226dad7491e3caa0a3e8c/pypdfium2-5.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f059f7bdbdf4352eb83691071096940d769d6ae5930b8734237fdb1bd78fbc2", size = 4186083 }, + { url = "https://files.pythonhosted.org/packages/23/16/21420a6f2bc5f981299c336817dd5d72709dad5fda30ac38cbd5f0f7b372/pypdfium2-5.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e10cbf41b21233ec5e20adfc170cf60edd77abead86a97dc708fff55a8a886c7", size = 3701734 }, + { url = "https://files.pythonhosted.org/packages/36/a1/bb89f49e2b3ea3e945b67859d2ec6e73e722a83c006099c675692641e51d/pypdfium2-5.12.1-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07eeebb2784f4cd38d386b924235df43217a397442796673296bb6efbdaad1d0", size = 4030403 }, + { url = "https://files.pythonhosted.org/packages/80/a7/cea5eb0c39e9c6fdf9853a3008bf021d08f962228073af90354b60c5ccdc/pypdfium2-5.12.1-py3-none-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5c3e6cbe43581af79526184643920ab03a9401a0c79f2226bea9d4d1e3d34008", size = 3994411 }, + { url = "https://files.pythonhosted.org/packages/fc/43/5e470213b27c13b0d94d03d97fc3740507edadea1d1e2ce2049bfbec4aa0/pypdfium2-5.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4648f0905441bcb141687ca2263bbf38a1aa056b943eef06019f91cff3e1da4a", size = 4993687 }, + { url = "https://files.pythonhosted.org/packages/db/da/af7972ca72f24ed720db6501333fd67bc66aa6aa5a5ec698551bd30ae62e/pypdfium2-5.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:bdff622181fab64f32328591c9c8287cdc745c9a1f2afc26ca3feba39e3e6645", size = 4534560 }, + { url = "https://files.pythonhosted.org/packages/a9/63/06a7f2cd691f7e336cbc53fd65453fee516042c8ddb016bc50d1cd2bed45/pypdfium2-5.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:236dbdc88aa54f14b27937ccb2ebe3dcf08c10dbb8652f432ea982dc9af39732", size = 5237681 }, + { url = "https://files.pythonhosted.org/packages/2b/f5/937b080671758ab0b3d3d69b2657006682e4c6a0134b36774be4ed1afcfb/pypdfium2-5.12.1-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:9c8856ce7dd77a7827476c7d75afe1197d6cd505f5cb4167b6aacf661f3f8ea5", size = 5143027 }, + { url = "https://files.pythonhosted.org/packages/32/02/094632700c24728fa443dc89d9d1c6e4fc05bb00778b22e8482fdb133da0/pypdfium2-5.12.1-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:5f257bb40fa44ce9ba18d2c919777dbd3f16bf22548b1d68fd56c7c92f1de530", size = 4647048 }, + { url = "https://files.pythonhosted.org/packages/fe/d0/12d84bf55a4fcf2c0ed242afc94168933194f672067e1d162aa60a8e4426/pypdfium2-5.12.1-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:974082344172da76a5c3c0782eaedfe6069dbe88db77d8c671ef36b61e9b14e2", size = 5088747 }, + { url = "https://files.pythonhosted.org/packages/92/9c/92a460bac1f6cfd6f96251802b3098a804fb251dfe0b5eb004ede958ae0e/pypdfium2-5.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:715ae16b34ea1d64884d58800155179ba700e9ea65a2f583b020666acd2bfb12", size = 5049695 }, + { url = "https://files.pythonhosted.org/packages/f7/67/53c61d366222550220b42a9212131407f49d3dbcf050178c620bf80fa899/pypdfium2-5.12.1-py3-none-win32.whl", hash = "sha256:e5358d2ce4ebc5c899aab1df9ca5d215357244e9168aa443225d3c1e649c7eac", size = 3725466 }, + { url = "https://files.pythonhosted.org/packages/a9/c3/08b62718faf2f6b6aa49207626e3113ff3ec1b3cd076c0ef8fd852f0e57c/pypdfium2-5.12.1-py3-none-win_amd64.whl", hash = "sha256:9609be73a6701a68f29dffe0335f7a2e4b3ba581542ed65d35d49f761a4600ca", size = 3859845 }, + { url = "https://files.pythonhosted.org/packages/6c/d5/ad551e55790134c8bc87ea16e0866a3721def0620fbfa19112c8ad4a25a6/pypdfium2-5.12.1-py3-none-win_arm64.whl", hash = "sha256:afc0b7e0c975a429abc75875209ce17b66d749f6ac5cbe8ba72470e83901e304", size = 3674605 }, +] + [[package]] name = "python-docx" version = "1.2.0"