From 87de7de55c2b98ea0238312cb3a7abc07ec6f5fd Mon Sep 17 00:00:00 2001 From: Kevin Raymond Date: Tue, 10 Nov 2026 12:35:35 -0600 Subject: [PATCH 1/3] whatif: measure a speed-up before you make it `prumo whatif` rebuilds the document with one change at a time and times it against the plain build. The changes are the preamble precompiled with mylatexformat, draft mode, and tikz's external library (experimental, and only with --shell-escape). The builds run in a temp copy without the probe, one at a time: a warm-up each, then the baseline and the variants in turns. The table compares medians, calls anything inside the spread (2 ms at least) noise, puts the probe's bound beside each saving, and compares the output by page count, the words on each page and a 24 dpi render. The externalize line hands the hook to tikz's picture builds too. Without it each picture build typesets the whole document, which is where a page count going from 2 to 3 comes from. XeTeX and LuaTeX cannot precompile a preamble, and the row says why. doctor has a mylatexformat row. --check-log now tells a file opened only to test that it exists from one read outside the hooks (siunitx with tikz opens color.sty and never reads it). The new fixture has eight claims, all tested. The TeX image installs mylatexformat, and make_artifacts.sh takes fixture names. --- docker/Dockerfile.tex | 3 + docs/03-architecture.md | 7 +- .../whatif/artifacts/lualatex/fontmap.txt | 27 + .../whatif/artifacts/lualatex/golden.json | 926 +++++++++++++++ .../whatif/artifacts/lualatex/kpsewhich.txt | 8 + fixtures/whatif/artifacts/lualatex/main.fls | 215 ++++ fixtures/whatif/artifacts/lualatex/main.log | 761 ++++++++++++ fixtures/whatif/artifacts/lualatex/main.prumo | 384 ++++++ .../whatif/artifacts/lualatex/pdffonts.txt | 6 + fixtures/whatif/artifacts/lualatex/report.txt | 76 ++ .../whatif/artifacts/pdflatex/fontmap.txt | 5 + .../whatif/artifacts/pdflatex/golden.json | 885 ++++++++++++++ .../whatif/artifacts/pdflatex/kpsewhich.txt | 8 + fixtures/whatif/artifacts/pdflatex/main.fls | 327 ++++++ fixtures/whatif/artifacts/pdflatex/main.log | 728 ++++++++++++ fixtures/whatif/artifacts/pdflatex/main.prumo | 384 ++++++ .../whatif/artifacts/pdflatex/pdffonts.txt | 5 + fixtures/whatif/artifacts/pdflatex/report.txt | 76 ++ fixtures/whatif/artifacts/xelatex/fontmap.txt | 4 + fixtures/whatif/artifacts/xelatex/golden.json | 880 ++++++++++++++ .../whatif/artifacts/xelatex/kpsewhich.txt | 8 + fixtures/whatif/artifacts/xelatex/main.fls | 316 +++++ fixtures/whatif/artifacts/xelatex/main.log | 707 +++++++++++ fixtures/whatif/artifacts/xelatex/main.prumo | 377 ++++++ .../whatif/artifacts/xelatex/pdffonts.txt | 6 + fixtures/whatif/artifacts/xelatex/report.txt | 77 ++ fixtures/whatif/main.tex | 42 + scripts/make_artifacts.sh | 4 +- src/doctor.rs | 49 +- src/lib.rs | 1 + src/main.rs | 62 + src/run.rs | 41 +- src/unused.rs | 43 +- src/whatif.rs | 1040 +++++++++++++++++ tests/fixtures.rs | 54 +- tests/tex.rs | 82 +- 36 files changed, 8578 insertions(+), 46 deletions(-) create mode 100644 fixtures/whatif/artifacts/lualatex/fontmap.txt create mode 100644 fixtures/whatif/artifacts/lualatex/golden.json create mode 100644 fixtures/whatif/artifacts/lualatex/kpsewhich.txt create mode 100644 fixtures/whatif/artifacts/lualatex/main.fls create mode 100644 fixtures/whatif/artifacts/lualatex/main.log create mode 100644 fixtures/whatif/artifacts/lualatex/main.prumo create mode 100644 fixtures/whatif/artifacts/lualatex/pdffonts.txt create mode 100644 fixtures/whatif/artifacts/lualatex/report.txt create mode 100644 fixtures/whatif/artifacts/pdflatex/fontmap.txt create mode 100644 fixtures/whatif/artifacts/pdflatex/golden.json create mode 100644 fixtures/whatif/artifacts/pdflatex/kpsewhich.txt create mode 100644 fixtures/whatif/artifacts/pdflatex/main.fls create mode 100644 fixtures/whatif/artifacts/pdflatex/main.log create mode 100644 fixtures/whatif/artifacts/pdflatex/main.prumo create mode 100644 fixtures/whatif/artifacts/pdflatex/pdffonts.txt create mode 100644 fixtures/whatif/artifacts/pdflatex/report.txt create mode 100644 fixtures/whatif/artifacts/xelatex/fontmap.txt create mode 100644 fixtures/whatif/artifacts/xelatex/golden.json create mode 100644 fixtures/whatif/artifacts/xelatex/kpsewhich.txt create mode 100644 fixtures/whatif/artifacts/xelatex/main.fls create mode 100644 fixtures/whatif/artifacts/xelatex/main.log create mode 100644 fixtures/whatif/artifacts/xelatex/main.prumo create mode 100644 fixtures/whatif/artifacts/xelatex/pdffonts.txt create mode 100644 fixtures/whatif/artifacts/xelatex/report.txt create mode 100644 fixtures/whatif/main.tex create mode 100644 src/whatif.rs diff --git a/docker/Dockerfile.tex b/docker/Dockerfile.tex index 899dd7a..92f2d72 100644 --- a/docker/Dockerfile.tex +++ b/docker/Dockerfile.tex @@ -22,6 +22,9 @@ RUN tlmgr install cm-super courier fontname siunitx translations csquotes \ && tlmgr path add +# mylatexformat: `whatif` precompiles the whatif fixture's preamble with it. +RUN tlmgr install mylatexformat + # Fixture builds run as the invoking user with HOME under /tmp; keep tlmgr quiet about it. ENV HOME=/tmp/home diff --git a/docs/03-architecture.md b/docs/03-architecture.md index 7f9f44d..86f7a40 100644 --- a/docs/03-architecture.md +++ b/docs/03-architecture.md @@ -24,8 +24,9 @@ `prumo run` produces the model and caches it under `//`, where `` defaults to a per-document directory in the user's cache (`run::default_out`; `prumo where`). Every report command either -uses the cache or runs the build. `prune` merges models across a corpus. `unused` is the only -command that runs more than one build. +uses the cache or runs the build. `prune` merges models across a corpus. `compare` builds once +per engine. `unused` and `whatif` rebuild the document with one change at a time; `whatif`'s +timed builds run without the probe. ## BuildModel @@ -108,6 +109,7 @@ prumo fonts [--json] prumo packages [--tlpdb PATH] [--json] # tlmgr package set, with the file that caused each prumo prune [--tlpdb PATH] [--keep LIST] [--keep-hyphenation all|used] [--apply --yes] prumo unused [--build ] [--jobs N] [--compare text|render] # phase 2 +prumo whatif [--try preamble,draft,externalize] [--runs N] [--shell-escape] [--engine E] [--json] prumo doctor [--check-log] # engines, kernel date, pdffonts, tlpdb, recorder prumo where [] [--out DIR] # the cache root, or a document's artifact directory prumo clean [] [--stale] [--yes] [--json] # list the cache (sizes, documents); delete with --yes @@ -163,6 +165,7 @@ prumo/ report/ text, json, dot renderers prune.rs corpus merge, keep set, plan unused.rs phase 2 + whatif.rs measured speed-ups: precompiled preamble, draft mode, externalized pictures tex/prumo.sty embedded via include_str! scripts/prumo_report.py reference, frozen fixtures/ documents + checked-in build artifacts for no-TeX tests diff --git a/fixtures/whatif/artifacts/lualatex/fontmap.txt b/fixtures/whatif/artifacts/lualatex/fontmap.txt new file mode 100644 index 0000000..5033efc --- /dev/null +++ b/fixtures/whatif/artifacts/lualatex/fontmap.txt @@ -0,0 +1,27 @@ +cmex10 CMEX10 +L3 programming layer <2026-09-09> + *********** + + +Package rerunfilecheck Warning: File `main.out' has changed. +(rerunfilecheck) Rerun to get outlines right +(rerunfilecheck) or use package `bookmark'. + +Package rerunfilecheck Info: Checksums for `main.out': +(rerunfilecheck) Before: +(rerunfilecheck) After: 5E5F33D6957BF682D675E947AD6FD41B;91. +) + +Here is how much of LuaTeX's memory you used: + 24869 strings out of 475522 + 125036,1144653 words of node,token memory allocated + 501 words of node memory still in use: + 6 hlist, 2 vlist, 2 rule, 2 glue, 4 kern, 1 glyph, 7 attribute, 54 glue_spec, + 7 attribute_list, 2 write nodes + avail lists: 1:1,2:254,3:140,4:15,5:66,6:12,7:448,8:4,9:153,11:3 + 47364 multiletter control sequences out of 65536+600000 + 29 fonts using 1732383 bytes + 114i,6n,104p,552b,929s stack positions out of 10000i,1000n,20000p,200000b,200000s + +Output written on main.pdf (1 page, 25905 bytes). + +PDF statistics: 41 PDF objects out of 1000 (max. 8388607) + 27 compressed objects within 1 object stream + 3 named destinations out of 1000 (max. 1048576) + 16 words of extra memory for PDF output out of 10000 (max. 100000000) + diff --git a/fixtures/whatif/artifacts/lualatex/main.prumo b/fixtures/whatif/artifacts/lualatex/main.prumo new file mode 100644 index 0000000..60a8aca --- /dev/null +++ b/fixtures/whatif/artifacts/lualatex/main.prumo @@ -0,0 +1,384 @@ +start 13581 0 luatex 2026-11-01 +file+ 13582 0 main.tex +cls+ 13613 1 article +file+ 13632 1 article.cls +file+ 13664 2 size10.clo +file- 13945 2 size10.clo +file- 14001 1 article.cls +cls- 14014 1 article +pkg+ 14045 1 amsmath +opts 14045 1 +file+ 14064 1 amsmath.sty +pkg+ 14116 2 amstext +opts 14116 2 +file+ 14135 2 amstext.sty +pkg+ 14165 3 amsgen +opts 14165 3 +file+ 14183 3 amsgen.sty +file- 14197 3 amsgen.sty +pkg- 14209 3 amsgen +file- 14218 2 amstext.sty +pkg- 14230 2 amstext +pkg+ 14257 2 amsbsy +opts 14258 2 +file+ 14276 2 amsbsy.sty +file- 14297 2 amsbsy.sty +pkg- 14309 2 amsbsy +pkg+ 14334 2 amsopn +opts 14335 2 namelimits, +file+ 14353 2 amsopn.sty +file- 14378 2 amsopn.sty +pkg- 14390 2 amsopn +file- 14545 1 amsmath.sty +pkg- 14558 1 amsmath +pkg+ 14587 1 siunitx +opts 14587 1 +file+ 14606 1 siunitx.sty +pkg+ 15156 2 translations +opts 15156 2 +file+ 15178 2 translations.sty +pkg+ 15212 3 etoolbox +opts 15212 3 +file+ 15231 3 etoolbox.sty +file- 15320 3 etoolbox.sty +pkg- 15333 3 etoolbox +pkg+ 15363 3 pdftexcmds +opts 15364 3 +file+ 15384 3 pdftexcmds.sty +pkg+ 15421 4 infwarerr +opts 15422 4 +file+ 15442 4 infwarerr.sty +file- 15457 4 infwarerr.sty +pkg- 15469 4 infwarerr +pkg+ 15498 4 iftex +opts 15498 4 +file+ 15516 4 iftex.sty +file- 15535 4 iftex.sty +pkg- 15547 4 iftex +pkg+ 15576 4 ltxcmds +opts 15577 4 +file+ 15595 4 ltxcmds.sty +file- 15617 4 ltxcmds.sty +pkg- 15630 4 ltxcmds +file- 15665 3 pdftexcmds.sty +pkg- 15678 3 pdftexcmds +file- 16100 2 translations.sty +pkg- 16114 2 translations +pkg+ 17295 2 array +opts 17295 2 +file+ 17314 2 array.sty +file- 17354 2 array.sty +pkg- 17366 2 array +file- 17665 1 siunitx.sty +pkg- 17678 1 siunitx +pkg+ 17709 1 booktabs +opts 17709 1 +file+ 17729 1 booktabs.sty +file- 17754 1 booktabs.sty +pkg- 17767 1 booktabs +pkg+ 17794 1 tikz +opts 17794 1 +file+ 17812 1 tikz.sty +pkg+ 17840 2 pgf +opts 17840 2 +file+ 17858 2 pgf.sty +pkg+ 17888 3 pgfrcs +opts 17888 3 +file+ 17907 3 pgfrcs.sty +file+ 17928 4 pgfutil-common.tex +file- 17959 4 pgfutil-common.tex +file+ 17977 4 pgfutil-latex.def +file- 18004 4 pgfutil-latex.def +file+ 18022 4 pgfrcs.code.tex +file+ 18049 5 pgf.revision.tex +file- 18059 5 pgf.revision.tex +file- 18068 4 pgfrcs.code.tex +file- 18073 3 pgfrcs.sty +pkg- 18085 3 pgfrcs +pkg+ 18118 3 pgfcore +opts 18119 3 +file+ 18138 3 pgfcore.sty +pkg+ 18170 4 graphicx +opts 18170 4 +file+ 18189 4 graphicx.sty +pkg+ 18221 5 keyval +opts 18221 5 +file+ 18239 5 keyval.sty +file- 18254 5 keyval.sty +pkg- 18266 5 keyval +pkg+ 18296 5 graphics +opts 18296 5 +file+ 18316 5 graphics.sty +pkg+ 18348 6 trig +opts 18348 6 +file+ 18366 6 trig.sty +file- 18379 6 trig.sty +pkg- 18391 6 trig +file+ 18417 6 graphics.cfg +file- 18429 6 graphics.cfg +file+ 18455 6 luatex.def +file- 18506 6 luatex.def +file- 18523 5 graphics.sty +pkg- 18545 5 graphics +file- 18562 4 graphicx.sty +pkg- 18574 4 graphicx +pkg+ 18604 4 pgfsys +opts 18604 4 +file+ 18623 4 pgfsys.sty +file+ 18657 5 pgfsys.code.tex +file+ 18679 6 pgfkeys.code.tex +file+ 18808 7 pgfkeyslibraryfiltered.code.tex +file- 18900 7 pgfkeyslibraryfiltered.code.tex +file- 18911 6 pgfkeys.code.tex +file+ 19001 6 pgf.cfg +file- 19024 6 pgf.cfg +file+ 19054 6 pgfsys-luatex.def +file+ 19099 7 pgfsys-common-pdf.def +file- 19121 7 pgfsys-common-pdf.def +file- 19138 6 pgfsys-luatex.def +file- 19155 5 pgfsys.code.tex +file+ 19177 5 pgfsyssoftpath.code.tex +file- 19193 5 pgfsyssoftpath.code.tex +file+ 19213 5 pgfsysprotocol.code.tex +file- 19224 5 pgfsysprotocol.code.tex +file- 19230 4 pgfsys.sty +pkg- 19242 4 pgfsys +pkg+ 19279 4 xcolor +opts 19279 4 +file+ 19298 4 xcolor.sty +file+ 19670 5 color.cfg +file- 19685 5 color.cfg +file+ 19760 5 mathcolor.ltx +file- 19784 5 mathcolor.ltx +file- 19870 4 xcolor.sty +pkg- 19887 4 xcolor +file+ 19913 4 pgfcore.code.tex +file+ 19940 5 pgfmath.code.tex +file+ 19964 6 pgfmathutil.code.tex +file- 19985 6 pgfmathutil.code.tex +file+ 20006 6 pgfmathparser.code.tex +file- 20056 6 pgfmathparser.code.tex +file+ 20079 6 pgfmathfunctions.code.tex +file- 20110 6 pgfmathfunctions.code.tex +file+ 20135 6 pgfmathfunctions.basic.code.tex +file- 20184 6 pgfmathfunctions.basic.code.tex +file+ 20211 6 pgfmathfunctions.trigonometric.code.tex +file- 20366 6 pgfmathfunctions.trigonometric.code.tex +file+ 20393 6 pgfmathfunctions.random.code.tex +file- 20413 6 pgfmathfunctions.random.code.tex +file+ 20439 6 pgfmathfunctions.comparison.code.tex +file- 20471 6 pgfmathfunctions.comparison.code.tex +file+ 20495 6 pgfmathfunctions.base.code.tex +file- 20520 6 pgfmathfunctions.base.code.tex +file+ 20544 6 pgfmathfunctions.round.code.tex +file- 20562 6 pgfmathfunctions.round.code.tex +file+ 20586 6 pgfmathfunctions.misc.code.tex +file- 20617 6 pgfmathfunctions.misc.code.tex +file+ 20646 6 pgfmathfunctions.integerarithmetics.code.tex +file- 20665 6 pgfmathfunctions.integerarithmetics.code.tex +file+ 20687 6 pgfmathcalc.code.tex +file- 20705 6 pgfmathcalc.code.tex +file+ 20725 6 pgfmathfloat.code.tex +file- 20910 6 pgfmathfloat.code.tex +file- 20917 5 pgfmath.code.tex +file+ 20939 5 pgfint.code.tex +file- 20953 5 pgfint.code.tex +file+ 20974 5 pgfcorepoints.code.tex +file- 21034 5 pgfcorepoints.code.tex +file+ 21059 5 pgfcorepathconstruct.code.tex +file- 21100 5 pgfcorepathconstruct.code.tex +file+ 21123 5 pgfcorepathusage.code.tex +file- 21166 5 pgfcorepathusage.code.tex +file+ 21188 5 pgfcorescopes.code.tex +file- 21231 5 pgfcorescopes.code.tex +file+ 21255 5 pgfcoregraphicstate.code.tex +file- 21271 5 pgfcoregraphicstate.code.tex +file+ 21295 5 pgfcoretransformations.code.tex +file- 21320 5 pgfcoretransformations.code.tex +file+ 21342 5 pgfcorequick.code.tex +file- 21355 5 pgfcorequick.code.tex +file+ 21376 5 pgfcoreobjects.code.tex +file- 21387 5 pgfcoreobjects.code.tex +file+ 21410 5 pgfcorepathprocessing.code.tex +file- 21432 5 pgfcorepathprocessing.code.tex +file+ 21453 5 pgfcorearrows.code.tex +file- 21636 5 pgfcorearrows.code.tex +file+ 21659 5 pgfcoreshade.code.tex +file- 21692 5 pgfcoreshade.code.tex +file+ 21713 5 pgfcoreimage.code.tex +file- 21744 5 pgfcoreimage.code.tex +file+ 21767 5 pgfcoreexternal.code.tex +file- 21821 5 pgfcoreexternal.code.tex +file+ 21843 5 pgfcorelayers.code.tex +file- 21858 5 pgfcorelayers.code.tex +file+ 21881 5 pgfcoretransparency.code.tex +file- 21908 5 pgfcoretransparency.code.tex +file+ 21930 5 pgfcorepatterns.code.tex +file- 21947 5 pgfcorepatterns.code.tex +file+ 21967 5 pgfcorerdf.code.tex +file- 21982 5 pgfcorerdf.code.tex +file- 21991 4 pgfcore.code.tex +file- 21997 3 pgfcore.sty +pkg- 22010 3 pgfcore +file+ 22035 3 pgfmoduleshapes.code.tex +file- 22117 3 pgfmoduleshapes.code.tex +file+ 22142 3 pgfmoduleplot.code.tex +file- 22195 3 pgfmoduleplot.code.tex +pkg+ 22234 3 pgfcomp-version-0-65 +opts 22234 3 +file+ 22260 3 pgfcomp-version-0-65.sty +file- 22355 3 pgfcomp-version-0-65.sty +pkg- 22371 3 pgfcomp-version-0-65 +pkg+ 22410 3 pgfcomp-version-1-18 +opts 22411 3 +file+ 22437 3 pgfcomp-version-1-18.sty +file- 22460 3 pgfcomp-version-1-18.sty +pkg- 22476 3 pgfcomp-version-1-18 +file- 22483 2 pgf.sty +pkg- 22496 2 pgf +pkg+ 22525 2 pgffor +opts 22526 2 +file+ 22546 2 pgffor.sty +pkg+ 22587 3 pgfkeys +opts 22587 3 +file+ 22608 3 pgfkeys.sty +file+ 22628 4 pgfkeys.code.tex +file- 22638 4 pgfkeys.code.tex +file- 22644 3 pgfkeys.sty +pkg- 22657 3 pgfkeys +pkg+ 22688 3 pgfmath +opts 22688 3 +file+ 22709 3 pgfmath.sty +file+ 22746 4 pgfmath.code.tex +file- 22757 4 pgfmath.code.tex +file- 22763 3 pgfmath.sty +pkg- 22776 3 pgfmath +file+ 22797 3 pgffor.code.tex +file- 22842 3 pgffor.code.tex +file- 22848 2 pgffor.sty +pkg- 22861 2 pgffor +file+ 22883 2 tikz.code.tex +file+ 22914 3 pgflibraryplothandlers.code.tex +file- 22958 3 pgflibraryplothandlers.code.tex +file+ 23238 3 pgfmodulematrix.code.tex +file- 23265 3 pgfmodulematrix.code.tex +file+ 23522 3 tikzlibrarytopaths.code.tex +file- 23565 3 tikzlibrarytopaths.code.tex +file- 23572 2 tikz.code.tex +file- 23578 1 tikz.sty +pkg- 23591 1 tikz +file+ 23635 1 pgflibraryarrows.meta.code.tex +file- 24264 1 pgflibraryarrows.meta.code.tex +file+ 24291 1 tikzlibrarypositioning.code.tex +file- 24321 1 tikzlibrarypositioning.code.tex +file+ 24346 1 tikzlibrarycalc.code.tex +file- 24371 1 tikzlibrarycalc.code.tex +pkg+ 24450 1 hyperref +opts 24450 1 +file+ 24472 1 hyperref.sty +pkg+ 24525 2 kvsetkeys +opts 24525 2 +file+ 24547 2 kvsetkeys.sty +file- 24568 2 kvsetkeys.sty +pkg- 24581 2 kvsetkeys +pkg+ 24616 2 kvdefinekeys +opts 24617 2 +file+ 24639 2 kvdefinekeys.sty +file- 24654 2 kvdefinekeys.sty +pkg- 24667 2 kvdefinekeys +pkg+ 24700 2 pdfescape +opts 24700 2 +file+ 24722 2 pdfescape.sty +file- 24764 2 pdfescape.sty +pkg- 24777 2 pdfescape +pkg+ 24807 2 hycolor +opts 24808 2 +file+ 24829 2 hycolor.sty +file- 24936 2 hycolor.sty +pkg- 24949 2 hycolor +pkg+ 24981 2 nameref +opts 24981 2 +file+ 25003 2 nameref.sty +pkg+ 25040 3 refcount +opts 25040 3 +file+ 25061 3 refcount.sty +file- 25102 3 refcount.sty +pkg- 25116 3 refcount +pkg+ 25152 3 gettitlestring +opts 25152 3 +file+ 25175 3 gettitlestring.sty +pkg+ 25213 4 kvoptions +opts 25214 4 +file+ 25236 4 kvoptions.sty +file- 25295 4 kvoptions.sty +pkg- 25309 4 kvoptions +file- 25339 3 gettitlestring.sty +pkg- 25353 3 gettitlestring +file- 25549 2 nameref.sty +pkg- 25563 2 nameref +pkg+ 25606 2 stringenc +opts 25607 2 +file+ 25629 2 stringenc.sty +file- 25690 2 stringenc.sty +pkg- 25703 2 stringenc +file+ 25755 2 pd1enc.def +file- 25823 2 pd1enc.def +pkg+ 25856 2 intcalc +opts 25857 2 +file+ 25878 2 intcalc.sty +file- 25898 2 intcalc.sty +pkg- 25911 2 intcalc +file+ 25989 2 puenc.def +file- 26422 2 puenc.def +pkg+ 26567 2 url +opts 26568 2 +file+ 26587 2 url.sty +file- 26611 2 url.sty +pkg- 26624 2 url +pkg+ 26693 2 bitset +opts 26693 2 +file+ 26714 2 bitset.sty +pkg+ 26775 3 bigintcalc +opts 26775 3 +file+ 26797 3 bigintcalc.sty +file- 26843 3 bigintcalc.sty +pkg- 26857 3 bigintcalc +file- 26915 2 bitset.sty +pkg- 26928 2 bitset +file- 27286 1 hyperref.sty +file+ 27309 1 hluatex.def +pkg+ 27391 2 rerunfilecheck +opts 27392 2 +file+ 27416 2 rerunfilecheck.sty +pkg+ 27532 3 uniquecounter +opts 27532 3 +file+ 27556 3 uniquecounter.sty +file- 27596 3 uniquecounter.sty +pkg- 27611 3 uniquecounter +file- 27621 2 rerunfilecheck.sty +pkg- 27634 2 rerunfilecheck +file- 27647 1 hluatex.def +pkg- 27676 1 hyperref +predoc 28244 1 +file+ 28304 1 l3backend-luatex.def +file- 28356 1 l3backend-luatex.def +begindoc 28379 1 +file+ 28683 1 translations-basic-dictionary-english.trsl +file- 28813 1 translations-basic-dictionary-english.trsl +file+ 28843 1 supp-pdf.mkii +file- 28888 1 supp-pdf.mkii +pkg+ 28915 1 epstopdf-base +opts 28916 1 +file+ 28933 1 epstopdf-base.sty +file+ 28983 2 epstopdf-sys.cfg +file- 28989 2 epstopdf-sys.cfg +file- 28999 1 epstopdf-base.sty +pkg- 29008 1 epstopdf-base +docstart 29251 1 +env+ 33865 1 tikzpicture +env- 34107 1 tikzpicture +enddoc 34154 1 +ship 34174 1 1 +page 34487 1 1 +lastpage 34497 1 +end 34530 1 diff --git a/fixtures/whatif/artifacts/lualatex/pdffonts.txt b/fixtures/whatif/artifacts/lualatex/pdffonts.txt new file mode 100644 index 0000000..20bdc13 --- /dev/null +++ b/fixtures/whatif/artifacts/lualatex/pdffonts.txt @@ -0,0 +1,6 @@ +name type encoding emb sub uni object ID +------------------------------------ ----------------- ---------------- --- --- --- --------- +CHCSTU+LMRoman12-Bold CID Type 0C Identity-H yes yes yes 12 0 +IWFNKH+LMRoman10-Regular CID Type 0C Identity-H yes yes yes 13 0 +HMRTCK+CMR10 Type 1 Builtin yes yes yes 14 0 +CUJHND+CMMI10 Type 1 Builtin yes yes yes 15 0 diff --git a/fixtures/whatif/artifacts/lualatex/report.txt b/fixtures/whatif/artifacts/lualatex/report.txt new file mode 100644 index 0000000..00fcbbd --- /dev/null +++ b/fixtures/whatif/artifacts/lualatex/report.txt @@ -0,0 +1,76 @@ +== prumo report: main (total 320 ms, preamble 226 ms, 1 pages, 204 files read) + +-- slowest units (exclusive) -- + 104.6 ms 319.6 ms incl file main.tex + 31.5 ms 47.2 ms incl pkg siunitx + 15.8 ms 49.2 ms incl pkg hyperref + 9.6 ms 9.6 ms incl file pgflibraryarrows.meta.code.tex + 8.8 ms 10.5 ms incl file tikz.code.tex + 8.7 ms 9.3 ms incl pkg xcolor + 8.0 ms 14.6 ms incl pkg translations + 6.7 ms 31.7 ms incl file pgfcore.code.tex + 6.6 ms 6.6 ms incl file puenc.def + 5.1 ms 15.1 ms incl file pgfmath.code.tex + 4.7 ms 8.9 ms incl pkg nameref + 4.4 ms 7.8 ms incl pkg amsmath + +-- phases -- + startup 207.2 ms + preamble 223.7 ms + begin-document 15.4 ms + typesetting 74.8 ms + shipout 4.8 ms + end-document 1.0 ms + +-- environments -- + 3.7 ms 1x max 3.7 ms tikzpicture + +-- pages -- + page 1: 79.9 ms + +-- load tree -- + main.tex (319.6 ms incl, 104.6 ms excl) + article (6.1 ms incl, 1.8 ms excl) + size10.clo (4.3 ms incl, 4.3 ms excl) + amsmath (7.8 ms incl, 4.4 ms excl) + amstext (1.7 ms incl, 1.1 ms excl) + amsgen (0.7 ms incl, 0.7 ms excl) + amsbsy (0.8 ms incl, 0.8 ms excl) + amsopn (0.9 ms incl, 0.9 ms excl) + siunitx (47.2 ms incl, 31.5 ms excl) + translations (14.6 ms incl, 8.0 ms excl) + etoolbox (1.8 ms incl, 1.8 ms excl) + pdftexcmds (4.8 ms incl, 2.5 ms excl) + infwarerr (0.7 ms incl, 0.7 ms excl) + iftex (0.7 ms incl, 0.7 ms excl) + ltxcmds (0.8 ms incl, 0.8 ms excl) + array (1.1 ms incl, 1.1 ms excl) + booktabs (0.9 ms incl, 0.9 ms excl) + tikz (88.5 ms incl, 1.8 ms excl) + pgf (71.0 ms incl, 3.5 ms excl) + pgfrcs (3.0 ms incl, 1.4 ms excl) + pgfutil-common.tex (0.5 ms incl, 0.5 ms excl) + pgfutil-latex.def (0.4 ms incl, 0.4 ms excl) + pgfrcs.code.tex (0.7 ms incl, 0.5 ms excl) + pgf.revision.tex (0.2 ms incl, 0.2 ms excl) + pgfcore (59.4 ms incl, 2.5 ms excl) + graphicx (6.2 ms incl, 1.7 ms excl) + keyval (0.7 ms incl, 0.7 ms excl) + graphics (3.8 ms incl, 2.2 ms excl) + trig (0.7 ms incl, 0.7 ms excl) + graphics.cfg (0.2 ms incl, 0.2 ms excl) + luatex.def (0.8 ms incl, 0.8 ms excl) + pgfsys (9.7 ms incl, 1.7 ms excl) + pgfsys.code.tex (7.6 ms incl, 2.4 ms excl) + pgfkeys.code.tex (3.5 ms incl, 2.1 ms excl) + pgfkeyslibraryfiltered.code.tex (1.4 ms incl, 1.4 ms excl) + pgf.cfg (0.4 ms incl, 0.4 ms excl) + pgfsys-luatex.def (1.3 ms incl, 0.9 ms excl) + pgfsys-common-pdf.def (0.3 ms incl, 0.3 ms excl) + pgfsyssoftpath.code.tex (0.2 ms incl, 0.2 ms excl) + pgfsysprotocol.code.tex (0.2 ms incl, 0.2 ms excl) + +-- fonts -- + TFMs loaded: 11, embedded in PDF: 4, Type 3: 0, bitmap .pk read: 0 + loaded but not embedded: cmex10, cmex7, cmmi5, cmmi7, cmr5, cmr7, cmsy10, cmsy5, cmsy7 + OpenType, opened by the engine: LMRoman10-Regular <- lmroman10-regular.luc, LMRoman12-Bold <- lmroman12-bold.otf diff --git a/fixtures/whatif/artifacts/pdflatex/fontmap.txt b/fixtures/whatif/artifacts/pdflatex/fontmap.txt new file mode 100644 index 0000000..3b18e1b --- /dev/null +++ b/fixtures/whatif/artifacts/pdflatex/fontmap.txt @@ -0,0 +1,5 @@ +cmbx12 CMBX12 +L3 programming layer <2026-09-09> + *********** + + +Package rerunfilecheck Warning: File `main.out' has changed. +(rerunfilecheck) Rerun to get outlines right +(rerunfilecheck) or use package `bookmark'. + +Package rerunfilecheck Info: Checksums for `main.out': +(rerunfilecheck) Before: +(rerunfilecheck) After: 5E5F33D6957BF682D675E947AD6FD41B;91. + ) +Here is how much of TeX's memory you used: + 24488 strings out of 468449 + 518436 string characters out of 5440369 + 870848 words of memory out of 5000000 + 53425 multiletter control sequences out of 15000+600000 + 629548 words of font info for 44 fonts, out of 8000000 for 9000 + 319 hyphenation exceptions out of 8191 + 114i,6n,110p,553b,1032s stack positions out of 10000i,1000n,20000p,200000b,200000s + + +Output written on main.pdf (1 page, 32857 bytes). +PDF statistics: + 32 PDF objects out of 1000 (max. 8388607) + 22 compressed objects within 1 object stream + 3 named destinations out of 1000 (max. 500000) + 13 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/fixtures/whatif/artifacts/pdflatex/main.prumo b/fixtures/whatif/artifacts/pdflatex/main.prumo new file mode 100644 index 0000000..e324077 --- /dev/null +++ b/fixtures/whatif/artifacts/pdflatex/main.prumo @@ -0,0 +1,384 @@ +start 2341 0 pdftex 2026-11-01 +file+ 2341 0 main.tex +cls+ 2372 1 article +file+ 2392 1 article.cls +file+ 2422 2 size10.clo +file- 2446 2 size10.clo +file- 2499 1 article.cls +cls- 2511 1 article +pkg+ 2541 1 amsmath +opts 2541 1 +file+ 2561 1 amsmath.sty +pkg+ 2609 2 amstext +opts 2610 2 +file+ 2630 2 amstext.sty +pkg+ 2660 3 amsgen +opts 2660 3 +file+ 2679 3 amsgen.sty +file- 2692 3 amsgen.sty +pkg- 2704 3 amsgen +file- 2713 2 amstext.sty +pkg- 2725 2 amstext +pkg+ 2754 2 amsbsy +opts 2754 2 +file+ 2774 2 amsbsy.sty +file- 2794 2 amsbsy.sty +pkg- 2806 2 amsbsy +pkg+ 2832 2 amsopn +opts 2833 2 namelimits, +file+ 2852 2 amsopn.sty +file- 2877 2 amsopn.sty +pkg- 2889 2 amsopn +file- 3034 1 amsmath.sty +pkg- 3046 1 amsmath +pkg+ 3076 1 siunitx +opts 3076 1 +file+ 3096 1 siunitx.sty +pkg+ 3649 2 translations +opts 3649 2 +file+ 3671 2 translations.sty +pkg+ 3707 3 etoolbox +opts 3708 3 +file+ 3728 3 etoolbox.sty +file- 3813 3 etoolbox.sty +pkg- 3826 3 etoolbox +pkg+ 3858 3 pdftexcmds +opts 3858 3 +file+ 3879 3 pdftexcmds.sty +pkg+ 3918 4 infwarerr +opts 3918 4 +file+ 3939 4 infwarerr.sty +file- 3952 4 infwarerr.sty +pkg- 3966 4 infwarerr +pkg+ 3994 4 iftex +opts 3995 4 +file+ 4013 4 iftex.sty +file- 4029 4 iftex.sty +pkg- 4041 4 iftex +pkg+ 4071 4 ltxcmds +opts 4071 4 +file+ 4091 4 ltxcmds.sty +file- 4113 4 ltxcmds.sty +pkg- 4126 4 ltxcmds +file- 4140 3 pdftexcmds.sty +pkg- 4153 3 pdftexcmds +file- 4574 2 translations.sty +pkg- 4589 2 translations +pkg+ 5768 2 array +opts 5768 2 +file+ 5787 2 array.sty +file- 5823 2 array.sty +pkg- 5835 2 array +file- 6132 1 siunitx.sty +pkg- 6144 1 siunitx +pkg+ 6175 1 booktabs +opts 6175 1 +file+ 6196 1 booktabs.sty +file- 6218 1 booktabs.sty +pkg- 6231 1 booktabs +pkg+ 6258 1 tikz +opts 6258 1 +file+ 6277 1 tikz.sty +pkg+ 6304 2 pgf +opts 6304 2 +file+ 6321 2 pgf.sty +pkg+ 6351 3 pgfrcs +opts 6351 3 +file+ 6370 3 pgfrcs.sty +file+ 6391 4 pgfutil-common.tex +file- 6419 4 pgfutil-common.tex +file+ 6439 4 pgfutil-latex.def +file- 6464 4 pgfutil-latex.def +file+ 6483 4 pgfrcs.code.tex +file+ 6510 5 pgf.revision.tex +file- 6520 5 pgf.revision.tex +file- 6530 4 pgfrcs.code.tex +file- 6536 3 pgfrcs.sty +pkg- 6547 3 pgfrcs +pkg+ 6580 3 pgfcore +opts 6581 3 +file+ 6601 3 pgfcore.sty +pkg+ 6633 4 graphicx +opts 6633 4 +file+ 6653 4 graphicx.sty +pkg+ 6686 5 keyval +opts 6687 5 +file+ 6705 5 keyval.sty +file- 6718 5 keyval.sty +pkg- 6730 5 keyval +pkg+ 6760 5 graphics +opts 6760 5 +file+ 6780 5 graphics.sty +pkg+ 6810 6 trig +opts 6810 6 +file+ 6829 6 trig.sty +file- 6840 6 trig.sty +pkg- 6852 6 trig +file+ 6878 6 graphics.cfg +file- 6889 6 graphics.cfg +file+ 6914 6 pdftex.def +file- 6963 6 pdftex.def +file- 6979 5 graphics.sty +pkg- 7001 5 graphics +file- 7017 4 graphicx.sty +pkg- 7030 4 graphicx +pkg+ 7059 4 pgfsys +opts 7060 4 +file+ 7079 4 pgfsys.sty +file+ 7112 5 pgfsys.code.tex +file+ 7136 6 pgfkeys.code.tex +file+ 7238 7 pgfkeyslibraryfiltered.code.tex +file- 7294 7 pgfkeyslibraryfiltered.code.tex +file- 7300 6 pgfkeys.code.tex +file+ 7350 6 pgf.cfg +file- 7359 6 pgf.cfg +file+ 7378 6 pgfsys-pdftex.def +file+ 7401 7 pgfsys-common-pdf.def +file- 7415 7 pgfsys-common-pdf.def +file- 7429 6 pgfsys-pdftex.def +file- 7444 5 pgfsys.code.tex +file+ 7465 5 pgfsyssoftpath.code.tex +file- 7480 5 pgfsyssoftpath.code.tex +file+ 7501 5 pgfsysprotocol.code.tex +file- 7512 5 pgfsysprotocol.code.tex +file- 7518 4 pgfsys.sty +pkg- 7530 4 pgfsys +pkg+ 7567 4 xcolor +opts 7568 4 +file+ 7588 4 xcolor.sty +file+ 7967 5 color.cfg +file- 7981 5 color.cfg +file+ 8052 5 mathcolor.ltx +file- 8074 5 mathcolor.ltx +file- 8158 4 xcolor.sty +pkg- 8170 4 xcolor +file+ 8192 4 pgfcore.code.tex +file+ 8216 5 pgfmath.code.tex +file+ 8240 6 pgfmathutil.code.tex +file- 8259 6 pgfmathutil.code.tex +file+ 8280 6 pgfmathparser.code.tex +file- 8323 6 pgfmathparser.code.tex +file+ 8345 6 pgfmathfunctions.code.tex +file- 8390 6 pgfmathfunctions.code.tex +file+ 8416 6 pgfmathfunctions.basic.code.tex +file- 8460 6 pgfmathfunctions.basic.code.tex +file+ 8487 6 pgfmathfunctions.trigonometric.code.tex +file- 8612 6 pgfmathfunctions.trigonometric.code.tex +file+ 8636 6 pgfmathfunctions.random.code.tex +file- 8654 6 pgfmathfunctions.random.code.tex +file+ 8679 6 pgfmathfunctions.comparison.code.tex +file- 8708 6 pgfmathfunctions.comparison.code.tex +file+ 8731 6 pgfmathfunctions.base.code.tex +file- 8753 6 pgfmathfunctions.base.code.tex +file+ 8777 6 pgfmathfunctions.round.code.tex +file- 8793 6 pgfmathfunctions.round.code.tex +file+ 8816 6 pgfmathfunctions.misc.code.tex +file- 8845 6 pgfmathfunctions.misc.code.tex +file+ 8873 6 pgfmathfunctions.integerarithmetics.code.tex +file- 8889 6 pgfmathfunctions.integerarithmetics.code.tex +file+ 8909 6 pgfmathcalc.code.tex +file- 8927 6 pgfmathcalc.code.tex +file+ 8947 6 pgfmathfloat.code.tex +file- 9117 6 pgfmathfloat.code.tex +file- 9123 5 pgfmath.code.tex +file+ 9142 5 pgfint.code.tex +file- 9155 5 pgfint.code.tex +file+ 9175 5 pgfcorepoints.code.tex +file- 9227 5 pgfcorepoints.code.tex +file+ 9251 5 pgfcorepathconstruct.code.tex +file- 9288 5 pgfcorepathconstruct.code.tex +file+ 9310 5 pgfcorepathusage.code.tex +file- 9350 5 pgfcorepathusage.code.tex +file+ 9371 5 pgfcorescopes.code.tex +file- 9409 5 pgfcorescopes.code.tex +file+ 9433 5 pgfcoregraphicstate.code.tex +file- 9449 5 pgfcoregraphicstate.code.tex +file+ 9471 5 pgfcoretransformations.code.tex +file- 9495 5 pgfcoretransformations.code.tex +file+ 9516 5 pgfcorequick.code.tex +file- 9528 5 pgfcorequick.code.tex +file+ 9549 5 pgfcoreobjects.code.tex +file- 9560 5 pgfcoreobjects.code.tex +file+ 9583 5 pgfcorepathprocessing.code.tex +file- 9602 5 pgfcorepathprocessing.code.tex +file+ 9623 5 pgfcorearrows.code.tex +file- 9793 5 pgfcorearrows.code.tex +file+ 9814 5 pgfcoreshade.code.tex +file- 9844 5 pgfcoreshade.code.tex +file+ 9865 5 pgfcoreimage.code.tex +file- 9893 5 pgfcoreimage.code.tex +file+ 9915 5 pgfcoreexternal.code.tex +file- 9966 5 pgfcoreexternal.code.tex +file+ 9988 5 pgfcorelayers.code.tex +file- 10001 5 pgfcorelayers.code.tex +file+ 10024 5 pgfcoretransparency.code.tex +file- 10050 5 pgfcoretransparency.code.tex +file+ 10071 5 pgfcorepatterns.code.tex +file- 10087 5 pgfcorepatterns.code.tex +file+ 10107 5 pgfcorerdf.code.tex +file- 10121 5 pgfcorerdf.code.tex +file- 10130 4 pgfcore.code.tex +file- 10135 3 pgfcore.sty +pkg- 10148 3 pgfcore +file+ 10172 3 pgfmoduleshapes.code.tex +file- 10242 3 pgfmoduleshapes.code.tex +file+ 10265 3 pgfmoduleplot.code.tex +file- 10313 3 pgfmoduleplot.code.tex +pkg+ 10350 3 pgfcomp-version-0-65 +opts 10350 3 +file+ 10376 3 pgfcomp-version-0-65.sty +file- 10462 3 pgfcomp-version-0-65.sty +pkg- 10477 3 pgfcomp-version-0-65 +pkg+ 10514 3 pgfcomp-version-1-18 +opts 10514 3 +file+ 10539 3 pgfcomp-version-1-18.sty +file- 10560 3 pgfcomp-version-1-18.sty +pkg- 10575 3 pgfcomp-version-1-18 +file- 10582 2 pgf.sty +pkg- 10593 2 pgf +pkg+ 10621 2 pgffor +opts 10621 2 +file+ 10640 2 pgffor.sty +pkg+ 10678 3 pgfkeys +opts 10679 3 +file+ 10699 3 pgfkeys.sty +file+ 10717 4 pgfkeys.code.tex +file- 10727 4 pgfkeys.code.tex +file- 10733 3 pgfkeys.sty +pkg- 10746 3 pgfkeys +pkg+ 10775 3 pgfmath +opts 10775 3 +file+ 10795 3 pgfmath.sty +file+ 10832 4 pgfmath.code.tex +file- 10841 4 pgfmath.code.tex +file- 10847 3 pgfmath.sty +pkg- 10860 3 pgfmath +file+ 10880 3 pgffor.code.tex +file- 10921 3 pgffor.code.tex +file- 10927 2 pgffor.sty +pkg- 10939 2 pgffor +file+ 10959 2 tikz.code.tex +file+ 10989 3 pgflibraryplothandlers.code.tex +file- 11030 3 pgflibraryplothandlers.code.tex +file+ 11288 3 pgfmodulematrix.code.tex +file- 11313 3 pgfmodulematrix.code.tex +file+ 11549 3 tikzlibrarytopaths.code.tex +file- 11588 3 tikzlibrarytopaths.code.tex +file- 11595 2 tikz.code.tex +file- 11601 1 tikz.sty +pkg- 11613 1 tikz +file+ 11655 1 pgflibraryarrows.meta.code.tex +file- 12227 1 pgflibraryarrows.meta.code.tex +file+ 12253 1 tikzlibrarypositioning.code.tex +file- 12281 1 tikzlibrarypositioning.code.tex +file+ 12304 1 tikzlibrarycalc.code.tex +file- 12328 1 tikzlibrarycalc.code.tex +pkg+ 12403 1 hyperref +opts 12403 1 +file+ 12424 1 hyperref.sty +pkg+ 12476 2 kvsetkeys +opts 12476 2 +file+ 12497 2 kvsetkeys.sty +file- 12515 2 kvsetkeys.sty +pkg- 12529 2 kvsetkeys +pkg+ 12562 2 kvdefinekeys +opts 12562 2 +file+ 12585 2 kvdefinekeys.sty +file- 12599 2 kvdefinekeys.sty +pkg- 12612 2 kvdefinekeys +pkg+ 12643 2 pdfescape +opts 12643 2 +file+ 12664 2 pdfescape.sty +file- 12704 2 pdfescape.sty +pkg- 12717 2 pdfescape +pkg+ 12747 2 hycolor +opts 12747 2 +file+ 12768 2 hycolor.sty +file- 12873 2 hycolor.sty +pkg- 12885 2 hycolor +pkg+ 12915 2 nameref +opts 12915 2 +file+ 12936 2 nameref.sty +pkg+ 12968 3 refcount +opts 12968 3 +file+ 12989 3 refcount.sty +file- 13028 3 refcount.sty +pkg- 13041 3 refcount +pkg+ 13075 3 gettitlestring +opts 13075 3 +file+ 13099 3 gettitlestring.sty +pkg+ 13136 4 kvoptions +opts 13137 4 +file+ 13157 4 kvoptions.sty +file- 13215 4 kvoptions.sty +pkg- 13228 4 kvoptions +file- 13256 3 gettitlestring.sty +pkg- 13270 3 gettitlestring +file- 13460 2 nameref.sty +pkg- 13474 2 nameref +pkg+ 13516 2 stringenc +opts 13516 2 +file+ 13538 2 stringenc.sty +file- 13594 2 stringenc.sty +pkg- 13607 2 stringenc +file+ 13654 2 pd1enc.def +file- 13721 2 pd1enc.def +pkg+ 13754 2 intcalc +opts 13754 2 +file+ 13775 2 intcalc.sty +file- 13792 2 intcalc.sty +pkg- 13805 2 intcalc +file+ 13873 2 puenc.def +file- 14246 2 puenc.def +pkg+ 14373 2 url +opts 14373 2 +file+ 14392 2 url.sty +file- 14413 2 url.sty +pkg- 14425 2 url +pkg+ 14490 2 bitset +opts 14490 2 +file+ 14510 2 bitset.sty +pkg+ 14567 3 bigintcalc +opts 14568 3 +file+ 14590 3 bigintcalc.sty +file- 14632 3 bigintcalc.sty +pkg- 14645 3 bigintcalc +file- 14669 2 bitset.sty +pkg- 14681 2 bitset +file- 15012 1 hyperref.sty +file+ 15032 1 hpdftex.def +pkg+ 15106 2 rerunfilecheck +opts 15107 2 +file+ 15130 2 rerunfilecheck.sty +pkg+ 15243 3 uniquecounter +opts 15244 3 +file+ 15267 3 uniquecounter.sty +file- 15306 3 uniquecounter.sty +pkg- 15320 3 uniquecounter +file- 15330 2 rerunfilecheck.sty +pkg- 15342 2 rerunfilecheck +file- 15354 1 hpdftex.def +pkg- 15382 1 hyperref +predoc 15924 1 +file+ 15984 1 l3backend-pdftex.def +file- 16018 1 l3backend-pdftex.def +begindoc 16038 1 +file+ 16341 1 translations-basic-dictionary-english.trsl +file- 16467 1 translations-basic-dictionary-english.trsl +file+ 16496 1 supp-pdf.mkii +file- 16540 1 supp-pdf.mkii +pkg+ 16568 1 epstopdf-base +opts 16568 1 +file+ 16585 1 epstopdf-base.sty +file+ 16636 2 epstopdf-sys.cfg +file- 16644 2 epstopdf-sys.cfg +file- 16653 1 epstopdf-base.sty +pkg- 16662 1 epstopdf-base +docstart 16894 1 +env+ 17060 1 tikzpicture +env- 17262 1 tikzpicture +enddoc 17298 1 +ship 17312 1 1 +page 17703 1 1 +lastpage 17710 1 +end 17729 1 diff --git a/fixtures/whatif/artifacts/pdflatex/pdffonts.txt b/fixtures/whatif/artifacts/pdflatex/pdffonts.txt new file mode 100644 index 0000000..85ddb9e --- /dev/null +++ b/fixtures/whatif/artifacts/pdflatex/pdffonts.txt @@ -0,0 +1,5 @@ +name type encoding emb sub uni object ID +------------------------------------ ----------------- ---------------- --- --- --- --------- +KXHLBK+CMBX12 Type 1 Builtin yes yes yes 12 0 +BOTZNO+CMR10 Type 1 Builtin yes yes yes 13 0 +CUJHND+CMMI10 Type 1 Builtin yes yes yes 14 0 diff --git a/fixtures/whatif/artifacts/pdflatex/report.txt b/fixtures/whatif/artifacts/pdflatex/report.txt new file mode 100644 index 0000000..8e7f1a3 --- /dev/null +++ b/fixtures/whatif/artifacts/pdflatex/report.txt @@ -0,0 +1,76 @@ +== prumo report: main (total 235 ms, preamble 209 ms, 1 pages, 321 files read) + +-- slowest units (exclusive) -- + 36.1 ms 234.8 ms incl file main.tex + 31.4 ms 46.8 ms incl pkg siunitx + 14.6 ms 45.5 ms incl pkg hyperref + 8.7 ms 8.7 ms incl file pgflibraryarrows.meta.code.tex + 8.7 ms 9.2 ms incl pkg xcolor + 8.1 ms 9.7 ms incl file tikz.code.tex + 8.0 ms 14.3 ms incl pkg translations + 6.4 ms 29.6 ms incl file pgfcore.code.tex + 5.7 ms 5.7 ms incl file puenc.def + 4.9 ms 14.0 ms incl file pgfmath.code.tex + 4.4 ms 8.5 ms incl pkg nameref + 4.3 ms 7.7 ms incl pkg amsmath + +-- phases -- + startup 35.7 ms + preamble 207.3 ms + begin-document 14.8 ms + typesetting 6.2 ms + shipout 6.0 ms + end-document 0.6 ms + +-- environments -- + 3.1 ms 1x max 3.1 ms tikzpicture + +-- pages -- + page 1: 12.3 ms + +-- load tree -- + main.tex (234.8 ms incl, 36.1 ms excl) + article (2.1 ms incl, 1.8 ms excl) + size10.clo (0.4 ms incl, 0.4 ms excl) + amsmath (7.7 ms incl, 4.3 ms excl) + amstext (1.8 ms incl, 1.1 ms excl) + amsgen (0.7 ms incl, 0.7 ms excl) + amsbsy (0.8 ms incl, 0.8 ms excl) + amsopn (0.9 ms incl, 0.9 ms excl) + siunitx (46.8 ms incl, 31.4 ms excl) + translations (14.3 ms incl, 8.0 ms excl) + etoolbox (1.8 ms incl, 1.8 ms excl) + pdftexcmds (4.5 ms incl, 2.2 ms excl) + infwarerr (0.7 ms incl, 0.7 ms excl) + iftex (0.7 ms incl, 0.7 ms excl) + ltxcmds (0.8 ms incl, 0.8 ms excl) + array (1.0 ms incl, 1.0 ms excl) + booktabs (0.9 ms incl, 0.9 ms excl) + tikz (81.7 ms incl, 1.7 ms excl) + pgf (65.4 ms incl, 3.3 ms excl) + pgfrcs (3.0 ms incl, 1.5 ms excl) + pgfutil-common.tex (0.4 ms incl, 0.4 ms excl) + pgfutil-latex.def (0.4 ms incl, 0.4 ms excl) + pgfrcs.code.tex (0.7 ms incl, 0.6 ms excl) + pgf.revision.tex (0.2 ms incl, 0.2 ms excl) + pgfcore (54.4 ms incl, 2.4 ms excl) + graphicx (6.1 ms incl, 1.7 ms excl) + keyval (0.7 ms incl, 0.7 ms excl) + graphics (3.7 ms incl, 2.1 ms excl) + trig (0.6 ms incl, 0.6 ms excl) + graphics.cfg (0.2 ms incl, 0.2 ms excl) + pdftex.def (0.7 ms incl, 0.7 ms excl) + pgfsys (7.2 ms incl, 1.7 ms excl) + pgfsys.code.tex (5.1 ms incl, 1.6 ms excl) + pgfkeys.code.tex (2.5 ms incl, 1.6 ms excl) + pgfkeyslibraryfiltered.code.tex (0.9 ms incl, 0.9 ms excl) + pgf.cfg (0.1 ms incl, 0.1 ms excl) + pgfsys-pdftex.def (0.8 ms incl, 0.6 ms excl) + pgfsys-common-pdf.def (0.2 ms incl, 0.2 ms excl) + pgfsyssoftpath.code.tex (0.2 ms incl, 0.2 ms excl) + pgfsysprotocol.code.tex (0.2 ms incl, 0.2 ms excl) + +-- fonts -- + TFMs loaded: 3, embedded in PDF: 3, Type 3: 0, bitmap .pk read: 0 + loaded but not embedded: cmex7, cmr12 + preloaded by the format, embedded: cmmi10, cmr10 diff --git a/fixtures/whatif/artifacts/xelatex/fontmap.txt b/fixtures/whatif/artifacts/xelatex/fontmap.txt new file mode 100644 index 0000000..a93121b --- /dev/null +++ b/fixtures/whatif/artifacts/xelatex/fontmap.txt @@ -0,0 +1,4 @@ +pzdr Dingbats +L3 programming layer <2026-09-09> + *********** + + +Package rerunfilecheck Warning: File `main.out' has changed. +(rerunfilecheck) Rerun to get outlines right +(rerunfilecheck) or use package `bookmark'. + +Package rerunfilecheck Info: Checksums for `main.out': +(rerunfilecheck) Before: +(rerunfilecheck) After: 5E5F33D6957BF682D675E947AD6FD41B;91. + ) +Here is how much of TeX's memory you used: + 24114 strings out of 469155 + 513523 string characters out of 5451604 + 862370 words of memory out of 5000000 + 52994 multiletter control sequences out of 15000+600000 + 629327 words of font info for 46 fonts, out of 8000000 for 9000 + 319 hyphenation exceptions out of 8191 + 114i,6n,110p,561b,919s stack positions out of 10000i,1000n,20000p,200000b,200000s + +Output written on main.pdf (1 page). diff --git a/fixtures/whatif/artifacts/xelatex/main.prumo b/fixtures/whatif/artifacts/xelatex/main.prumo new file mode 100644 index 0000000..e6d70d6 --- /dev/null +++ b/fixtures/whatif/artifacts/xelatex/main.prumo @@ -0,0 +1,377 @@ +start 5040 0 xetex 2026-11-01 +file+ 5040 0 main.tex +cls+ 5071 1 article +file+ 5091 1 article.cls +file+ 5123 2 size10.clo +file- 5560 2 size10.clo +file- 5621 1 article.cls +cls- 5634 1 article +pkg+ 5666 1 amsmath +opts 5666 1 +file+ 5685 1 amsmath.sty +pkg+ 5731 2 amstext +opts 5732 2 +file+ 5750 2 amstext.sty +pkg+ 5779 3 amsgen +opts 5779 3 +file+ 5797 3 amsgen.sty +file- 5810 3 amsgen.sty +pkg- 5822 3 amsgen +file- 5831 2 amstext.sty +pkg- 5843 2 amstext +pkg+ 5869 2 amsbsy +opts 5869 2 +file+ 5887 2 amsbsy.sty +file- 5907 2 amsbsy.sty +pkg- 5919 2 amsbsy +pkg+ 5943 2 amsopn +opts 5944 2 namelimits, +file+ 5961 2 amsopn.sty +file- 5986 2 amsopn.sty +pkg- 5997 2 amsopn +file- 6138 1 amsmath.sty +pkg- 6151 1 amsmath +pkg+ 6182 1 siunitx +opts 6182 1 +file+ 6201 1 siunitx.sty +pkg+ 6722 2 translations +opts 6722 2 +file+ 6743 2 translations.sty +pkg+ 6777 3 etoolbox +opts 6777 3 +file+ 6795 3 etoolbox.sty +file- 6875 3 etoolbox.sty +pkg- 6888 3 etoolbox +pkg+ 6918 3 pdftexcmds +opts 6918 3 +file+ 6937 3 pdftexcmds.sty +pkg+ 6974 4 infwarerr +opts 6974 4 +file+ 6992 4 infwarerr.sty +file- 7006 4 infwarerr.sty +pkg- 7019 4 infwarerr +pkg+ 7045 4 iftex +opts 7045 4 +file+ 7063 4 iftex.sty +file- 7078 4 iftex.sty +pkg- 7089 4 iftex +pkg+ 7117 4 ltxcmds +opts 7117 4 +file+ 7136 4 ltxcmds.sty +file- 7155 4 ltxcmds.sty +pkg- 7167 4 ltxcmds +file- 7181 3 pdftexcmds.sty +pkg- 7194 3 pdftexcmds +file- 7599 2 translations.sty +pkg- 7612 2 translations +pkg+ 8741 2 array +opts 8741 2 +file+ 8759 2 array.sty +file- 8795 2 array.sty +pkg- 8808 2 array +file- 9093 1 siunitx.sty +pkg- 9107 1 siunitx +pkg+ 9137 1 booktabs +opts 9137 1 +file+ 9156 1 booktabs.sty +file- 9179 1 booktabs.sty +pkg- 9191 1 booktabs +pkg+ 9216 1 tikz +opts 9216 1 +file+ 9232 1 tikz.sty +pkg+ 9259 2 pgf +opts 9259 2 +file+ 9276 2 pgf.sty +pkg+ 9303 3 pgfrcs +opts 9303 3 +file+ 9321 3 pgfrcs.sty +file+ 9343 4 pgfutil-common.tex +file- 9371 4 pgfutil-common.tex +file+ 9389 4 pgfutil-latex.def +file- 9416 4 pgfutil-latex.def +file+ 9435 4 pgfrcs.code.tex +file+ 9462 5 pgf.revision.tex +file- 9474 5 pgf.revision.tex +file- 9483 4 pgfrcs.code.tex +file- 9488 3 pgfrcs.sty +pkg- 9500 3 pgfrcs +pkg+ 9531 3 pgfcore +opts 9531 3 +file+ 9550 3 pgfcore.sty +pkg+ 9578 4 graphicx +opts 9578 4 +file+ 9597 4 graphicx.sty +pkg+ 9628 5 keyval +opts 9629 5 +file+ 9647 5 keyval.sty +file- 9660 5 keyval.sty +pkg- 9671 5 keyval +pkg+ 9699 5 graphics +opts 9699 5 +file+ 9717 5 graphics.sty +pkg+ 9746 6 trig +opts 9746 6 +file+ 9763 6 trig.sty +file- 9774 6 trig.sty +pkg- 9786 6 trig +file+ 9810 6 graphics.cfg +file- 9821 6 graphics.cfg +file+ 9845 6 xetex.def +file- 9873 6 xetex.def +file- 9888 5 graphics.sty +pkg- 9900 5 graphics +file- 9915 4 graphicx.sty +pkg- 9927 4 graphicx +pkg+ 9955 4 pgfsys +opts 9955 4 +file+ 9973 4 pgfsys.sty +file+ 10003 5 pgfsys.code.tex +file+ 10026 6 pgfkeys.code.tex +file+ 10125 7 pgfkeyslibraryfiltered.code.tex +file- 10182 7 pgfkeyslibraryfiltered.code.tex +file- 10189 6 pgfkeys.code.tex +file+ 10235 6 pgf.cfg +file- 10246 6 pgf.cfg +file+ 10263 6 pgfsys-xetex.def +file+ 10286 7 pgfsys-dvipdfmx.def +file+ 10309 8 pgfsys-common-pdf.def +file- 10323 8 pgfsys-common-pdf.def +file- 10340 7 pgfsys-dvipdfmx.def +file- 10347 6 pgfsys-xetex.def +file- 10361 5 pgfsys.code.tex +file+ 10381 5 pgfsyssoftpath.code.tex +file- 10397 5 pgfsyssoftpath.code.tex +file+ 10416 5 pgfsysprotocol.code.tex +file- 10429 5 pgfsysprotocol.code.tex +file- 10435 4 pgfsys.sty +pkg- 10447 4 pgfsys +pkg+ 10481 4 xcolor +opts 10482 4 +file+ 10500 4 xcolor.sty +file+ 10856 5 color.cfg +file- 10870 5 color.cfg +file+ 10936 5 mathcolor.ltx +file- 10960 5 mathcolor.ltx +file- 11037 4 xcolor.sty +pkg- 11049 4 xcolor +file+ 11069 4 pgfcore.code.tex +file+ 11095 5 pgfmath.code.tex +file+ 11118 6 pgfmathutil.code.tex +file- 11138 6 pgfmathutil.code.tex +file+ 11157 6 pgfmathparser.code.tex +file- 11199 6 pgfmathparser.code.tex +file+ 11220 6 pgfmathfunctions.code.tex +file- 11250 6 pgfmathfunctions.code.tex +file+ 11272 6 pgfmathfunctions.basic.code.tex +file- 11315 6 pgfmathfunctions.basic.code.tex +file+ 11340 6 pgfmathfunctions.trigonometric.code.tex +file- 11452 6 pgfmathfunctions.trigonometric.code.tex +file+ 11475 6 pgfmathfunctions.random.code.tex +file- 11496 6 pgfmathfunctions.random.code.tex +file+ 11519 6 pgfmathfunctions.comparison.code.tex +file- 11550 6 pgfmathfunctions.comparison.code.tex +file+ 11572 6 pgfmathfunctions.base.code.tex +file- 11594 6 pgfmathfunctions.base.code.tex +file+ 11616 6 pgfmathfunctions.round.code.tex +file- 11634 6 pgfmathfunctions.round.code.tex +file+ 11655 6 pgfmathfunctions.misc.code.tex +file- 11684 6 pgfmathfunctions.misc.code.tex +file+ 11710 6 pgfmathfunctions.integerarithmetics.code.tex +file- 11727 6 pgfmathfunctions.integerarithmetics.code.tex +file+ 11747 6 pgfmathcalc.code.tex +file- 11764 6 pgfmathcalc.code.tex +file+ 11783 6 pgfmathfloat.code.tex +file- 11945 6 pgfmathfloat.code.tex +file- 11952 5 pgfmath.code.tex +file+ 11971 5 pgfint.code.tex +file- 11985 5 pgfint.code.tex +file+ 12003 5 pgfcorepoints.code.tex +file- 12054 5 pgfcorepoints.code.tex +file+ 12076 5 pgfcorepathconstruct.code.tex +file- 12112 5 pgfcorepathconstruct.code.tex +file+ 12132 5 pgfcorepathusage.code.tex +file- 12171 5 pgfcorepathusage.code.tex +file+ 12191 5 pgfcorescopes.code.tex +file- 12231 5 pgfcorescopes.code.tex +file+ 12252 5 pgfcoregraphicstate.code.tex +file- 12268 5 pgfcoregraphicstate.code.tex +file+ 12291 5 pgfcoretransformations.code.tex +file- 12314 5 pgfcoretransformations.code.tex +file+ 12333 5 pgfcorequick.code.tex +file- 12347 5 pgfcorequick.code.tex +file+ 12366 5 pgfcoreobjects.code.tex +file- 12379 5 pgfcoreobjects.code.tex +file+ 12400 5 pgfcorepathprocessing.code.tex +file- 12419 5 pgfcorepathprocessing.code.tex +file+ 12438 5 pgfcorearrows.code.tex +file- 12621 5 pgfcorearrows.code.tex +file+ 12644 5 pgfcoreshade.code.tex +file- 12674 5 pgfcoreshade.code.tex +file+ 12694 5 pgfcoreimage.code.tex +file- 12722 5 pgfcoreimage.code.tex +file+ 12743 5 pgfcoreexternal.code.tex +file- 12794 5 pgfcoreexternal.code.tex +file+ 12814 5 pgfcorelayers.code.tex +file- 12830 5 pgfcorelayers.code.tex +file+ 12851 5 pgfcoretransparency.code.tex +file- 12877 5 pgfcoretransparency.code.tex +file+ 12898 5 pgfcorepatterns.code.tex +file- 12915 5 pgfcorepatterns.code.tex +file+ 12934 5 pgfcorerdf.code.tex +file- 12949 5 pgfcorerdf.code.tex +file- 12957 4 pgfcore.code.tex +file- 12963 3 pgfcore.sty +pkg- 12975 3 pgfcore +file+ 12998 3 pgfmoduleshapes.code.tex +file- 13052 3 pgfmoduleshapes.code.tex +file+ 13072 3 pgfmoduleplot.code.tex +file- 13122 3 pgfmoduleplot.code.tex +pkg+ 13156 3 pgfcomp-version-0-65 +opts 13156 3 +file+ 13180 3 pgfcomp-version-0-65.sty +file- 13263 3 pgfcomp-version-0-65.sty +pkg- 13278 3 pgfcomp-version-0-65 +pkg+ 13312 3 pgfcomp-version-1-18 +opts 13313 3 +file+ 13336 3 pgfcomp-version-1-18.sty +file- 13357 3 pgfcomp-version-1-18.sty +pkg- 13371 3 pgfcomp-version-1-18 +file- 13378 2 pgf.sty +pkg- 13389 2 pgf +pkg+ 13415 2 pgffor +opts 13415 2 +file+ 13433 2 pgffor.sty +pkg+ 13469 3 pgfkeys +opts 13470 3 +file+ 13488 3 pgfkeys.sty +file+ 13506 4 pgfkeys.code.tex +file- 13518 4 pgfkeys.code.tex +file- 13523 3 pgfkeys.sty +pkg- 13536 3 pgfkeys +pkg+ 13563 3 pgfmath +opts 13563 3 +file+ 13581 3 pgfmath.sty +file+ 13615 4 pgfmath.code.tex +file- 13627 4 pgfmath.code.tex +file- 13632 3 pgfmath.sty +pkg- 13644 3 pgfmath +file+ 13663 3 pgffor.code.tex +file- 13704 3 pgffor.code.tex +file- 13710 2 pgffor.sty +pkg- 13721 2 pgffor +file+ 13741 2 tikz.code.tex +file+ 13770 3 pgflibraryplothandlers.code.tex +file- 13809 3 pgflibraryplothandlers.code.tex +file+ 14057 3 pgfmodulematrix.code.tex +file- 14082 3 pgfmodulematrix.code.tex +file+ 14301 3 tikzlibrarytopaths.code.tex +file- 14342 3 tikzlibrarytopaths.code.tex +file- 14349 2 tikz.code.tex +file- 14355 1 tikz.sty +pkg- 14366 1 tikz +file+ 14406 1 pgflibraryarrows.meta.code.tex +file- 14958 1 pgflibraryarrows.meta.code.tex +file+ 14982 1 tikzlibrarypositioning.code.tex +file- 15010 1 tikzlibrarypositioning.code.tex +file+ 15031 1 tikzlibrarycalc.code.tex +file- 15056 1 tikzlibrarycalc.code.tex +pkg+ 15125 1 hyperref +opts 15126 1 +file+ 15145 1 hyperref.sty +pkg+ 15194 2 kvsetkeys +opts 15194 2 +file+ 15213 2 kvsetkeys.sty +file- 15231 2 kvsetkeys.sty +pkg- 15244 2 kvsetkeys +pkg+ 15274 2 kvdefinekeys +opts 15274 2 +file+ 15295 2 kvdefinekeys.sty +file- 15310 2 kvdefinekeys.sty +pkg- 15323 2 kvdefinekeys +pkg+ 15352 2 pdfescape +opts 15353 2 +file+ 15372 2 pdfescape.sty +file- 15412 2 pdfescape.sty +pkg- 15424 2 pdfescape +pkg+ 15451 2 hycolor +opts 15452 2 +file+ 15470 2 hycolor.sty +file- 15571 2 hycolor.sty +pkg- 15583 2 hycolor +pkg+ 15610 2 nameref +opts 15611 2 +file+ 15630 2 nameref.sty +pkg+ 15660 3 refcount +opts 15661 3 +file+ 15679 3 refcount.sty +file- 15716 3 refcount.sty +pkg- 15728 3 refcount +pkg+ 15759 3 gettitlestring +opts 15760 3 +file+ 15781 3 gettitlestring.sty +pkg+ 15815 4 kvoptions +opts 15816 4 +file+ 15835 4 kvoptions.sty +file- 15889 4 kvoptions.sty +pkg- 15902 4 kvoptions +file- 15929 3 gettitlestring.sty +pkg- 15942 3 gettitlestring +file- 16124 2 nameref.sty +pkg- 16137 2 nameref +pkg+ 16174 2 stringenc +opts 16175 2 +file+ 16195 2 stringenc.sty +file- 16246 2 stringenc.sty +pkg- 16258 2 stringenc +file+ 16301 2 pd1enc.def +file- 16357 2 pd1enc.def +pkg+ 16387 2 intcalc +opts 16388 2 +file+ 16407 2 intcalc.sty +file- 16424 2 intcalc.sty +pkg- 16436 2 intcalc +file+ 16497 2 puenc.def +file- 16849 2 puenc.def +pkg+ 16968 2 url +opts 16968 2 +file+ 16986 2 url.sty +file- 17007 2 url.sty +pkg- 17018 2 url +pkg+ 17079 2 bitset +opts 17079 2 +file+ 17098 2 bitset.sty +pkg+ 17151 3 bigintcalc +opts 17151 3 +file+ 17171 3 bigintcalc.sty +file- 17211 3 bigintcalc.sty +pkg- 17224 3 bigintcalc +file- 17245 2 bitset.sty +pkg- 17257 2 bitset +file- 17573 1 hyperref.sty +file+ 17594 1 hxetex.def +pkg+ 17659 2 rerunfilecheck +opts 17660 2 +file+ 17682 2 rerunfilecheck.sty +pkg+ 17788 3 uniquecounter +opts 17789 3 +file+ 17810 3 uniquecounter.sty +file- 17847 3 uniquecounter.sty +pkg- 17860 3 uniquecounter +file- 17870 2 rerunfilecheck.sty +pkg- 17883 2 rerunfilecheck +file- 17894 1 hxetex.def +pkg- 17921 1 hyperref +predoc 18433 1 +file+ 18485 1 l3backend-xetex.def +file- 18525 1 l3backend-xetex.def +begindoc 18547 1 +file+ 18826 1 translations-basic-dictionary-english.trsl +file- 18949 1 translations-basic-dictionary-english.trsl +docstart 19198 1 +env+ 19387 1 tikzpicture +env- 19585 1 tikzpicture +enddoc 19621 1 +ship 19637 1 1 +page 19782 1 1 +lastpage 19786 1 +end 19806 1 diff --git a/fixtures/whatif/artifacts/xelatex/pdffonts.txt b/fixtures/whatif/artifacts/xelatex/pdffonts.txt new file mode 100644 index 0000000..bd4cbfa --- /dev/null +++ b/fixtures/whatif/artifacts/xelatex/pdffonts.txt @@ -0,0 +1,6 @@ +name type encoding emb sub uni object ID +------------------------------------ ----------------- ---------------- --- --- --- --------- +QVIUWV+LMRoman12-Bold-Identity-H CID Type 0C Identity-H yes yes yes 5 0 +IRQUGZ+LMRoman10-Regular-Identity-H CID Type 0C Identity-H yes yes yes 7 0 +DTQSCH+CMR10 Type 1C Builtin yes yes yes 9 0 +KONLXK+CMMI10 Type 1C Builtin yes yes yes 10 0 diff --git a/fixtures/whatif/artifacts/xelatex/report.txt b/fixtures/whatif/artifacts/xelatex/report.txt new file mode 100644 index 0000000..7db63d9 --- /dev/null +++ b/fixtures/whatif/artifacts/xelatex/report.txt @@ -0,0 +1,77 @@ +== prumo report: main (total 225 ms, preamble 206 ms, 1 pages, 311 files read) + +-- slowest units (exclusive) -- + 30.9 ms 225.3 ms incl file main.tex + 30.0 ms 44.6 ms incl pkg siunitx + 13.7 ms 42.7 ms incl pkg hyperref + 8.4 ms 8.4 ms incl file pgflibraryarrows.meta.code.tex + 8.1 ms 8.7 ms incl pkg xcolor + 7.7 ms 13.6 ms incl pkg translations + 7.7 ms 9.3 ms incl file tikz.code.tex + 6.7 ms 6.7 ms incl file size10.clo + 6.1 ms 28.8 ms incl file pgfcore.code.tex + 5.4 ms 5.4 ms incl file puenc.def + 4.7 ms 13.3 ms incl file pgfmath.code.tex + 4.2 ms 8.0 ms incl pkg nameref + +-- phases -- + startup 76.9 ms + preamble 204.4 ms + begin-document 11.7 ms + typesetting 6.5 ms + shipout 2.2 ms + end-document 0.6 ms + +-- environments -- + 3.0 ms 1x max 3.0 ms tikzpicture + +-- pages -- + page 1: 8.9 ms + +-- load tree -- + main.tex (225.3 ms incl, 30.9 ms excl) + article (8.6 ms incl, 1.9 ms excl) + size10.clo (6.7 ms incl, 6.7 ms excl) + amsmath (7.4 ms incl, 4.1 ms excl) + amstext (1.7 ms incl, 1.1 ms excl) + amsgen (0.7 ms incl, 0.7 ms excl) + amsbsy (0.8 ms incl, 0.8 ms excl) + amsopn (0.8 ms incl, 0.8 ms excl) + siunitx (44.6 ms incl, 30.0 ms excl) + translations (13.6 ms incl, 7.7 ms excl) + etoolbox (1.7 ms incl, 1.7 ms excl) + pdftexcmds (4.2 ms incl, 2.1 ms excl) + infwarerr (0.7 ms incl, 0.7 ms excl) + iftex (0.7 ms incl, 0.7 ms excl) + ltxcmds (0.8 ms incl, 0.8 ms excl) + array (1.0 ms incl, 1.0 ms excl) + booktabs (0.8 ms incl, 0.8 ms excl) + tikz (78.6 ms incl, 1.6 ms excl) + pgf (63.0 ms incl, 3.1 ms excl) + pgfrcs (3.0 ms incl, 1.4 ms excl) + pgfutil-common.tex (0.4 ms incl, 0.4 ms excl) + pgfutil-latex.def (0.4 ms incl, 0.4 ms excl) + pgfrcs.code.tex (0.7 ms incl, 0.5 ms excl) + pgf.revision.tex (0.2 ms incl, 0.2 ms excl) + pgfcore (52.6 ms incl, 2.2 ms excl) + graphicx (5.3 ms incl, 1.6 ms excl) + keyval (0.7 ms incl, 0.7 ms excl) + graphics (3.1 ms incl, 1.9 ms excl) + trig (0.6 ms incl, 0.6 ms excl) + graphics.cfg (0.2 ms incl, 0.2 ms excl) + xetex.def (0.4 ms incl, 0.4 ms excl) + pgfsys (7.5 ms incl, 1.6 ms excl) + pgfsys.code.tex (5.5 ms incl, 1.5 ms excl) + pgfkeys.code.tex (2.5 ms incl, 1.6 ms excl) + pgfkeyslibraryfiltered.code.tex (0.9 ms incl, 0.9 ms excl) + pgf.cfg (0.2 ms incl, 0.2 ms excl) + pgfsys-xetex.def (1.3 ms incl, 0.5 ms excl) + pgfsys-dvipdfmx.def (0.8 ms incl, 0.6 ms excl) + pgfsys-common-pdf.def (0.2 ms incl, 0.2 ms excl) + pgfsyssoftpath.code.tex (0.2 ms incl, 0.2 ms excl) + +-- fonts -- + TFMs loaded: 2, embedded in PDF: 4, Type 3: 0, bitmap .pk read: 0 + loaded but not embedded: cmex7, pzdr + preloaded by the format, embedded: cmmi10, cmr10 + OpenType, opened by the engine: LMRoman10-Regular (not in the .fls), LMRoman12-Bold (not in the .fls) diff --git a/fixtures/whatif/main.tex b/fixtures/whatif/main.tex new file mode 100644 index 0000000..d3cd72b --- /dev/null +++ b/fixtures/whatif/main.tex @@ -0,0 +1,42 @@ +% Fixture: whatif, a heavy preamble and one picture. +% +% Known answers on TeX Live 2026 (tests/tex.rs::whatif_fixture_claims; 7 in +% tests/fixtures.rs::whatif_fixture_bounds): +% 1. On pdfLaTeX the precompiled preamble (mylatexformat) saves more than a third of the +% build, outside the noise, with the same page count and text. +% 2. On XeLaTeX the format cannot be dumped: the class loads Latin Modern as an OpenType +% font, and XeTeX does not dump those. The preamble row says "could not precompile" +% with that error, and draft still runs. +% 3. On LuaLaTeX the format build fails too (a Lua error during the dump) and is reported +% the same way. +% 4. With --shell-escape, externalizing the picture keeps the page count and text on all +% three engines, and the first run, which builds the picture, is the slowest. +% 5. Without --shell-escape, externalize is refused when asked for and not run by default. +% 6. Draft mode runs on all three engines and its output is not compared (no PDF). +% 7. The probe's bound for externalize is the one tikzpicture's time, and for preamble the +% preamble phase, the longest phase on every engine. +% 8. The document's directory is unchanged. +\documentclass{article} +\usepackage{amsmath} +\usepackage{siunitx} +\usepackage{booktabs} +\usepackage{tikz} +\usetikzlibrary{arrows.meta,positioning,calc} +\usepackage{xcolor} +\usepackage{hyperref} +\begin{document} +\section{A picture} +The box is \SI{2}{\centi\metre} wide. + +\begin{tikzpicture} + \node[draw, minimum width=2cm] (a) {box}; + \node[draw, right=of a] (b) {other}; + \draw[-Stealth] (a) -- (b); +\end{tikzpicture} + +\begin{tabular}{lr} + \toprule + a & \num{1.5} \\ + \bottomrule +\end{tabular} +\end{document} diff --git a/scripts/make_artifacts.sh b/scripts/make_artifacts.sh index 335b52d..5453b9e 100755 --- a/scripts/make_artifacts.sh +++ b/scripts/make_artifacts.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Build every fixture (a directory under fixtures/ with a \documentclass) with every engine using the probe, and check in the # artifacts parsers are tested against (.prumo, .fls, log tail, pdffonts, golden JSON). -# Usage: scripts/make_artifacts.sh (run from repo root; needs TeX Live + poppler-utils) +# Usage: scripts/make_artifacts.sh [fixture...] (run from repo root; needs TeX Live + poppler-utils; +# with names, only those fixtures are rebuilt) set -euo pipefail root=$(cd "$(dirname "$0")/.." && pwd) # Fixed build directory so the PWD line in every .fls is stable across regenerations. @@ -17,6 +18,7 @@ envs=tikzpicture,axis,tcolorbox,tblr,longtable,tabularx,minted,lstlisting,forest pretex="\\RequirePackage{prumo}\\PrumoEnvironments{$envs}" for dir in "$root"/fixtures/*/; do fx=$(basename "$dir") + if [ $# -gt 0 ] && ! printf '%s\n' "$@" | grep -qx "$fx"; then continue; fi main=$(grep -l '\\documentclass' "$dir"/*.tex 2>/dev/null | head -1) || true [ -n "$main" ] || continue job=$(basename "${main%.tex}") diff --git a/src/doctor.rs b/src/doctor.rs index 179f042..b6872c8 100644 --- a/src/doctor.rs +++ b/src/doctor.rs @@ -117,6 +117,28 @@ fn kpsewhich(args: &[&str]) -> Option { (!s.is_empty()).then_some(s) } +/// Where the TeX installation keeps `name` (`mylatexformat.ltx`), if it has it. +pub fn tex_file(name: &str) -> Option { + kpsewhich(&[name]).map(PathBuf::from) +} + +/// mylatexformat, for `whatif preamble`. Only asked when there is a TeX to ask. +fn mylatexformat_check() -> Option { + which("kpsewhich")?; + let name = "mylatexformat".to_string(); + Some(match tex_file("mylatexformat.ltx") { + Some(p) => Check { name, status: Status::Ok, detail: p.display().to_string() }, + None => Check { + name, + status: Status::Warn, + detail: format!( + "not found; whatif cannot precompile the preamble ({})", + Installer::detect().install("mylatexformat") + ), + }, + }) +} + /// Read `\edef\fmtversion {YYYY-MM-DD}` from `latex.ltx` (the braces may be on the next line). pub(crate) fn kernel_date_from_ltx(text: &str) -> Option { let i = text.find("\\fmtversion")?; @@ -353,6 +375,7 @@ pub fn run() -> Report { checks.push(c); } checks.push(tool_check("latexmk", "--version", "not on PATH; --latexmk unavailable")); + checks.extend(mylatexformat_check()); let (kernel, kernel_ok) = kernel_check(); checks.push(kernel); let (recorder, recorder_ok) = recorder_check(); @@ -439,6 +462,11 @@ pub struct Coverage { /// Macro-level files from the `.fls` with no `file+` record (fonts, maps, formats and /// the probe itself are not counted: they are never `\input`). pub outside_hooks: Vec, + /// Macro-level files from the `.fls` that neither a record nor the log saw opened: the + /// file was tested for existence, not read (siunitx's `\RequirePackage{color}` after + /// xcolor has claimed `color.sty`). Only with a log; without one they are in + /// `outside_hooks`. + pub opened_not_read: Vec, /// Files the log shows opened that no record saw (by base name). pub log_only: Vec, /// Files a record saw that the log parser did not find: a parser miss, worth a look. @@ -460,6 +488,8 @@ pub fn hook_coverage(model: &BuildModel, files: &[InputFile], log: Option<&TexLo }; }); let mut c = Coverage { schema: crate::JSON_SCHEMA, ..Default::default() }; + let logged: Option> = + log.map(|l| l.files.iter().map(|f| f.path.rsplit('/').next().unwrap_or(&f.path).to_string()).collect()); for f in files { let macro_level = matches!( f.kind, @@ -481,12 +511,17 @@ pub fn hook_coverage(model: &BuildModel, files: &[InputFile], log: Option<&TexLo base.strip_suffix(".sty").map(|s| format!("{s}-ltx.sty")).is_some_and(|s| hooked.contains(&s)); if macro_level && base != "prumo.sty" && !hooked.contains(&base) && !substituted && f.path.extension().is_some() { - c.outside_hooks.push(Outside { path: f.path.clone(), kind: f.kind }); + let o = Outside { path: f.path.clone(), kind: f.kind }; + // The kernel reads .aux-like files with the primitive, and the log shows those. + match &logged { + Some(l) if !l.contains(&base) && !matches!(f.kind, InputKind::Aux | InputKind::AuxLike) => { + c.opened_not_read.push(o) + } + _ => c.outside_hooks.push(o), + } } } - if let Some(log) = log { - let logged: BTreeSet = - log.files.iter().map(|f| f.path.rsplit('/').next().unwrap_or(&f.path).to_string()).collect(); + if let (Some(log), Some(logged)) = (log, logged) { c.log_only = logged.iter().filter(|n| !hooked.contains(*n) && n.as_str() != "prumo.sty").cloned().collect(); // `.sty`/`.cls` keys above are derived from unit names; only bare file frames are checked here. c.hooked_not_in_log = hooked @@ -518,6 +553,12 @@ pub fn coverage_text(job: &str, c: &Coverage, has_log: bool) -> String { let _ = writeln!(s, " {} ({:?}{note})", o.path.display(), o.kind); } } + if !c.opened_not_read.is_empty() { + let _ = writeln!(s, " opened but never read (an existence check: no record, and the log shows no read):"); + for o in &c.opened_not_read { + let _ = writeln!(s, " {} ({:?})", o.path.display(), o.kind); + } + } if has_log { let _ = writeln!( s, diff --git a/src/lib.rs b/src/lib.rs index bb6630b..a016758 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ pub mod run; pub mod texlog; pub mod tlpdb; pub mod unused; +pub mod whatif; /// The `schema` field of every `--json` report. It moves only when a field is renamed or /// removed, never when one is added. diff --git a/src/main.rs b/src/main.rs index eb32bde..8d3beec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -268,6 +268,38 @@ enum Cmd { #[arg(long)] json: bool, }, + /// Measured speed-ups: rebuild with one change (precompiled preamble, draft mode, + /// externalized tikz pictures), time it against the plain build, compare the output. + /// Runs in a temp copy without the probe, one build at a time. + Whatif { + /// The document. Its directory is copied; nothing in it is modified. + main: PathBuf, + /// Engine to run. Default: the document's `% !TEX program =` magic comment, else pdflatex. + #[arg(long, value_enum)] + engine: Option, + /// Experiments, comma-separated. Default: all three; externalize runs only with + /// --shell-escape. + #[arg(long = "try", value_name = "EXPERIMENTS", value_enum, value_delimiter = ',')] + tries: Vec, + /// Timed runs of each build after its warm-up; the median is compared. + #[arg(long, default_value_t = 5, value_parser = clap::value_parser!(u32).range(2..))] + runs: u32, + /// Pass -shell-escape to every build. externalize needs it: tikz builds each + /// picture with a shell command. + #[arg(long)] + shell_escape: bool, + /// Per-build timeout in seconds (default: 10× the probed build, at least 30; the + /// first externalized run gets that once more per picture). + #[arg(long, default_value_t = 0)] + timeout: u64, + /// Artifact directory for the probed run and the logs (default: the document's + /// cache directory, see `prumo where`). + #[arg(long)] + out: Option, + /// Machine-readable output (schema 1). + #[arg(long)] + json: bool, + }, /// The TeX Live packages this document needs, with the file and LaTeX unit that /// caused each (needs a texlive.tlpdb; file-level only without one). Packages { @@ -676,6 +708,36 @@ fn main() -> anyhow::Result<()> { } } } + Cmd::Whatif { main, .. } if main.extension().is_some_and(|e| e == "prumo") => { + anyhow::bail!("{}: whatif rebuilds the document; give its .tex", main.display()) + } + Cmd::Whatif { main, engine, tries, runs, shell_escape, timeout, out, json } => { + let engine = + engine.or_else(|| prumo::prune::magic_engine(&main)).unwrap_or(prumo::run::EngineCmd::Pdflatex); + let explicit = !tries.is_empty(); + let tries = if explicit { tries } else { prumo::whatif::Experiment::ALL.to_vec() }; + let opts = prumo::whatif::WhatifOptions { + main, + engine, + tries, + explicit, + runs, + shell_escape, + timeout_secs: timeout, + out, + env: vec![], + }; + let r = prumo::whatif::run(&opts, |m| { + if !json { + eprintln!("prumo: {m}"); + } + })?; + if json { + println!("{}", serde_json::to_string_pretty(&r)?); + } else { + print!("{}", prumo::whatif::text(&r)); + } + } Cmd::Prune { paths, engines, latexmk, tlpdb, keep, keep_hyphenation, out, apply, yes, json } => { let out = out.unwrap_or_else(|| run::cache_root().join("prune")); if !json { diff --git a/src/run.rs b/src/run.rs index 7cf39c7..61742b3 100644 --- a/src/run.rs +++ b/src/run.rs @@ -494,7 +494,7 @@ pub fn cache_dir(main: &Path, out: Option<&Path>) -> Result { } /// Copy a project tree, skipping `.prumo`, VCS directories and `target`. -fn copy_tree(from: &Path, to: &Path) -> Result<(), RunError> { +pub(crate) fn copy_tree(from: &Path, to: &Path) -> Result<(), RunError> { std::fs::create_dir_all(to).map_err(io("create", to))?; for entry in std::fs::read_dir(from).map_err(io("read", from))? { let entry = entry.map_err(io("read", from))?; @@ -547,13 +547,6 @@ pub(crate) fn build_once_timed( pretex: Option<&str>, timeout: Option, ) -> Result { - let tool = build.tool(); - let has_dir = Path::new(tool).components().count() > 1; - let found = if has_dir { Path::new(tool).is_file() } else { crate::doctor::which(tool).is_some() }; - if !found { - return Err(RunError::ToolMissing { tool: tool.into() }); - } - let mut cmd = Command::new(tool); let mut argv: Vec = build.argv.clone(); let pre = match (pretex, probe.map(|p| p.load())) { (Some(p), Some(load)) => Some(format!("{p}{load}")), @@ -581,21 +574,41 @@ pub(crate) fn build_once_timed( None => format!("{stem}.tex"), }); } - cmd.args(&argv[1..]).current_dir(dir); + let mut env = env.to_vec(); if let Some(p) = probe.map(|p| p.dir) { // kpathsea separates search paths with `:`, except on Windows where it is `;`. // A trailing separator keeps the default search path. const SEP: char = if cfg!(windows) { ';' } else { ':' }; let existing = std::env::var("TEXINPUTS").unwrap_or_default(); let tail = if existing.is_empty() || existing.ends_with(SEP) { String::new() } else { SEP.to_string() }; - cmd.env("TEXINPUTS", format!("{}{SEP}{existing}{tail}", p.display())); + env.insert(0, ("TEXINPUTS".into(), format!("{}{SEP}{existing}{tail}", p.display()))); + } + spawn_timed(dir, &argv, &env, &dir.join("build.out"), timeout) +} + +/// Run `argv` in `dir` with `env` added, stdout and stderr to `out_path`, and time it from +/// spawn to exit. The process runs in its own process group, and when `timeout` is up the +/// whole group is killed (latexmk's engine child and tikz's picture builds included). +pub(crate) fn spawn_timed( + dir: &Path, + argv: &[String], + env: &[(String, String)], + out_path: &Path, + timeout: Option, +) -> Result { + let tool = argv.first().map(String::as_str).unwrap_or_default(); + let has_dir = Path::new(tool).components().count() > 1; + let found = if has_dir { Path::new(tool).is_file() } else { crate::doctor::which(tool).is_some() }; + if !found { + return Err(RunError::ToolMissing { tool: tool.into() }); } + let mut cmd = Command::new(tool); + cmd.args(&argv[1..]).current_dir(dir); for (k, v) in env { cmd.env(k, v); } - let out_path = dir.join("build.out"); - let file = std::fs::File::create(&out_path).map_err(io("create", &out_path))?; - let err = file.try_clone().map_err(io("create", &out_path))?; + let file = std::fs::File::create(out_path).map_err(io("create", out_path))?; + let err = file.try_clone().map_err(io("create", out_path))?; cmd.stdin(std::process::Stdio::null()).stdout(file).stderr(err); #[cfg(unix)] { @@ -609,7 +622,7 @@ pub(crate) fn build_once_timed( if let Some(status) = child.try_wait().map_err(run_err)? { // The wall time is the finalization's other end: poll often enough that it is // good to the millisecond. - return Ok(Built { argv, exit_code: status.code(), wall: started.elapsed() }); + return Ok(Built { argv: argv.to_vec(), exit_code: status.code(), wall: started.elapsed() }); } if let Some(t) = timeout && started.elapsed() > t diff --git a/src/unused.rs b/src/unused.rs index f2b575a..f97ddb0 100644 --- a/src/unused.rs +++ b/src/unused.rs @@ -416,13 +416,13 @@ pub(crate) fn blocking_pretex(package: &str, options: &str) -> String { ) } -fn pdftotext(pdf: &Path) -> Option { +pub(crate) fn pdftotext(pdf: &Path) -> Option { let out = Command::new("pdftotext").args(["-layout", pdf.to_str()?, "-"]).output().ok()?; out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()) } /// `pdfinfo` without the fields that change on every build. -fn pdfinfo(pdf: &Path) -> Option> { +pub(crate) fn pdfinfo(pdf: &Path) -> Option> { let out = Command::new("pdfinfo").arg(pdf).output().ok()?; let text = String::from_utf8_lossy(&out.stdout); Some( @@ -435,7 +435,7 @@ fn pdfinfo(pdf: &Path) -> Option> { } /// Pages rasterized to gray at `dpi`, as PGM pixel buffers (one per page). -fn render(pdf: &Path, dpi: u32) -> Option>> { +pub(crate) fn render(pdf: &Path, dpi: u32) -> Option>> { let dir = tempfile::tempdir().ok()?; let prefix = dir.path().join("p"); let ok = @@ -465,6 +465,19 @@ fn render(pdf: &Path, dpi: u32) -> Option>> { Some(pages) } +/// The first page (1-based) whose gray pixels differ by more than 32 levels in more than +/// `per_mille` of the page, with the count and the page size; a page whose size changed +/// counts too. +pub(crate) fn render_diff(a: &[Vec], b: &[Vec], per_mille: usize) -> Option<(usize, usize, usize)> { + for (i, (x, y)) in a.iter().zip(b).enumerate() { + let differing = x.iter().zip(y).filter(|(p, q)| p.abs_diff(**q) > 32).count(); + if x.len() != y.len() || differing * 1000 > x.len().max(1) * per_mille { + return Some((i + 1, differing, x.len())); + } + } + None +} + /// Coarse ink check in `Text` mode, strict comparison in `Render` mode. fn raster_dpi(compare: Compare) -> u32 { match compare { @@ -473,7 +486,7 @@ fn raster_dpi(compare: Compare) -> u32 { } } -fn first_diff_line(a: &str, b: &str) -> String { +pub(crate) fn first_diff_line(a: &str, b: &str) -> String { for (i, (x, y)) in a.lines().zip(b.lines()).enumerate() { if x != y { return format!("line {}: {:?} vs {:?}", i + 1, x.trim(), y.trim()); @@ -532,20 +545,14 @@ fn judge(base: &Baseline, out: &RunOutput, compare: Compare) -> Verdict { Compare::Text => 5, Compare::Render => 1, }; - for (i, (x, y)) in a.iter().zip(&b).enumerate() { - let differing = x.iter().zip(y).filter(|(p, q)| p.abs_diff(**q) > 32).count(); - if x.len() != y.len() || differing * 1000 > x.len().max(1) * per_mille { - let what = if compare == Compare::Text { "ink differs" } else { "renders differently" }; - return Verdict::AffectsOutput { - evidence: format!( - "page {} {what} ({} of {} pixels at {} dpi; text identical)", - i + 1, - differing, - x.len(), - raster_dpi(compare) - ), - }; - } + if let Some((page, differing, total)) = render_diff(a, &b, per_mille) { + let what = if compare == Compare::Text { "ink differs" } else { "renders differently" }; + return Verdict::AffectsOutput { + evidence: format!( + "page {page} {what} ({differing} of {total} pixels at {} dpi; text identical)", + raster_dpi(compare) + ), + }; } } if info != base.info { diff --git a/src/whatif.rs b/src/whatif.rs new file mode 100644 index 0000000..44851ef --- /dev/null +++ b/src/whatif.rs @@ -0,0 +1,1040 @@ +//! `prumo whatif`: measured speed-up experiments. Each experiment rebuilds the document with +//! one change and times it against the plain build. Everything runs in a temp copy; the +//! user's files are never touched. +//! +//! - `preamble`: the preamble precompiled into a format with mylatexformat +//! (`pdftex -ini -jobname=-fmt "&pdflatex" mylatexformat.ltx .tex`, then +//! `pdflatex -fmt=-fmt .tex`). pdfTeX only in practice: XeTeX cannot dump the +//! OpenType font the class loads, and LuaTeX's dump fails in lua-uni-algos (TeX Live 2026). +//! - `draft`: no PDF (`-draftmode`, `--draftmode`, `-no-pdf`), which is what writing the PDF +//! and including images cost. +//! - `externalize`: each tikzpicture built once into its own PDF and included after that +//! (tikz's external library, injected from the command line). Needs `--shell-escape`. +//! +//! The timed builds run without the probe and one at a time, since parallel builds slow each +//! other down. Each build gets a warm-up run, then `runs` rounds of the baseline and every +//! variant in turn, so drift hits them all alike. Medians are compared, and a difference no +//! larger than the spread (slowest run minus fastest) is "within noise". One probed run, +//! the cache's when it is current, gives the most each change can save. + +use crate::model::{BuildModel, FrameKind}; +use crate::run::{self, EngineCmd, RunOptions, RunOutput}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum Experiment { + /// Precompile the preamble into a format (mylatexformat). + Preamble, + /// Build without writing a PDF. + Draft, + /// Build each tikzpicture once and include it (needs --shell-escape). Experimental. + Externalize, +} + +impl Experiment { + pub const ALL: [Experiment; 3] = [Experiment::Preamble, Experiment::Draft, Experiment::Externalize]; + + pub fn name(self) -> &'static str { + match self { + Experiment::Preamble => "preamble", + Experiment::Draft => "draft", + Experiment::Externalize => "externalize", + } + } + + /// tikz's external library breaks documents in ways a page count and text check do not + /// always catch. + pub fn experimental(self) -> bool { + self == Experiment::Externalize + } +} + +/// Wall times of one build command. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct Sample { + pub median_ms: f64, + /// Slowest run minus fastest. + pub spread_ms: f64, + pub runs_ms: Vec, + /// The warm-up run, left out of the median. + pub warmup_ms: f64, +} + +impl Sample { + fn new(warmup_ms: f64, runs_ms: Vec) -> Sample { + Sample { median_ms: round3(median(&runs_ms)), spread_ms: round3(spread(&runs_ms)), runs_ms, warmup_ms } + } +} + +/// Milliseconds to the microsecond, as the other reports print them. +fn round3(ms: f64) -> f64 { + (ms * 1000.0).round() / 1000.0 +} + +fn median(xs: &[f64]) -> f64 { + let mut v = xs.to_vec(); + v.sort_by(f64::total_cmp); + match v.len() { + 0 => 0.0, + n if n % 2 == 1 => v[n / 2], + n => (v[n / 2 - 1] + v[n / 2]) / 2.0, + } +} + +fn spread(xs: &[f64]) -> f64 { + let max = xs.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let min = xs.iter().copied().fold(f64::INFINITY, f64::min); + if xs.is_empty() { 0.0 } else { max - min } +} + +/// The variant's output against the baseline's. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +#[serde(tag = "status", rename_all = "kebab-case")] +pub enum Output { + /// Same page count, the same words on every page (`pdftotext`), and the same coarse + /// render (24 dpi gray; at most 0.5 % of a page's pixels may differ, as in `unused`). + Identical, + Changed { + evidence: String, + }, + NotCompared { + reason: String, + }, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +#[serde(tag = "status", rename_all = "kebab-case")] +pub enum Outcome { + Measured { + variant: Sample, + /// Baseline median minus variant median; negative when the variant is slower. + saved_ms: f64, + saved_pct: f64, + /// The saving is no larger than the larger of the two spreads, or 2 ms (the timing + /// resolution) when both are smaller. + within_noise: bool, + /// Paid once: building the format, or the first externalized run (every picture + /// built). Not in the median. + one_time_ms: Option, + output: Output, + }, + /// The variant could not be built: the reason and the first error. + Failed { error: String }, + /// Not attempted here, and why. + NotRun { reason: String }, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct Row { + pub experiment: Experiment, + pub experimental: bool, + /// The most the probe says the change can save: the preamble phase for `preamble`, the + /// tikzpicture environments for `externalize`. A cross-check on the measurement. + pub bound_ms: Option, + #[serde(flatten)] + pub outcome: Outcome, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct WhatifReport { + pub schema: u32, + pub job: String, + pub engine: EngineCmd, + pub runs: u32, + pub shell_escape: bool, + pub baseline: Sample, + pub baseline_pages: usize, + pub experiments: Vec, +} + +#[derive(Debug, Clone)] +pub struct WhatifOptions { + pub main: PathBuf, + pub engine: EngineCmd, + /// The experiments to run, in order. + pub tries: Vec, + /// `tries` was asked for (`--try`) rather than the default set. An externalize the + /// user asked for without `--shell-escape` is refused; a default one is not run. + pub explicit: bool, + /// Timed runs per build after the warm-up. + pub runs: u32, + /// Pass `-shell-escape` to every build. Externalize needs it. + pub shell_escape: bool, + /// Per build, in seconds; 0 = 10× the probed build, at least 30 s. + pub timeout_secs: u64, + pub out: Option, + pub env: Vec<(String, String)>, +} + +#[derive(Debug, thiserror::Error)] +pub enum WhatifError { + #[error(transparent)] + Run(#[from] run::RunError), + #[error("the build failed ({}); fix the document first:\n{tail}", run::exit_text(*.code))] + BaselineFailed { code: Option, tail: String }, + #[error("pdftotext is not on PATH (poppler-utils); whatif compares each variant's text with the baseline's")] + NoPdftotext, + #[error( + "externalize runs a shell command for every picture (tikz builds each one with its own engine call); pass --shell-escape to allow it" + )] + NeedsShellEscape, + #[error("cannot {what} {path}: {source}")] + Io { + what: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, +} + +/// The binary that builds a format for `engine`. LuaLaTeX's format is LuaHBTeX's since +/// TeX Live 2020. +fn ini_binary(engine: EngineCmd) -> &'static str { + match engine { + EngineCmd::Pdflatex => "pdftex", + EngineCmd::Xelatex => "xetex", + EngineCmd::Lualatex if crate::doctor::which("luahbtex").is_some() => "luahbtex", + EngineCmd::Lualatex => "luatex", + } +} + +/// The switch that stops the engine writing a PDF. +fn draft_flag(engine: EngineCmd) -> &'static str { + match engine { + EngineCmd::Pdflatex => "-draftmode", + EngineCmd::Xelatex => "-no-pdf", + EngineCmd::Lualatex => "--draftmode", + } +} + +/// The code that turns externalization on, for the command line. The picture builds need +/// it too: tikz runs each one as `\def\tikzexternalrealjob{}\input{}`, which +/// without the hook reads the document with no external library and typesets all of it as +/// "the picture", a whole page put where the picture was. So the system call carries the +/// hook as well, and names the engine the document runs on (tikz's default is pdflatex +/// unless the driver says otherwise). +pub(crate) fn externalize_pretex(engine: EngineCmd) -> String { + let inner = "\\string\\AddToHook{package/tikz/after}{\\string\\usetikzlibrary{external}\\string\\tikzexternalize}"; + format!( + "\\AddToHook{{package/tikz/after}}{{\\usetikzlibrary{{external}}\\tikzexternalize[system call={{{} \\tikzexternalcheckshellescape -halt-on-error -interaction=batchmode -jobname \"\\image\" \"{inner}\\texsource\"}}]}}", + engine.command() + ) +} + +/// One build command of an experiment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Step { + Baseline, + /// Precompile the preamble: `-fmt.fmt`. + Format, + /// The document on that format. + Precompiled, + Draft, + Externalize, +} + +/// The command line of a step, run in the copy's directory. +pub(crate) fn argv(step: Step, engine: EngineCmd, job: &str, shell_escape: bool) -> Vec { + let mut v = vec![engine.command().to_string(), "-interaction=nonstopmode".to_string()]; + if shell_escape || step == Step::Externalize { + v.push("-shell-escape".into()); + } + let tex = format!("{job}.tex"); + match step { + Step::Baseline => v.push(tex), + Step::Format => { + v[0] = ini_binary(engine).to_string(); + v.insert(1, "-ini".into()); + v.extend([ + format!("-jobname={job}-fmt"), + format!("&{}", engine.command()), + "mylatexformat.ltx".into(), + tex, + ]); + } + Step::Precompiled => v.extend([format!("-fmt={job}-fmt"), tex]), + Step::Draft => v.extend([draft_flag(engine).to_string(), tex]), + Step::Externalize => { + v.extend([format!("-jobname={job}"), format!("{}\\input{{{job}}}", externalize_pretex(engine))]) + } + } + v +} + +/// The first thing that went wrong in a failed build: its first `!` error, else a Lua error +/// (LuaTeX reports those without `!`), else the exit status. +pub(crate) fn first_error(log_text: &str, code: Option) -> String { + let log = crate::texlog::parse_str(log_text); + if let Some(e) = log.errors.first() { + return format!("! {e}"); + } + // LuaTeX wraps at max_print_line or one past it, which `texlog::unwrap` leaves alone. + let lines: Vec<&str> = log_text.lines().collect(); + for (k, line) in lines.iter().enumerate() { + if lua_error(line).is_none() { + continue; + } + let mut joined = line.to_string(); + let mut j = k; + while lines[j].chars().count() >= crate::texlog::MAX_PRINT_LINE + && lines.get(j + 1).is_some_and(|n| !n.is_empty() && !n.starts_with('<') && !n.starts_with('!')) + { + j += 1; + joined.push_str(lines[j]); + } + return lua_error(&joined).unwrap_or(joined); + } + run::exit_text(code) +} + +/// `.../lua-uni-stage-tables.lua:141: attempt to ...` as `lua-uni-stage-tables.lua:141: +/// attempt to ...`. +fn lua_error(line: &str) -> Option { + let i = line.find(".lua:")?; + let rest = &line[i + ".lua:".len()..]; + let digits = rest.len() - rest.trim_start_matches(|c: char| c.is_ascii_digit()).len(); + if digits == 0 || !rest[digits..].starts_with(": ") { + return None; + } + let start = line[..i].rfind(['/', ' ', '(', ')']).map_or(0, |j| j + 1); + Some(line[start..].trim().to_string()) +} + +/// What a failed build is called in the row. +fn failure(step: Step, error: &str) -> String { + match step { + Step::Format => format!("could not precompile: {error}"), + Step::Precompiled => format!("the build on the precompiled preamble failed: {error}"), + Step::Draft => format!("the draft build failed: {error}"), + Step::Externalize => format!("could not externalize: {error}"), + Step::Baseline => error.to_string(), + } +} + +/// Run one step in `dir`: its wall time in ms, or the first error. +fn build( + dir: &Path, + step: Step, + opts: &WhatifOptions, + job: &str, + timeout: std::time::Duration, +) -> Result, String)> { + let args = argv(step, opts.engine, job, opts.shell_escape); + let log_job = if step == Step::Format { format!("{job}-fmt") } else { job.to_string() }; + let out = dir.join(format!("prumo-{}.txt", step_name(step))); + match run::spawn_timed(dir, &args, &opts.env, &out, Some(timeout)) { + Ok(b) if b.exit_code == Some(0) => Ok(round3(b.wall.as_secs_f64() * 1000.0)), + Ok(b) => { + let text = std::fs::read(dir.join(format!("{log_job}.log"))) + .map(|t| String::from_utf8_lossy(&t).into_owned()) + .unwrap_or_default(); + Err((b.exit_code, first_error(&text, b.exit_code))) + } + Err(run::RunError::TimedOut { secs }) => Err((None, format!("timed out after {secs:.0} s"))), + Err(e) => Err((None, e.to_string())), + } +} + +fn step_name(step: Step) -> &'static str { + match step { + Step::Baseline => "baseline", + Step::Format => "format", + Step::Precompiled => "preamble", + Step::Draft => "draft", + Step::Externalize => "externalize", + } +} + +/// The probed run, from the cache when it is current: the bounds come from it, and a +/// document that does not build stops here. +fn probed_run(opts: &WhatifOptions, out_root: &Path) -> Result { + let ro = RunOptions { + build: opts.shell_escape.then(|| format!("{} -shell-escape", opts.engine.command())), + out: Some(out_root.to_path_buf()), + env: opts.env.clone(), + ..RunOptions::new(opts.main.clone(), opts.engine) + }; + let out = match run::cached(&opts.main, Some(out_root))?.filter(|c| !run::is_stale(c, &ro)) { + Some(c) => c, + None => run::run(&ro)?, + }; + if out.info.exit_code != Some(0) { + let text = std::fs::read(&out.log).map(|t| String::from_utf8_lossy(&t).into_owned()).unwrap_or_default(); + return Err(WhatifError::BaselineFailed { + code: out.info.exit_code, + tail: first_error(&text, out.info.exit_code), + }); + } + Ok(out) +} + +/// Some package in the load tree has this name. +fn loads(model: &BuildModel, package: &str) -> bool { + let mut found = false; + model.root.walk(&mut |f, _| found |= f.kind == FrameKind::Pkg && f.name == package); + found +} + +/// The probe's bound for an experiment, in ms. +pub fn bound_ms(model: &BuildModel, e: Experiment) -> Option { + match e { + Experiment::Preamble => model.run_phases().preamble.map(crate::report::ms), + Experiment::Externalize => Some(crate::report::ms( + model.env_table().iter().find(|r| r.name == "tikzpicture").map(|r| r.total).unwrap_or(0), + )), + Experiment::Draft => None, + } +} + +/// Why an experiment cannot run on this document here, if it cannot. +fn not_run_reason(e: Experiment, model: &BuildModel, opts: &WhatifOptions) -> Option { + match e { + Experiment::Preamble if crate::doctor::tex_file("mylatexformat.ltx").is_none() => Some(format!( + "mylatexformat is not installed ({})", + crate::doctor::Installer::detect().install("mylatexformat") + )), + Experiment::Externalize if !loads(model, "tikz") => Some("the document does not load tikz".into()), + Experiment::Externalize if model.root.find("tikzlibraryexternal.code.tex").is_some() => { + Some("the document already externalizes its pictures".into()) + } + Experiment::Externalize if !opts.shell_escape => { + Some("needs --shell-escape (tikz builds each picture with a shell command)".into()) + } + _ => None, + } +} + +/// What the output comparison looks at: page count, each page's words, a coarse render. +pub(crate) struct PdfLook { + pages: usize, + words: Vec>, + render: Option>>, +} + +/// The dpi and tolerance of the render check: `unused`'s coarse ink check. +const RENDER_DPI: u32 = 24; +const RENDER_PER_MILLE: usize = 5; + +fn look_at(pdf: &Path) -> Option { + let text = crate::unused::pdftotext(pdf)?; + let pages = crate::unused::pdfinfo(pdf).and_then(|i| i.get("Pages").and_then(|p| p.parse().ok())).unwrap_or(0); + Some(PdfLook { pages, words: page_words(&text), render: crate::unused::render(pdf, RENDER_DPI) }) +} + +/// Each page's words, in reading order (pages are split at form feeds). Words rather than +/// lines: an externalized picture is an included PDF, and `pdftotext -layout` can place its +/// text a column or a line away from where it put the same text drawn inline. +pub(crate) fn page_words(text: &str) -> Vec> { + let mut pages: Vec> = + text.split('\u{c}').map(|p| p.split_whitespace().map(str::to_string).collect()).collect(); + while pages.last().is_some_and(|p| p.is_empty()) { + pages.pop(); + } + pages +} + +/// Where two pages' words differ, sorted, so order within a page does not count: the first +/// word one page has more of than the other. +pub(crate) fn words_differ(a: &[String], b: &[String]) -> Option { + fn count(ws: &[String]) -> std::collections::BTreeMap<&str, i64> { + let mut m = std::collections::BTreeMap::new(); + for w in ws { + *m.entry(w.as_str()).or_default() += 1; + } + m + } + let (ca, cb) = (count(a), count(b)); + let keys: std::collections::BTreeSet<&str> = ca.keys().chain(cb.keys()).copied().collect(); + keys.into_iter().find_map(|k| { + let (x, y) = (ca.get(k).copied().unwrap_or(0), cb.get(k).copied().unwrap_or(0)); + match x.cmp(&y) { + std::cmp::Ordering::Equal => None, + std::cmp::Ordering::Greater => Some(format!("{k:?} only in the baseline")), + std::cmp::Ordering::Less => Some(format!("{k:?} only in the variant")), + } + }) +} + +fn compare_output(base: &PdfLook, pdf: &Path) -> Output { + let Some(v) = look_at(pdf) else { + return Output::Changed { evidence: "no readable PDF".into() }; + }; + if v.pages != base.pages { + return Output::Changed { + evidence: format!("{} page{} vs {}", v.pages, if v.pages == 1 { "" } else { "s" }, base.pages), + }; + } + for (i, (a, b)) in base.words.iter().zip(&v.words).enumerate() { + if let Some(w) = words_differ(a, b) { + return Output::Changed { evidence: format!("page {}: {w}", i + 1) }; + } + } + if base.words.len() != v.words.len() { + return Output::Changed { evidence: "the text has a different number of pages".into() }; + } + if let (Some(a), Some(b)) = (&base.render, &v.render) + && let Some((page, differing, total)) = crate::unused::render_diff(a, b, RENDER_PER_MILLE) + { + return Output::Changed { + evidence: format!( + "page {page} looks different ({differing} of {total} pixels at {RENDER_DPI} dpi; same words)" + ), + }; + } + Output::Identical +} + +/// One experiment's copy of the project and what it measured so far. +struct Lane { + experiment: Experiment, + step: Step, + dir: PathBuf, + warmup_ms: f64, + one_time_ms: Option, + runs_ms: Vec, + failed: Option, +} + +/// Save a build directory's logs and engine output to the cache, so a failure can be read +/// after the temp copy is gone. +fn keep_logs(dir: &Path, to: &Path) { + if std::fs::create_dir_all(to).is_err() { + return; + } + for e in std::fs::read_dir(dir).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + if name.starts_with("prumo-") && name.ends_with(".txt") || name.ends_with(".log") { + let _ = std::fs::copy(e.path(), to.join(&name)); + } + } +} + +/// The whole pass. `progress` gets one line per step. +pub fn run(opts: &WhatifOptions, mut progress: impl FnMut(&str)) -> Result { + if crate::doctor::which("pdftotext").is_none() { + return Err(WhatifError::NoPdftotext); + } + if opts.explicit && opts.tries.contains(&Experiment::Externalize) && !opts.shell_escape { + return Err(WhatifError::NeedsShellEscape); + } + let (job, project) = run::job_of(&opts.main)?; + let out_root = match &opts.out { + Some(o) => o.clone(), + None => run::default_out(&opts.main)?, + }; + let logs = out_root.join(&job).join("whatif"); + progress(&format!("probed run and logs in {}", crate::doctor::host_path(&out_root.join(&job)))); + let probed = probed_run(opts, &out_root)?; + let model = run::model_of(&probed)?; + let probed_ms = crate::report::ms(model.wall.unwrap_or(model.end)); + let timeout = std::time::Duration::from_secs_f64(if opts.timeout_secs > 0 { + opts.timeout_secs as f64 + } else { + (probed_ms / 1000.0 * 10.0).max(30.0) + }); + // Every picture is a build of its own on the first externalized run. + let pictures = model.env_table().iter().find(|r| r.name == "tikzpicture").map(|r| r.count).unwrap_or(0); + + let tmp = tempfile::Builder::new().prefix("prumo-whatif-").tempdir().map_err(|source| WhatifError::Io { + what: "create", + path: std::env::temp_dir(), + source, + })?; + let stage = |name: &str| -> Result { + let d = tmp.path().join(name); + run::copy_tree(&project, &d)?; + Ok(d) + }; + let mut rows: Vec<(Experiment, Option, Option)> = Vec::new(); + let mut lanes: Vec = Vec::new(); + let mut tries: Vec = Vec::new(); + for e in &opts.tries { + if !tries.contains(e) { + tries.push(*e); + } + } + for e in tries { + let bound = bound_ms(&model, e); + match not_run_reason(e, &model, opts) { + Some(reason) => rows.push((e, bound, Some(Outcome::NotRun { reason }))), + None => { + rows.push((e, bound, None)); + let step = match e { + Experiment::Preamble => Step::Precompiled, + Experiment::Draft => Step::Draft, + Experiment::Externalize => Step::Externalize, + }; + lanes.push(Lane { + experiment: e, + step, + dir: stage(e.name())?, + warmup_ms: 0.0, + one_time_ms: None, + runs_ms: vec![], + failed: None, + }); + } + } + } + let base_dir = stage("baseline")?; + let baseline_failed = |(code, e): (Option, String)| { + keep_logs(&base_dir, &logs.join("baseline")); + WhatifError::BaselineFailed { code, tail: e } + }; + + // Warm-up: the baseline, then each variant, the format first. A variant that fails + // here is out; its row says why. + let runs = opts.runs.max(1); + progress(&format!( + "{} on {}: a warm-up, then {runs} timed run{} of the baseline and {} variant{}, one build at a time", + job, + opts.engine.command(), + if runs == 1 { "" } else { "s" }, + lanes.len(), + if lanes.len() == 1 { "" } else { "s" } + )); + let base_warm = build(&base_dir, Step::Baseline, opts, &job, timeout).map_err(baseline_failed)?; + for lane in &mut lanes { + if lane.experiment == Experiment::Preamble { + progress("preamble: building the format"); + match build(&lane.dir, Step::Format, opts, &job, timeout) { + Ok(ms) => lane.one_time_ms = Some(ms), + Err((_, e)) => { + lane.failed = Some(failure(Step::Format, &e)); + continue; + } + } + } + progress(&format!("{}: warm-up", lane.experiment.name())); + let t = if lane.step == Step::Externalize { timeout * (1 + pictures) } else { timeout }; + match build(&lane.dir, lane.step, opts, &job, t) { + Ok(ms) => { + lane.warmup_ms = ms; + if lane.step == Step::Externalize { + lane.one_time_ms = Some(ms); + } + } + Err((_, e)) => lane.failed = Some(failure(lane.step, &e)), + } + } + let mut base_runs = Vec::new(); + for r in 1..=runs { + progress(&format!("run {r}/{runs}")); + base_runs.push(build(&base_dir, Step::Baseline, opts, &job, timeout).map_err(baseline_failed)?); + for lane in lanes.iter_mut().filter(|l| l.failed.is_none()) { + match build(&lane.dir, lane.step, opts, &job, timeout) { + Ok(ms) => lane.runs_ms.push(ms), + Err((_, e)) => lane.failed = Some(failure(lane.step, &e)), + } + } + } + let baseline = Sample::new(base_warm, base_runs); + let base_pdf = base_dir.join(format!("{job}.pdf")); + let base_out = look_at(&base_pdf) + .ok_or_else(|| WhatifError::BaselineFailed { code: Some(0), tail: format!("no readable {job}.pdf") })?; + keep_logs(&base_dir, &logs.join("baseline")); + + let mut experiments = Vec::new(); + for (e, bound, outcome) in rows { + let outcome = match outcome { + Some(o) => o, + None => { + let lane = lanes.iter().find(|l| l.experiment == e).expect("a lane per row that ran"); + keep_logs(&lane.dir, &logs.join(e.name())); + match &lane.failed { + Some(error) => Outcome::Failed { error: error.clone() }, + None => measured(&baseline, lane, &base_out, &job), + } + } + }; + experiments.push(Row { experiment: e, experimental: e.experimental(), bound_ms: bound, outcome }); + } + Ok(WhatifReport { + schema: crate::JSON_SCHEMA, + job, + engine: opts.engine, + runs, + shell_escape: opts.shell_escape, + baseline, + baseline_pages: base_out.pages, + experiments, + }) +} + +fn measured(baseline: &Sample, lane: &Lane, base_out: &PdfLook, job: &str) -> Outcome { + let variant = Sample::new(lane.warmup_ms, lane.runs_ms.clone()); + let output = match lane.step { + Step::Draft => Output::NotCompared { reason: "draft mode writes no PDF".into() }, + _ => compare_output(base_out, &lane.dir.join(format!("{job}.pdf"))), + }; + outcome_of(baseline, variant, lane.one_time_ms, output) +} + +/// The smallest difference that is not noise: wall times are polled every millisecond +/// (`run::spawn_timed`), so each run can read up to 1 ms late and two runs can differ by 2 ms +/// on resolution alone. +const MIN_NOISE_MS: f64 = 2.0; + +/// The numbers of a measured row. +pub(crate) fn outcome_of(baseline: &Sample, variant: Sample, one_time_ms: Option, output: Output) -> Outcome { + let saved_ms = round3(baseline.median_ms - variant.median_ms); + let saved_pct = + if baseline.median_ms > 0.0 { (saved_ms / baseline.median_ms * 1000.0).round() / 10.0 } else { 0.0 }; + let within_noise = saved_ms.abs() <= baseline.spread_ms.max(variant.spread_ms).max(MIN_NOISE_MS); + Outcome::Measured { variant, saved_ms, saved_pct, within_noise, one_time_ms, output } +} + +/// How to make the change for real, for the notes under the table. +fn how(e: Experiment, engine: EngineCmd, job: &str) -> String { + match e { + Experiment::Preamble => format!( + "once per preamble change, {} -ini -jobname={job}-fmt \"&{}\" mylatexformat.ltx {job}.tex; then {} -fmt={job}-fmt {job}.tex", + ini_binary(engine), + engine.command(), + engine.command() + ), + Experiment::Draft => format!( + "{} {} {job}.tex while writing, and a full build before reading the PDF. The saving is what writing the PDF and including images cost", + engine.command(), + draft_flag(engine) + ), + Experiment::Externalize => format!( + "\\usetikzlibrary{{external}}\\tikzexternalize in the preamble and {} -shell-escape. A picture is rebuilt only when it changes", + engine.command() + ), + } +} + +const EXPERIMENTAL: &str = " * experimental: tikz's external library breaks remember picture, overlays and references between pictures. Check the PDF, not only this table."; + +pub fn text(r: &WhatifReport) -> String { + use std::fmt::Write as _; + let mut s = String::new(); + let _ = writeln!( + s, + "== prumo whatif: {} ({}, median of {} run{} after a warm-up, {} page{}{})", + r.job, + r.engine.command(), + r.runs, + if r.runs == 1 { "" } else { "s" }, + r.baseline_pages, + if r.baseline_pages == 1 { "" } else { "s" }, + if r.shell_escape { ", -shell-escape" } else { "" } + ); + // The experimental mark goes on rows that ran; a row that did not run has nothing to doubt. + let ran = |row: &Row| !matches!(row.outcome, Outcome::NotRun { .. }); + let label = |row: &Row| format!("{}{}", row.experiment.name(), if row.experimental && ran(row) { "*" } else { "" }); + let _ = writeln!( + s, + " {:<13} {:>9} {:>9} {:<14} {:>8} {:>9} output", + "experiment", "baseline", "variant", "saved", "bound", "one-time" + ); + let ms0 = |v: f64| format!("{v:.0} ms"); + for row in &r.experiments { + let bound = row.bound_ms.map(ms0).unwrap_or_else(|| "-".into()); + match &row.outcome { + Outcome::Measured { variant, saved_ms, saved_pct, within_noise, one_time_ms, output } => { + let saved = if *within_noise { + "within noise".to_string() + } else if *saved_ms >= 0.0 { + format!("{saved_ms:.0} ms {saved_pct:.0}%") + } else { + format!("{:.0} ms slower", -saved_ms) + }; + let output = match output { + Output::Identical => "identical".to_string(), + Output::Changed { evidence } => format!("CHANGED ({evidence})"), + Output::NotCompared { .. } => "n/a (no PDF)".to_string(), + }; + let _ = writeln!( + s, + " {:<13} {:>9} {:>9} {:<14} {:>8} {:>9} {output}", + label(row), + ms0(r.baseline.median_ms), + ms0(variant.median_ms), + saved, + bound, + one_time_ms.map(ms0).unwrap_or_else(|| "-".into()), + ); + } + Outcome::Failed { error } => { + let _ = writeln!(s, " {:<13} {error}", label(row)); + } + Outcome::NotRun { reason } => { + let _ = writeln!(s, " {:<13} not run: {reason}", label(row)); + } + } + } + let measured: Vec<(&Row, &Sample)> = r + .experiments + .iter() + .filter_map(|row| match &row.outcome { + Outcome::Measured { variant, .. } => Some((row, variant)), + _ => None, + }) + .collect(); + let experimental = r.experiments.iter().any(|row| row.experimental && ran(row)); + if measured.is_empty() && !experimental { + return s; + } + s.push('\n'); + if measured.is_empty() { + let _ = writeln!(s, "{EXPERIMENTAL}"); + return s; + } + let spreads: Vec = std::iter::once(format!("baseline {:.0} ms", r.baseline.spread_ms)) + .chain(measured.iter().map(|(row, v)| format!("{} {:.0} ms", row.experiment.name(), v.spread_ms))) + .collect(); + let _ = writeln!( + s, + " spread (slowest run minus fastest): {}. A difference no larger than the larger spread (2 ms at least) is noise.", + spreads.join(", ") + ); + if measured.iter().any(|(row, _)| row.bound_ms.is_some()) { + let _ = writeln!( + s, + " bound: the most the probe says the change can save (the preamble phase; the tikzpicture environments)." + ); + } + for (row, _) in &measured { + let Outcome::Measured { saved_ms, within_noise, output, .. } = &row.outcome else { continue }; + let name = row.experiment.name(); + if let Output::Changed { evidence } = output + && !within_noise + { + let _ = + writeln!(s, " {name}: the output changed ({evidence}), so {saved_ms:.0} ms is not a clean saving."); + } + if let Some(b) = row.bound_ms + && !within_noise + && *saved_ms > b + r.baseline.spread_ms + { + let _ = writeln!( + s, + " {name}: saved {saved_ms:.0} ms, more than the probe's bound of {b:.0} ms. One of the two is off; run again with more --runs." + ); + } + } + if experimental { + let _ = writeln!(s, "{EXPERIMENTAL}"); + } + s.push('\n'); + for (row, _) in &measured { + let _ = writeln!(s, " {}: {}", row.experiment.name(), how(row.experiment, r.engine, &r.job)); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn output_is_compared_by_the_words_on_each_page() { + let base = "A title\n x = 1 (1)\n\u{c}Table 1\n a b\n\u{c}"; + // The same words, one moved a line down and the table a column right. + let moved = "A title\n\n (1) x = 1\n\u{c} Table 1\n a b\n\u{c}"; + let (a, b) = (page_words(base), page_words(moved)); + assert_eq!(a.len(), 2, "the trailing form feed is not a page"); + assert!(a.iter().zip(&b).all(|(x, y)| words_differ(x, y).is_none())); + let changed = page_words("A title\n x = 2 (1)\n\u{c}Table 1\n a b\n"); + assert_eq!(words_differ(&a[0], &changed[0]).as_deref(), Some("\"1\" only in the baseline")); + } + + #[test] + fn median_and_spread() { + assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0); + assert_eq!(median(&[4.0, 1.0, 3.0, 2.0]), 2.5); + assert_eq!(spread(&[3.0, 1.0, 2.0]), 2.0); + assert_eq!((median(&[]), spread(&[])), (0.0, 0.0)); + } + + #[test] + fn noise_is_the_larger_spread() { + let base = Sample::new(900.0, vec![880.0, 870.0, 890.0]); + assert_eq!((base.median_ms, base.spread_ms), (880.0, 20.0)); + let Outcome::Measured { saved_ms, saved_pct, within_noise, .. } = + outcome_of(&base, Sample::new(600.0, vec![564.0, 560.0, 570.0]), Some(622.0), Output::Identical) + else { + panic!() + }; + assert_eq!((saved_ms, within_noise), (316.0, false)); + assert_eq!(saved_pct, 35.9); + let Outcome::Measured { within_noise, .. } = + outcome_of(&base, Sample::new(0.0, vec![847.0, 820.0, 870.0]), None, Output::Identical) + else { + panic!() + }; + assert!(within_noise, "33 ms saved, the variant's spread is 50 ms"); + let Outcome::Measured { saved_ms, within_noise, .. } = + outcome_of(&base, Sample::new(0.0, vec![950.0, 951.0, 952.0]), None, Output::Identical) + else { + panic!() + }; + assert_eq!((saved_ms, within_noise), (-71.0, false), "slower is a result too"); + let quiet = Sample::new(0.0, vec![137.0, 137.0, 137.0]); + let Outcome::Measured { within_noise, .. } = + outcome_of(&quiet, Sample::new(0.0, vec![136.0, 136.0, 136.0]), None, Output::Identical) + else { + panic!() + }; + assert!(within_noise, "1 ms is below the 1 ms polling of two runs"); + } + + #[test] + fn command_lines() { + let a = |s, e, se| argv(s, e, "main", se); + assert_eq!(a(Step::Baseline, EngineCmd::Pdflatex, false), ["pdflatex", "-interaction=nonstopmode", "main.tex"]); + assert_eq!( + a(Step::Format, EngineCmd::Pdflatex, false), + [ + "pdftex", + "-ini", + "-interaction=nonstopmode", + "-jobname=main-fmt", + "&pdflatex", + "mylatexformat.ltx", + "main.tex" + ] + ); + assert_eq!( + a(Step::Precompiled, EngineCmd::Xelatex, true), + ["xelatex", "-interaction=nonstopmode", "-shell-escape", "-fmt=main-fmt", "main.tex"] + ); + assert_eq!(a(Step::Draft, EngineCmd::Pdflatex, false)[2], "-draftmode"); + assert_eq!(a(Step::Draft, EngineCmd::Xelatex, false)[2], "-no-pdf"); + assert_eq!(a(Step::Draft, EngineCmd::Lualatex, false)[2], "--draftmode"); + let x = a(Step::Externalize, EngineCmd::Lualatex, false); + assert_eq!(&x[..4], ["lualatex", "-interaction=nonstopmode", "-shell-escape", "-jobname=main"]); + assert_eq!( + x[4], + "\\AddToHook{package/tikz/after}{\\usetikzlibrary{external}\\tikzexternalize[system call={lualatex \\tikzexternalcheckshellescape -halt-on-error -interaction=batchmode -jobname \"\\image\" \"\\string\\AddToHook{package/tikz/after}{\\string\\usetikzlibrary{external}\\string\\tikzexternalize}\\texsource\"}]}\\input{main}" + ); + assert_eq!(x[4].matches('{').count(), x[4].matches('}').count()); + } + + #[test] + fn first_error_of_a_failed_build() { + let xetex = "entering extended mode\n! Can't \\dump a format with native fonts or font-mappings.\n<*> &xelatex mylatexformat.ltx main.tex\n"; + assert_eq!(first_error(xetex, Some(1)), "! Can't \\dump a format with native fonts or font-mappings."); + // LuaTeX's dump on TeX Live 2026: a Lua error, wrapped at 79 columns, and no `!`. + let luatex = "(/usr/local/texlive/2026/texmf-dist/tex/latex/lm/t1lmss.fd\nFile: t1lmss.fd 2015/05/01 v1.6.1 Font defs for Latin Modern\n))...f-dist/tex/luatex/lua-uni-algos/lua-uni-stage-tables.lua:141: attempt to pe\nrform arithmetic on a string value (local 'key').\n<*> &lualatex mylatexformat.ltx main.tex\n"; + assert_eq!( + first_error(luatex, Some(1)), + "lua-uni-stage-tables.lua:141: attempt to perform arithmetic on a string value (local 'key')." + ); + assert_eq!(first_error("nothing to see\n", Some(3)), "exit 3"); + assert_eq!(lua_error("see foo.lua:x: no"), None); + } + + fn report(rows: Vec) -> WhatifReport { + WhatifReport { + schema: 1, + job: "main".into(), + engine: EngineCmd::Pdflatex, + runs: 5, + shell_escape: true, + baseline: Sample::new(900.0, vec![880.0, 875.0, 885.0, 878.0, 890.0]), + baseline_pages: 2, + experiments: rows, + } + } + + fn row(e: Experiment, bound: Option, outcome: Outcome) -> Row { + Row { experiment: e, experimental: e.experimental(), bound_ms: bound, outcome } + } + + /// The shape of the brief's example: a clean saving, one within noise, one whose output + /// changed. + #[test] + fn table_says_what_is_noise_and_what_changed() { + let r = report(vec![]); + let b = r.baseline.clone(); + let r = report(vec![ + row( + Experiment::Preamble, + Some(372.0), + outcome_of( + &b, + Sample::new(0.0, vec![564.0, 560.0, 566.0, 563.0, 570.0]), + Some(622.0), + Output::Identical, + ), + ), + row( + Experiment::Draft, + None, + outcome_of( + &b, + Sample::new(0.0, vec![847.0, 830.0, 870.0, 850.0, 845.0]), + None, + Output::NotCompared { reason: "draft mode writes no PDF".into() }, + ), + ), + row( + Experiment::Externalize, + Some(300.0), + outcome_of( + &b, + Sample::new(1303.0, vec![464.0, 460.0, 470.0, 466.0, 462.0]), + Some(1303.0), + Output::Changed { evidence: "3 pages vs 2".into() }, + ), + ), + ]); + let t = text(&r); + assert!(t.contains("preamble 880 ms 564 ms 316 ms 36% 372 ms 622 ms identical"), "{t}"); + assert!( + t.contains("draft 880 ms 847 ms within noise - - n/a (no PDF)"), + "{t}" + ); + assert!(t.contains("externalize* 880 ms 464 ms 416 ms 47%"), "{t}"); + assert!(t.contains("CHANGED (3 pages vs 2)"), "{t}"); + assert!(t.contains("externalize: the output changed (3 pages vs 2), so 416 ms is not a clean saving"), "{t}"); + assert!(t.contains("externalize: saved 416 ms, more than the probe's bound of 300 ms"), "{t}"); + assert!(!t.contains("preamble: saved"), "316 ms is inside the bound of 372: {t}"); + assert!(t.contains("* experimental"), "{t}"); + assert!(t.contains("spread (slowest run minus fastest): baseline 15 ms, preamble 10 ms, draft 40 ms"), "{t}"); + assert!(t.contains("pdftex -ini -jobname=main-fmt \"&pdflatex\" mylatexformat.ltx main.tex"), "{t}"); + assert!(t.contains("draft: pdflatex -draftmode main.tex"), "{t}"); + let json = serde_json::to_value(&r).unwrap(); + assert_eq!(json["experiments"][0]["status"], "measured"); + assert_eq!(json["experiments"][0]["output"]["status"], "identical"); + assert_eq!(json["experiments"][1]["within_noise"], true); + assert_eq!(json["experiments"][2]["experimental"], true); + assert_eq!(json["experiments"][2]["output"]["evidence"], "3 pages vs 2"); + } + + #[test] + fn failures_and_skips_are_rows() { + let r = report(vec![ + row( + Experiment::Preamble, + Some(300.0), + Outcome::Failed { + error: "could not precompile: ! Can't \\dump a format with native fonts or font-mappings.".into(), + }, + ), + row( + Experiment::Externalize, + Some(40.0), + Outcome::NotRun { reason: "the document does not load tikz".into() }, + ), + ]); + let t = text(&r); + assert!(t.contains("preamble could not precompile: ! Can't \\dump a format"), "{t}"); + assert!(t.contains("externalize not run: the document does not load tikz"), "{t}"); + assert!(!t.contains("spread") && !t.contains("experimental"), "nothing ran, no notes: {t}"); + let failed = report(vec![row( + Experiment::Externalize, + Some(40.0), + Outcome::Failed { error: "could not externalize: ! Package tikz Error".into() }, + )]); + let t = text(&failed); + assert!(t.contains("externalize* could not externalize") && t.contains("* experimental"), "{t}"); + let json = serde_json::to_value(&r).unwrap(); + assert_eq!(json["experiments"][0]["status"], "failed"); + assert_eq!(json["experiments"][1]["status"], "not-run"); + } +} diff --git a/tests/fixtures.rs b/tests/fixtures.rs index 59ac66d..ad48037 100644 --- a/tests/fixtures.rs +++ b/tests/fixtures.rs @@ -61,8 +61,8 @@ fn artifact_sets() -> Vec { out.sort_by(|a, b| a.dir.cmp(&b.dir)); assert_eq!( out.len(), - 28, - "3 fixtures × 2 artifact sets × 3 engines, beamer, biblatex and environments × 3, fontspec on xelatex: {:?}", + 31, + "3 fixtures × 2 artifact sets × 3 engines, beamer, biblatex, environments and whatif × 3, fontspec on xelatex: {:?}", out.iter().map(|s| &s.dir).collect::>() ); out @@ -205,6 +205,12 @@ fn tlpdb_excerpt_resolves_the_fixtures_main_packages() { assert_eq!(pkg_of("longtable.sty"), Some("tools"), "{label}"); assert_eq!(pkg_of("listings.sty"), Some("listings"), "{label}"); } + "whatif" => { + assert_eq!(pkg_of("siunitx.sty"), Some("siunitx"), "{label}"); + assert_eq!(pkg_of("booktabs.sty"), Some("booktabs"), "{label}"); + assert_eq!(pkg_of("tikz.sty"), Some("pgf"), "{label}"); + assert_eq!(pkg_of("hyperref.sty"), Some("hyperref"), "{label}"); + } other => panic!("unknown fixture {other}"), } if s.fixture != "beamer" { @@ -337,6 +343,13 @@ fn log_file_opens_versus_hook_records() { outside.iter().all(|n| n.ends_with(".aux") || n.ends_with(".out") || blx(n)), "{label}: outside hooks: {outside:?}" ); + // siunitx asks for color at begindocument/before, after tikz loaded xcolor, which + // claims color.sty: the kernel opens the file and does not read it. LuaTeX's + // recorder does not list the open. + let opened: Vec = + c.opened_not_read.iter().map(|o| o.path.file_name().unwrap().to_string_lossy().into_owned()).collect(); + let expected: &[&str] = if s.fixture == "whatif" && s.engine_dir != "lualatex" { &["color.sty"] } else { &[] }; + assert_eq!(opened, expected, "{label}: opened but not read"); if s.fixture == "biblatex" { assert!(outside.iter().any(|n| n == "biblatex.def"), "{label}: doctor must list biblatex.def: {outside:?}"); assert!( @@ -453,6 +466,11 @@ fn packages_resolve_with_attribution() { assert_eq!(via(n), "main.tex", "{label}: {n}"); } } + "whatif" => { + for n in ["amsmath", "siunitx", "booktabs", "pgf", "hyperref"] { + assert_eq!(via(n), "main.tex", "{label}: {n}"); + } + } _ => unreachable!(), } let used = r.packages.iter().filter(|p| matches!(p.reason, Reason::Used { .. })).count(); @@ -598,7 +616,7 @@ fn fonts_match_the_oracle() { assert!(t.contains("-- loaded --") && t.contains("-- in the PDF --"), "{t}"); checked += 1; } - assert_eq!(checked, 19, "every TL2026 set has a fontmap.txt"); + assert_eq!(checked, 22, "every TL2026 set has a fontmap.txt"); } /// `prune::plan` on the excerpt with the three fixtures' TL2026 runs: nothing a document @@ -637,7 +655,7 @@ fn prune_plan_on_the_excerpt() { .collect(), }); } - assert_eq!(runs.len(), 8, "6 pdflatex + basic/xelatex + fontspec/xelatex"); + assert_eq!(runs.len(), 9, "7 pdflatex + basic/xelatex + fontspec/xelatex"); let biblatex = runs.iter().find(|r| r.main.ends_with("paper.tex")).unwrap(); assert_eq!(biblatex.tools, ["biber"], "a .bcf was written"); assert!(runs.iter().filter(|r| !r.main.ends_with("paper.tex")).all(|r| r.tools.is_empty())); @@ -651,7 +669,7 @@ fn prune_plan_on_the_excerpt() { env: vec![], }; let p = prune::plan(&db, &runs, &opts); - assert_eq!(p.documents.len(), 8); + assert_eq!(p.documents.len(), 9); assert_eq!(p.installed, 128, "130 stanzas minus the two 00texlive.* ones"); // Everything any document read stays; the acceptance list from docs/07-phases.md. for n in [ @@ -920,7 +938,7 @@ fn phases_and_environments_match_the_oracle() { assert!(j["phases"]["finalization_ms"].is_null(), "{label}: no wall time without a run"); assert!(j["phases"]["startup_ms"].as_f64().unwrap() > 0.0, "{label}"); } - assert_eq!(checked, 19, "every TeX Live 2026 set"); + assert_eq!(checked, 22, "every TeX Live 2026 set"); } /// fixtures/environments/main.tex: every claim in its header, on all three engines. @@ -970,6 +988,30 @@ fn environments_fixture_claims() { } } +/// fixtures/whatif/main.tex, claim 7: the bounds `whatif` prints come from the probe's +/// records. The rest of its header is tests/tex.rs::whatif_fixture_claims. +#[test] +fn whatif_fixture_bounds() { + use prumo::whatif::{Experiment, bound_ms}; + let sets: Vec = artifact_sets().into_iter().filter(|s| s.fixture == "whatif").collect(); + assert_eq!(sets.len(), 3); + for s in sets { + let label = s.engine_dir.clone(); + let recs = records::parse_file(&s.dir.join(format!("{}.prumo", s.job))).unwrap(); + let m = prumo::model::BuildModel::from_records(&recs, &label).unwrap(); + let envs = m.env_table(); + let tikz = envs.iter().find(|r| r.name == "tikzpicture").unwrap(); + assert_eq!(tikz.count, 1, "{label}: one picture"); + assert_eq!(bound_ms(&m, Experiment::Externalize), Some(prumo::report::ms(tikz.total)), "{label}"); + let p = m.run_phases(); + let preamble = p.preamble.unwrap(); + assert_eq!(bound_ms(&m, Experiment::Preamble), Some(prumo::report::ms(preamble)), "{label}"); + let others = [Some(p.startup), p.begin_document, p.typesetting, Some(p.shipout), p.end_document]; + assert!(others.iter().all(|o| o.unwrap() < preamble), "{label}: the preamble is the longest phase: {p:?}"); + assert_eq!(bound_ms(&m, Experiment::Draft), None); + } +} + /// `scripts/make_artifacts.sh` loads the probe the way prumo does. #[test] fn make_artifacts_watches_the_default_environments() { diff --git a/tests/tex.rs b/tests/tex.rs index 90f6a71..a4c27fb 100644 --- a/tests/tex.rs +++ b/tests/tex.rs @@ -11,7 +11,7 @@ use std::sync::Once; const ENGINES: [EngineCmd; 3] = [EngineCmd::Pdflatex, EngineCmd::Xelatex, EngineCmd::Lualatex]; /// Fixture, job, engines it builds on (fontspec pins XeTeX with a magic comment; biblatex /// has a single-pass set here, its latexmk claims are in `biblatex_through_latexmk`). -const FIXTURES: [(&str, &str, &[EngineCmd]); 7] = [ +const FIXTURES: [(&str, &str, &[EngineCmd]); 8] = [ ("basic", "main", &ENGINES), ("custom-class", "main", &ENGINES), ("corpus-b", "notes", &ENGINES), @@ -19,6 +19,7 @@ const FIXTURES: [(&str, &str, &[EngineCmd]); 7] = [ ("biblatex", "paper", &ENGINES), ("environments", "main", &ENGINES), ("fontspec", "glyphs", &[EngineCmd::Xelatex]), + ("whatif", "main", &ENGINES), ]; fn fixtures_root() -> PathBuf { @@ -986,3 +987,82 @@ fn probe_loaded_from_the_first_line() { assert_eq!(m.pages.len(), 3); assert!(m.run_phases().finalization.is_none()); } + +/// fixtures/whatif/main.tex: claims 1 to 6 and 8 of its header (7 is in tests/fixtures.rs). +#[test] +fn whatif_fixture_claims() { + use prumo::whatif::{self, Experiment, Outcome, Output, WhatifError, WhatifOptions}; + let env = setup(); + let dir = fixture_copy("whatif"); + let before: Vec<_> = std::fs::read_dir(dir.path()).unwrap().flatten().map(|e| e.file_name()).collect(); + let main = dir.path().join("main.tex"); + let opts = WhatifOptions { + main: main.clone(), + engine: EngineCmd::Pdflatex, + tries: Experiment::ALL.to_vec(), + explicit: false, + runs: 2, + shell_escape: true, + timeout_secs: 0, + out: None, + env: env.clone(), + }; + for engine in ENGINES { + let label = engine.command(); + let r = whatif::run(&WhatifOptions { engine, ..opts.clone() }, |_| {}) + .unwrap_or_else(|e| panic!("{label}: a variant's failure is a row, not an error: {e}")); + let t = whatif::text(&r); + let row = |e: Experiment| &r.experiments.iter().find(|x| x.experiment == e).unwrap().outcome; + match (engine, row(Experiment::Preamble)) { + // 1. pdfLaTeX: more than a third saved, outside the noise, same output. + ( + EngineCmd::Pdflatex, + Outcome::Measured { + saved_pct, + within_noise: false, + output: Output::Identical, + one_time_ms: Some(_), + .. + }, + ) => assert!(*saved_pct > 100.0 / 3.0, "{label}: {t}"), + // 2. XeLaTeX cannot dump the class's OpenType font. + (EngineCmd::Xelatex, Outcome::Failed { error }) => assert!( + error.starts_with("could not precompile: ! Can't \\dump a format with native fonts"), + "{label}: {error}" + ), + // 3. LuaLaTeX's dump fails too. + (EngineCmd::Lualatex, Outcome::Failed { error }) => { + assert!(error.starts_with("could not precompile: "), "{label}: {error}") + } + (_, other) => panic!("{label}: preamble {other:?}\n{t}"), + } + assert!(t.contains("preamble"), "{t}"); + // 4. Externalized, the output is the same, and the first run builds the picture. + match row(Experiment::Externalize) { + Outcome::Measured { output: Output::Identical, one_time_ms: Some(first), variant, .. } => { + assert!(*first > variant.median_ms, "{label}: first run {first} ms, then {} ms", variant.median_ms) + } + other => panic!("{label}: externalize {other:?}\n{t}"), + } + // 6. Draft runs and is not compared. + assert!( + matches!(row(Experiment::Draft), Outcome::Measured { output: Output::NotCompared { .. }, .. }), + "{label}: {t}" + ); + assert_eq!(r.baseline_pages, 1, "{label}"); + } + // 5. Without --shell-escape: refused when asked for, not run by default. + let no_se = WhatifOptions { shell_escape: false, tries: vec![Experiment::Externalize], ..opts.clone() }; + assert!(matches!( + whatif::run(&WhatifOptions { explicit: true, ..no_se.clone() }, |_| {}), + Err(WhatifError::NeedsShellEscape) + )); + let r = whatif::run(&no_se, |_| {}).unwrap(); + match &r.experiments[0].outcome { + Outcome::NotRun { reason } => assert!(reason.contains("--shell-escape"), "{reason}"), + other => panic!("{other:?}"), + } + // 8. The document's directory is unchanged. + let after: Vec<_> = std::fs::read_dir(dir.path()).unwrap().flatten().map(|e| e.file_name()).collect(); + assert_eq!(before, after); +} From c4b4cdcf00b771bcdd097a3b24dec421cfd6400f Mon Sep 17 00:00:00 2001 From: Kevin Raymond Date: Tue, 10 Nov 2026 12:35:41 -0600 Subject: [PATCH 2/3] Benchmarks: ten documents, the probe's cost, whatif, charts Seven public documents join the three that live here: the ACM sigconf sample with BibTeX and with biblatex, the metropolis demo, florilegium's manual, the HoTT book, Open Logic and the Stacks Project. corpus.toml pins each one to a commit and fetch.py takes it from its own repository, so nothing under share-alike or GFDL terms is stored here. kaobook is left out, because thmtools' sibling= fails on the 2026-11-01 kernel. bench.py gains --overhead (builds with and without the probe, in turns) and --whatif. charts.py draws four charts, light and dark, from the JSON: where the time goes, what the probe costs, package times on pdfLaTeX against LuaLaTeX, and what whatif saves. Color means phase in all of them, and every pair of colors is checked for colorblind readers. The benchmark README has the charts with their tables. The top-level README links it and says where the precompiled preamble fails. Findings 60 to 64. --- README.md | 55 +- docs/10-verified-findings.md | 39 + docs/STATUS.md | 67 +- docs/benchmarks/README.md | 468 +- docs/benchmarks/bench.py | 364 +- docs/benchmarks/charts.py | 522 + docs/benchmarks/corpus.toml | 116 + docs/benchmarks/fetch.py | 86 + docs/benchmarks/img/overhead-dark.svg | 363 + docs/benchmarks/img/overhead-light.svg | 363 + docs/benchmarks/img/packages-dark.svg | 79 + docs/benchmarks/img/packages-light.svg | 79 + docs/benchmarks/img/phases-dark.svg | 394 + docs/benchmarks/img/phases-light.svg | 394 + docs/benchmarks/img/whatif-dark.svg | 209 + docs/benchmarks/img/whatif-light.svg | 209 + docs/benchmarks/results/overhead.md | 26 + .../results/overhead/acmart-biblatex.json | 3108 ++++ docs/benchmarks/results/overhead/acmart.json | 2996 ++++ .../results/overhead/florilegium.json | 917 + docs/benchmarks/results/overhead/hott.json | 1491 ++ docs/benchmarks/results/overhead/large.json | 2695 +++ .../results/overhead/metropolis.json | 4200 +++++ docs/benchmarks/results/overhead/minimal.json | 238 + .../results/overhead/open-logic.json | 14959 ++++++++++++++++ docs/benchmarks/results/overhead/stacks.json | 483 + .../benchmarks/results/overhead/standard.json | 3521 ++++ docs/benchmarks/results/packages.md | 14 + docs/benchmarks/results/results.json | 4064 ++++- docs/benchmarks/results/results.md | 190 +- docs/benchmarks/results/whatif.md | 127 + .../results/whatif/acmart-biblatex.json | 223 + docs/benchmarks/results/whatif/acmart.json | 223 + .../results/whatif/florilegium.json | 111 + docs/benchmarks/results/whatif/hott.json | 111 + docs/benchmarks/results/whatif/large.json | 275 + .../benchmarks/results/whatif/metropolis.json | 297 + docs/benchmarks/results/whatif/minimal.json | 241 + .../benchmarks/results/whatif/open-logic.json | 259 + docs/benchmarks/results/whatif/stacks.json | 102 + docs/benchmarks/results/whatif/standard.json | 295 + docs/benchmarks/run.sh | 30 +- 42 files changed, 43866 insertions(+), 1137 deletions(-) create mode 100644 docs/benchmarks/charts.py create mode 100644 docs/benchmarks/corpus.toml create mode 100644 docs/benchmarks/fetch.py create mode 100644 docs/benchmarks/img/overhead-dark.svg create mode 100644 docs/benchmarks/img/overhead-light.svg create mode 100644 docs/benchmarks/img/packages-dark.svg create mode 100644 docs/benchmarks/img/packages-light.svg create mode 100644 docs/benchmarks/img/phases-dark.svg create mode 100644 docs/benchmarks/img/phases-light.svg create mode 100644 docs/benchmarks/img/whatif-dark.svg create mode 100644 docs/benchmarks/img/whatif-light.svg create mode 100644 docs/benchmarks/results/overhead.md create mode 100644 docs/benchmarks/results/overhead/acmart-biblatex.json create mode 100644 docs/benchmarks/results/overhead/acmart.json create mode 100644 docs/benchmarks/results/overhead/florilegium.json create mode 100644 docs/benchmarks/results/overhead/hott.json create mode 100644 docs/benchmarks/results/overhead/large.json create mode 100644 docs/benchmarks/results/overhead/metropolis.json create mode 100644 docs/benchmarks/results/overhead/minimal.json create mode 100644 docs/benchmarks/results/overhead/open-logic.json create mode 100644 docs/benchmarks/results/overhead/stacks.json create mode 100644 docs/benchmarks/results/overhead/standard.json create mode 100644 docs/benchmarks/results/packages.md create mode 100644 docs/benchmarks/results/whatif.md create mode 100644 docs/benchmarks/results/whatif/acmart-biblatex.json create mode 100644 docs/benchmarks/results/whatif/acmart.json create mode 100644 docs/benchmarks/results/whatif/florilegium.json create mode 100644 docs/benchmarks/results/whatif/hott.json create mode 100644 docs/benchmarks/results/whatif/large.json create mode 100644 docs/benchmarks/results/whatif/metropolis.json create mode 100644 docs/benchmarks/results/whatif/minimal.json create mode 100644 docs/benchmarks/results/whatif/open-logic.json create mode 100644 docs/benchmarks/results/whatif/stacks.json create mode 100644 docs/benchmarks/results/whatif/standard.json diff --git a/README.md b/README.md index 3bea1a6..3594767 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,8 @@ I had no way to see why. Commenting packages out one at a time is not insight. p is one instrumented build that shows where the time goes, which package loaded which, and what you can drop. -It has been run on my own corpus and on a few public documents (a thesis template, a beamer -deck, a 76-page arXiv paper; see [the benchmarks](docs/benchmarks/README.md)). Please report -what breaks! +It has been run on my own corpus and on public documents from an ACM paper to the 7,710-page +Stacks Project (see [the benchmarks](docs/benchmarks/README.md)). Please report what breaks! This project was made with the assistance of AI (Claude). @@ -63,6 +62,7 @@ prumo prune docs/ --engines pdflatex # corpus-wide keep list and tlmgr remove pl prumo prune docs/ --apply --yes # run the plan; every tlmgr command is logged prumo unused main.tex # which \usepackage lines can go: remove the line, rebuild, compare prumo unused obra.cls --corpus 'docs/**/*.tex' # a class judged against every document that uses it +prumo whatif main.tex --shell-escape # measured speed-ups: precompiled preamble, draft mode, externalized tikz pictures prumo time main.tex --env tblr,minted # which environments to time (default: tikzpicture, axis, tcolorbox, tblr, longtable, tabularx, minted, lstlisting, forest) prumo time output.prumo # records from a build prumo did not run (Overleaf: docs/overleaf.md) prumo doctor --check-log main.tex # files that bypassed the hooks; log vs records @@ -148,6 +148,54 @@ row. `prumo compare main.tex --runs 4` on the same paper: note: xelatex's first run took 4050 ms (font cache build); the reported run is warm. ``` +## What `whatif` measures + +`prumo whatif` rebuilds the document with one change at a time and times it against the +plain build. Here it is on the whatif fixture: + +``` +$ prumo whatif fixtures/whatif/main.tex --shell-escape +== prumo whatif: main (pdflatex, median of 5 runs after a warm-up, 1 page, -shell-escape) + experiment baseline variant saved bound one-time output + preamble 247 ms 90 ms 157 ms 64% 207 ms 319 ms identical + draft 247 ms 244 ms within noise - - n/a (no PDF) + externalize* 247 ms 245 ms within noise 3 ms 503 ms identical +``` + +The three experiments: + +- `preamble` precompiles everything before `\begin{document}` into a format with + mylatexformat and builds on that format. The one-time column is the format build. Only + pdfTeX can do it at all. XeTeX cannot write a format once the class has loaded its + OpenType font, and on LuaTeX either the format build or the build on it fails. Even on + pdfTeX it fails on preambles that open files or play with catcodes, as it did on three of + the six pdfLaTeX documents in the [benchmarks](docs/benchmarks/README.md). The row says + which step failed, with the error. +- `draft` builds without writing a PDF (`-draftmode`, or `-no-pdf` on XeTeX). What it saves + is what writing the PDF and including images cost. There is no PDF to compare. +- `externalize` is experimental. It turns on tikz's external library from the command line, + so each picture is built once into its own PDF and included from then on. It needs + `--shell-escape`, because tikz builds each picture with a shell command. The one-time + column is the first run, which builds every picture. The external library breaks + `remember picture`, overlays and references between pictures, so look at the PDF too. + +Every build runs once to warm up, then five times (`--runs`), one build at a time and without +the probe. The baseline and the variants take turns, so drift hits them all alike. The table +compares medians. A difference no larger than the spread, the slowest run minus the fastest, +is "within noise". The output column compares the page count, the words on each page +(`pdftotext`) and a coarse render of each page with the baseline's. When they differ, the +report says the saving is not a clean one. + +The bound column is a cross-check from one probed build: the preamble phase for `preamble`, +and the time inside `tikzpicture` environments for `externalize`. The fixture's one picture +takes 3 ms, so externalizing it saves nothing. The [benchmarks](docs/benchmarks/README.md) +run all three experiments on ten documents and three engines. Watch the one-time cost. +Externalizing Open Logic's pictures saves 4 s of a 31 s build, after a first run of 93 +minutes. + +Each experiment runs in its own copy of your directory, and the formats and picture files +stay there. Under the table, `whatif` says how to make each change yourself. + ## Running from Docker, nothing installed No TeX Live? No Rust? `docker/Dockerfile.user` builds an image with a full TeX Live, Poppler, @@ -327,6 +375,7 @@ know which of its macros, that is texprof's question. `--out` says), nothing next to your document. `prune --apply` is the one destructive command, it re-prints its plan first and needs `--yes`. - Fetch anything. It reads the local `texlive.tlpdb`; no network, ever. +- Make a speed-up for you. `whatif` measures a change on a copy and says how to make it. - Prove a package unused. "No observable effect" means this document, this build, these comparisons; a package can matter for input the document does not exercise, and the report says so every time. diff --git a/docs/10-verified-findings.md b/docs/10-verified-findings.md index bdbc3d9..149ac65 100644 --- a/docs/10-verified-findings.md +++ b/docs/10-verified-findings.md @@ -254,3 +254,42 @@ Items 1–18 hold, with these corrections and additions: inside page 1's brackets (`[1{.../pdftex.map}]`). On a one-line document the shipout phase is 45 ms of 132 ms on pdfLaTeX, all of it page 1; how much of that is the map was not isolated. +60. **A precompiled preamble works on pdfTeX only.** mylatexformat + (`pdftex -ini -jobname=main-fmt "&pdflatex" mylatexformat.ltx main.tex`, then + `pdflatex -fmt=main-fmt main.tex`) takes the whatif fixture from 244 ms to 91 ms on TeX + Live 2026 and from 137 ms to 58 ms on TeX Live 2021, with the same text. XeTeX stops with + `! Can't \dump a format with native fonts or font-mappings.` even without fontspec: the + class's `\normalsize` loads Latin Modern as an OpenType font for the TU encoding. On TeX + Live 2026, LuaHBTeX writes the format but exits 1 after a Lua error in lua-uni-algos + (`lua-uni-stage-tables.lua:141: attempt to perform arithmetic on a string value`), and a + build on that format dies with `(pdf backend): referenced object has wrong type`. On TeX + Live 2021 the dump succeeds and the build on it dies the same way. +61. **tikz's external library from the command line.** With + `\AddToHook{package/tikz/after}{\usetikzlibrary{external}\tikzexternalize}\input{main}` + on the command line, each picture build runs `\def\tikzexternalrealjob{main}\input{main}`, + which never sees the hook. It typesets the whole document as "the picture", and a full + letter page goes where the picture was: 1 page became 2 on the test document. That is + most likely the 2 → 3 pages in the brief, whose document was not run here. Passing `system call=` with the hook in the picture's + command line fixes it. The text and page count are then identical on all three engines, + on TeX Live 2026 and 2021. tikz's default system call is pdflatex unless the driver says + otherwise, so the system call also names the document's engine. Ten pictures of 150 lines + each went from 944 ms to 138 ms, against the probe's 828 ms in `tikzpicture`. +62. **An existence check is not a read.** siunitx runs `\RequirePackage{color}` at + `begindocument/before`. When tikz has already loaded xcolor, which claims `color.sty`, + the kernel opens `color.sty` and does not read it. pdfTeX's and XeTeX's `.fls` list it, + LuaTeX's does not, and neither the log nor the records show it. `doctor --check-log` + called it "read outside the LaTeX file hooks". With a log it now lists such files as + opened but never read. +63. **`pdftotext -layout` moves an externalized picture's text.** On the benchmark paper, + externalize was reported as changing the output: line 64 of the text differed. The pages + render the same (78 of 240,000 pixels differ at 50 dpi, antialiasing) and every page has + the same words. The picture is an included PDF now, and `-layout` places its text a column + or a line away. `whatif` compares the words on each page and a 24 dpi render instead + (the render check is `unused`'s), and then catches a real one: one page of the metropolis + deck on XeLaTeX and LuaLaTeX has the same words and 0.9% of its pixels changed. +64. **thmtools' `sibling=` fails on the 2026-11-01 kernel.** `\usepackage{amsthm,thmtools}`, + `\declaretheorem[numberwithin=section]{theorem}` and + `\declaretheorem[sibling=theorem]{proposition}` stop with `Command \c@proposition already + defined` (thmtools 0.76, TeX Live 2026). The same file builds on TeX Live 2021 (thmtools + 0.72, kernel 2021-11-15). Every kaobook example fails this way, so kaobook is not in the + benchmarks. diff --git a/docs/STATUS.md b/docs/STATUS.md index 170bfb5..7899de1 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -44,17 +44,26 @@ can check it. ### Benchmarks (`docs/benchmarks/`) -- `run.sh` runs `bench.py` in the full TeX Live 2026 image pinned to one P-core of the - 13900K: four documents (one line; a five-page paper with biber, pgfplots and siunitx; - florilegium's manual; a 3,991-page generated book), three engines, a cold run with fresh - font caches and five warm runs (three for the book), medians per phase. Results and - method in `docs/benchmarks/README.md`; the widest spread is 4%. -- On the one-line document LuaLaTeX spends 235 of 315 ms starting up, 22 ms on the preamble - and 54 ms writing the PDF. On the paper it is 236, 763 and 135 of 1,392 ms (pdfLaTeX 855 - ms in all). On the book LuaTeX's shipout is 2.7 ms a page against pdfTeX's 0.7. A cold - LuaLaTeX run is 4.4 s instead of 0.3 s. -- Scale: the book's 8,669 records are read in 3 ms, and `--html` writes 1.3 MB in 8 ms. - Plain output now lists the slowest `--top` pages instead of all 3,991. +- Ten documents now, three engines. Three live here (one line, the five-page paper, the + 3,991-page generated book). Seven are fetched at a pinned commit and never stored: the + ACM sigconf sample with BibTeX and with biblatex, the metropolis demo deck, florilegium's + manual, the HoTT book, Open Logic and the 7,710-page Stacks Project. `corpus.toml` has + each one's commit, license, engines and prepare step, `fetch.py` lays them out. kaobook + was left out, because thmtools' `sibling=` fails on the 2026-11-01 kernel, in five lines + too. +- `bench.py` has three modes: the phases (as before), `--overhead` (builds with and without + the probe in turns, plus per-package times from the probed records) and `--whatif`. + `charts.py` draws four SVG charts, light and dark, from the JSON. Color means phase in + all of them, the palette checked with the dataviz validator, the gray included after you + could not tell startup from body. +- The probe costs 7 ms on the one-line document and under 6% everywhere. On the paper, + LuaLaTeX's extra preamble time is biblatex (+151 ms) and fontspec's LuaTeX code (67 ms). +- On the corpus, the precompiled preamble works on two of the six pdfLaTeX + documents (paper 49%, deck 25%). Draft pays on XeLaTeX (13 to 38%). Externalize pays on + the deck and florilegium (15 to 32%), but its first run on a long book is a pass per + picture: 93 minutes on Open Logic for 4 s a build. +- The run had seven P-cores busy at once, and its phase numbers came out 0.3 to 5% slower + than the quieter run before it. The README says so. ### Public documents @@ -81,6 +90,36 @@ and a paragraph on texprof. texprof is not a pdfTeX: it is TeX with e-TeX and Pr DVI, and could not build a LaTeX format against the 2026-11-01 kernel. `docs/02-prior-art.md` is corrected; CLAUDE.md still says "pdfTeX only". +### whatif + +- `prumo whatif main.tex [--try preamble,draft,externalize] [--runs 5] [--shell-escape]`, + from the brief. Each experiment rebuilds its own copy of the directory with one change. + The timed builds run without the probe: a warm-up each, then the baseline and the variants + in turns. Medians are compared. A difference no larger than the larger spread (2 ms at + least, the polling resolution) is "within noise". The output is compared by page count, + the words on each page and a 24 dpi render (finding 63). `--json` has the same fields + under schema 1. +- The builds run one at a time, not on `unused`'s parallel pool, because parallel builds + slow each other down. The temp copy, the timeout and the text comparison come from `run` + and `unused`. +- `preamble` works on pdfTeX (the fixture goes from 244 to 91 ms). XeTeX and LuaTeX fail, + and the row gives the error (finding 60). `doctor` has a mylatexformat row and the TeX + image installs it. +- `externalize`: the brief's command line made every picture build typeset the whole + document (finding 61). The system call now carries the hook and names the engine. The text + is identical on all three engines, and ten pictures went from 944 to 138 ms. Without + `--shell-escape` it is refused when asked for and not run by default. The table and the + README mark it experimental. +- New fixture `whatif`, eight claims, all tested. `make_artifacts.sh whatif` regenerates one + fixture now. The fixture turned up finding 62: `doctor --check-log` now tells an + existence check from a read. +- My choices: `--runs` is at least 2. The comparison uses the last run's PDF. Logs go to + `//whatif//`. With `--shell-escape` the probed run is + `--build " -shell-escape"`, so `time` rebuilds afterwards. No `--latexmk` or + `--build`, since their wall time is several passes. +- `cargo test`, clippy and fmt pass, and so do the 19 TeX tests in the image. TeX Live 2021 + runs it too (on a copy of the fixture without siunitx, which that image lacks). + ### Only if cheap - `prumo time output.prumo` and `prumo graph output.prumo` read records from a build prumo @@ -89,9 +128,9 @@ is corrected; CLAUDE.md still says "pdfTeX only". - LuaLaTeX font definitions per page: the `luaotfload.patch_font` hypothesis is wrong for `\addfontfeature`, so nothing shipped (finding 57). -**Blocked:** nothing. **Needs me:** the v0.2.0 tag. **Open questions:** should the -LuaTeX font count go in after all, through `font.nextid()`? Should `compare` default to three -runs, so it reports a median? +**Blocked:** nothing. **Needs me:** review and merge the `whatif` PR, then the v0.2.0 tag. +**Open questions:** should the LuaTeX font count go in after all, through `font.nextid()`? +Should `compare` default to three runs, so it reports a median? ## Earlier milestones, one line each diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 4e4f6db..ea46cdd 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -1,49 +1,64 @@ # Benchmarks -Where a LaTeX run spends its time, phase by phase, for four documents on three engines. Every -number here comes from `prumo time --json`, and the script that made them is in this -directory, so you can run it on your machine and compare. +Where a LaTeX run spends its time, what prumo's probe costs, and what `prumo whatif`'s three +speed-ups save, on ten documents and three engines. Every number here comes from prumo's +own JSON, and the scripts that made them are in this directory, so you can run them on your +machine and compare. ## The documents -| | what | pages | -|---|---|---| -| `minimal` | `\documentclass{article}`, one word. The startup test. | 1 | -| `standard` | A paper: 11 pt article, fontenc and lmodern on pdfLaTeX, fontspec on XeLaTeX and LuaLaTeX, amsmath, mathtools, siunitx, booktabs, TikZ, pgfplots, microtype, hyperref, cleveref, biblatex with biber, 12 references, figures, a table, cross-references. | 5 | -| `florilegium` | The manual of [florilegium](https://github.com/kevinraymond/florilegium) at commit `833c4fc`. LuaLaTeX only, since the package is written in Lua. | 35 | -| `large` | A book generated by `gen_large.py`: 140 chapters, 2,800 sections of lipsum text, an equation and a cross-reference in each, a table every fifth section, a TikZ picture per chapter. | 3,991 | +| document (`--docs` name) | what | pages | engines | from | +|---|---|---:|---|---| +| minimal (`minimal`) | `\documentclass{article}`, one word. The startup test. | 1 | all three | here | +| paper (`standard`) | A paper: amsmath, siunitx, TikZ, pgfplots, microtype, hyperref, cleveref, biblatex with biber; fontspec on XeLaTeX and LuaLaTeX. | 5 | all three | here | +| ACM paper (`acmart`) | ACM's `sigconf` sample with BibTeX, a tagged PDF (`\DocumentMetadata`). | 7 | all three | [acmart](https://github.com/borisveytsman/acmart/tree/0537b243395b23ad85c5036929f4b1f28429b3a5) `0537b24`, LPPL 1.3c | +| ACM paper, biblatex (`acmart-biblatex`) | The same sample with biblatex and biber. | 8 | all three | the same | +| beamer deck (`metropolis`) | The metropolis beamer theme's demo deck. | 33 | all three | [mtheme](https://github.com/matze/mtheme/tree/2fa6084b9d34fec9d2d5470eb9a17d0bf712b6c8) `2fa6084`, CC BY-SA 4.0 | +| florilegium manual (`florilegium`) | The manual of florilegium, a Lua package. | 35 | LuaLaTeX | [florilegium](https://github.com/kevinraymond/florilegium/tree/833c4fc9ff776c51d0609fd6c96b4a2868806efa) `833c4fc`, MIT | +| HoTT book (`hott`) | The HoTT book. | 492 | pdfLaTeX | [HoTT/book](https://github.com/HoTT/book/tree/578b85cc8d586b1677ec4335148adeb443057d24) `578b85c`, CC BY-SA 3.0 | +| Open Logic (`open-logic`) | The Open Logic Project's complete text. | 1,013 | all three | [OpenLogic](https://github.com/OpenLogicProject/OpenLogic/tree/1e960beff9ed7835bf3e3f1335e21af3439cd107) `1e960be`, CC BY 4.0 | +| generated book (`large`) | A book `gen_large.py` writes: 140 chapters, 2,800 sections of lipsum, an equation and a cross-reference in each, a table every fifth section, a TikZ picture per chapter. | 3,991 | all three | here | +| Stacks Project (`stacks`) | The Stacks Project as one book. | 7,710 | pdfLaTeX | [stacks-project](https://github.com/stacks/stacks-project/tree/a04446e57ec1fbc252a871afcec7752fb2807b14) `a04446e`, GFDL 1.2 | + +Three of them live in this repository. The other seven are taken from their own repositories +at the commit above and are never stored here. Their licenses are their own, and share-alike +and GFDL terms do not mix with this repository's. `corpus.toml` lists every document with its +commit, its license, the engines it builds on and the step that prepares it (acmart builds +its class and samples from `.dtx` files, the Stacks Project writes its book file, the HoTT +book its version file). +`fetch.py` lays them out and keeps the clones in `~/.cache/prumo-bench`. -The first two are in `docs/`. The large one is generated at run time (`gen_large.py` is -deterministic), and the manual is taken from a florilegium clone with `git archive`. +The HoTT book builds on pdfLaTeX only (xypdf needs pdfTeX). The Stacks Project runs on +pdfLaTeX only, the engine its own build uses, because the other two would triple the longest +run. On XeLaTeX and LuaLaTeX, the ACM paper has one page fewer than on pdfLaTeX and Open Logic +two. + +kaobook was tried and left out. All of its examples fail on TeX Live 2026 with `Command +\c@proposition already defined`, and so does a five-line document with amsthm and thmtools' +`sibling=` key. thmtools 0.76 and the 2026-11-01 kernel do not get along. ## How they were run -- **Machine:** Intel Core i9-13900K, Debian 13, Linux 6.12. The container is pinned to one - performance core (`docker run --cpuset-cpus 4`); TeX is single-threaded. +- **Machine:** Intel Core i9-13900K, Debian 13, Linux 6.12. Each run is a container pinned to + one performance core; TeX is single-threaded. This set ran on seven of the eight + performance cores at once: the phases on one, the `whatif` and overhead runs on the + others. The phase numbers came out 0.3 to 5% slower than an earlier run on a quieter + machine. The `whatif` and overhead comparisons are made inside one container, build + against build in turns, so the other cores do not tilt them. - **TeX:** TeX Live 2026 from `texlive/texlive:latest` plus Poppler (the `prumo-dev:tl2026-full` image `make verify-prune` builds): pdfTeX 1.40.29, XeTeX 0.999998, LuaHBTeX 1.24.0, LaTeX 2026-11-01. -- **Each timed run** is `prumo time --rebuild --json` on a copy of the document whose `.aux`, - `.toc` and `.bbl` came from a latexmk build just before. So a run is one engine pass with - the cross-references and the bibliography in place, the way an editor rebuilds. prumo - times that process from spawn to exit. +- **Each timed run** is one engine pass on a copy of the document whose `.aux`, `.toc` and + `.bbl` came from a latexmk build just before, the way an editor rebuilds. The phases come + from `prumo time --rebuild --json`, which times the engine process from spawn to exit. - **Cold** is the first run with a fresh `HOME`: no luaotfload font name database and no fontconfig cache, as on a new machine or after a TeX Live update. The operating system's file cache is not dropped (that needs root on the host), so cold here means cold font caches, not a cold disk. -- **Warm** is five runs after that (three for the large document). Each phase is the median - of its own values over those runs, so the phase medians need not add up to the total's - median exactly. The spread row is the fastest and slowest warm run. - -To run it: - -``` -docs/benchmarks/run.sh # everything, about 30 minutes -docs/benchmarks/run.sh --docs minimal,standard # just these -``` - -`BENCH_IMAGE`, `BENCH_CPUS` and `FLORILEGIUM` (a clone's path) change the defaults. It writes -`results/results.json` (every run) and `results/results.md` (these tables). +- **Warm** is five runs after that, three for the generated book and Open Logic, two for the + Stacks Project. + Each phase is the median of its own values over those runs, so the phase medians need not + add up to the total's median exactly. ## What the phases are @@ -61,97 +76,360 @@ The timer is wall clock on all three engines, with its zero at the start of the | end-document | `\end{document}` starts | the last record, less shipouts | | finalization | the last record | process exit: closing the PDF, writing fonts, xdvipdfmx on XeTeX | -## Results +The charts fold them into four, one color each in every chart: startup (which no change to +the document touches), the preamble, the body (begin-document, typesetting and +end-document) and writing the PDF (shipout and finalization). Each `whatif` experiment +shrinks one of the last three, and its bar has that phase's color. + +## Where the time goes -Medians of the warm runs; the spread row is the fastest and slowest of them. The load average -was 2.61 when the run started, from containers on other cores. The widest spread is 4.0% of -its median (XeLaTeX on the paper); on the two long documents it is under 0.5%. + + + One panel per document, one bar per engine, cut into startup, preamble, body and writing the PDF. Short documents are mostly startup and preamble, long ones almost all body; LuaLaTeX's bar is the longest on every document. + + +
+The numbers, phase by phase ### minimal | phase (ms) | pdflatex | xelatex | lualatex | |---|---:|---:|---:| -| startup | 41.9 | 82.8 | 235.2 | -| preamble | 7.8 | 45.9 | 21.9 | -| begin-document | 1.6 | 1.9 | 2.0 | +| startup | 42.3 | 84.0 | 242.8 | +| preamble | 8.2 | 46.0 | 22.9 | +| begin-document | 1.8 | 2.0 | 2.2 | | typesetting | 0.1 | 0.1 | 0.1 | -| shipout | 45.1 | 0.2 | 35.4 | -| end-document | 0.6 | 0.5 | 0.6 | -| finalization | 38.0 | 135.5 | 19.0 | -| **total (wall)** | **134.9** | **267.4** | **314.9** | -| spread (min–max) | 134–136 | 265–269 | 311–316 | -| cold run (wall) | 134.9 | 3,188.5 | 4,416.3 | +| shipout | 45.3 | 0.2 | 35.9 | +| end-document | 0.7 | 0.6 | 0.7 | +| finalization | 41.1 | 146.8 | 19.1 | +| **total (wall)** | **140.0** | **279.1** | **324.7** | +| spread (min–max) | 137–142 | 275–285 | 322–329 | +| cold run (wall) | 144.2 | 3,273.8 | 4,629.5 | | pages | 1 | 1 | 1 | -### standard +### paper | phase (ms) | pdflatex | xelatex | lualatex | |---|---:|---:|---:| -| startup | 42.3 | 81.7 | 236.1 | -| preamble | 501.9 | 715.2 | 762.8 | -| begin-document | 49.0 | 46.8 | 52.3 | -| typesetting | 148.3 | 266.2 | 202.8 | -| shipout | 59.6 | 19.1 | 63.5 | -| end-document | 2.3 | 3.5 | 3.3 | -| finalization | 51.6 | 56.4 | 71.1 | -| **total (wall)** | **854.7** | **1,191.0** | **1,391.5** | -| spread (min–max) | 852–864 | 1,162–1,209 | 1,374–1,418 | -| cold run (wall) | 859.2 | 4,122.3 | 5,744.9 | +| startup | 43.0 | 83.8 | 247.9 | +| preamble | 499.8 | 719.9 | 780.5 | +| begin-document | 49.7 | 48.2 | 54.4 | +| typesetting | 150.6 | 270.5 | 211.0 | +| shipout | 60.5 | 31.1 | 64.3 | +| end-document | 2.4 | 3.8 | 3.5 | +| finalization | 54.3 | 58.1 | 71.3 | +| **total (wall)** | **859.6** | **1,219.4** | **1,431.5** | +| spread (min–max) | 855–871 | 1,205–1,232 | 1,388–1,439 | +| cold run (wall) | 863.6 | 4,193.7 | 5,977.3 | | pages | 5 | 5 | 5 | -### florilegium +### generated book (3 warm runs) + +| phase (ms) | pdflatex | xelatex | lualatex | +|---|---:|---:|---:| +| startup | 44.3 | 83.4 | 242.5 | +| preamble | 192.3 | 283.8 | 273.7 | +| begin-document | 58.6 | 49.0 | 77.3 | +| typesetting | 14,322.1 | 15,935.5 | 27,305.0 | +| shipout | 2,872.5 | 3,831.2 | 10,949.3 | +| end-document | 25.0 | 22.4 | 24.9 | +| finalization | 119.6 | 230.7 | 157.7 | +| **total (wall)** | **17,643.5** | **20,433.3** | **39,037.4** | +| spread (min–max) | 17,078–17,716 | 20,412–20,688 | 39,004–39,902 | +| cold run (wall) | 17,405.7 | 23,292.1 | 44,183.2 | +| pages | 3991 | 3991 | 3991 | + +pdflatex: 8,669 records, 164 KiB. `prumo time --json` reads the cached run in 3 ms, and `--html` writes 1,279 KiB in 8 ms. + +xelatex: 8,661 records, 165 KiB. `prumo time --json` reads the cached run in 3 ms, and `--html` writes 1,277 KiB in 8 ms. + +lualatex: 8,668 records, 169 KiB. `prumo time --json` reads the cached run in 3 ms, and `--html` writes 1,280 KiB in 8 ms. + +### florilegium manual | phase (ms) | lualatex | |---|---:| -| startup | 238.5 | -| preamble | 274.2 | -| begin-document | 12.1 | -| typesetting | 508.7 | -| shipout | 470.9 | -| end-document | 2.1 | -| finalization | 91.3 | -| **total (wall)** | **1,597.4** | -| spread (min–max) | 1,595–1,602 | -| cold run (wall) | 6,015.2 | +| startup | 245.8 | +| preamble | 294.3 | +| begin-document | 12.7 | +| typesetting | 519.1 | +| shipout | 473.5 | +| end-document | 2.2 | +| finalization | 92.4 | +| **total (wall)** | **1,633.2** | +| spread (min–max) | 1,615–1,663 | +| cold run (wall) | 6,163.4 | | pages | 35 | -### large +### ACM paper + +| phase (ms) | pdflatex | xelatex | lualatex | +|---|---:|---:|---:| +| startup | 41.6 | 82.7 | 242.2 | +| preamble | 749.5 | 1,145.6 | 1,515.8 | +| begin-document | 58.2 | 56.9 | 96.3 | +| typesetting | 995.5 | 1,136.0 | 1,300.8 | +| shipout | 274.0 | 454.7 | 283.9 | +| end-document | 258.3 | 7.5 | 146.7 | +| finalization | 71.4 | 70.0 | 146.2 | +| **total (wall)** | **2,451.0** | **2,951.1** | **3,726.2** | +| spread (min–max) | 2,420–2,518 | 2,943–2,976 | 3,713–3,837 | +| cold run (wall) | 2,469.5 | 5,948.5 | 10,267.5 | +| pages | 7 | 7 | 6 | + +### ACM paper, biblatex + +| phase (ms) | pdflatex | xelatex | lualatex | +|---|---:|---:|---:| +| startup | 42.2 | 82.8 | 238.2 | +| preamble | 936.8 | 1,487.6 | 1,845.2 | +| begin-document | 120.4 | 128.8 | 166.0 | +| typesetting | 1,454.2 | 1,650.9 | 1,765.3 | +| shipout | 302.5 | 559.8 | 299.5 | +| end-document | 289.7 | 16.1 | 164.3 | +| finalization | 72.4 | 67.5 | 142.6 | +| **total (wall)** | **3,217.8** | **3,992.9** | **4,638.7** | +| spread (min–max) | 3,215–3,253 | 3,900–4,002 | 4,615–4,803 | +| cold run (wall) | 3,215.5 | 6,905.8 | 11,349.9 | +| pages | 8 | 8 | 7 | + +### beamer deck + +| phase (ms) | pdflatex | xelatex | lualatex | +|---|---:|---:|---:| +| startup | 42.1 | 82.4 | 240.5 | +| preamble | 355.2 | 499.8 | 565.3 | +| begin-document | 29.7 | 27.8 | 37.5 | +| typesetting | 592.3 | 729.1 | 765.6 | +| shipout | 55.4 | 15.7 | 59.3 | +| end-document | 0.8 | 0.7 | 0.7 | +| finalization | 47.1 | 60.5 | 87.8 | +| **total (wall)** | **1,122.5** | **1,415.5** | **1,753.4** | +| spread (min–max) | 1,120–1,126 | 1,413–1,420 | 1,744–1,764 | +| cold run (wall) | 1,123.7 | 4,317.2 | 10,903.7 | +| pages | 33 | 33 | 33 | + +### HoTT book + +| phase (ms) | pdflatex | +|---|---:| +| startup | 42.9 | +| preamble | 284.1 | +| begin-document | 29.8 | +| typesetting | 2,355.2 | +| shipout | 1,304.0 | +| end-document | 32.4 | +| finalization | 61.4 | +| **total (wall)** | **4,110.1** | +| spread (min–max) | 4,103–4,112 | +| cold run (wall) | 4,112.4 | +| pages | 492 | + +### Open Logic (3 warm runs) | phase (ms) | pdflatex | xelatex | lualatex | |---|---:|---:|---:| -| startup | 42.3 | 82.5 | 236.3 | -| preamble | 183.6 | 279.7 | 276.7 | -| begin-document | 56.7 | 47.0 | 76.5 | -| typesetting | 13,594.2 | 15,694.2 | 27,220.9 | -| shipout | 2,781.5 | 3,778.4 | 10,933.5 | -| end-document | 23.0 | 21.4 | 24.7 | -| finalization | 111.1 | 227.4 | 157.7 | -| **total (wall)** | **16,793.5** | **20,136.1** | **38,914.9** | -| spread (min–max) | 16,757–16,801 | 20,130–20,142 | 38,899–38,944 | -| cold run (wall) | 16,395.2 | 23,068.1 | 43,350.8 | -| pages | 3,991 | 3,991 | 3,991 | +| startup | 42.9 | 83.4 | 241.7 | +| preamble | 479.2 | 512.0 | 534.7 | +| begin-document | 52.2 | 41.7 | 59.0 | +| typesetting | 29,050.9 | 29,592.5 | 34,840.7 | +| shipout | 781.3 | 1,028.1 | 3,577.1 | +| end-document | 39.5 | 37.7 | 42.0 | +| finalization | 82.7 | 159.2 | 162.4 | +| **total (wall)** | **30,530.3** | **31,454.3** | **39,431.6** | +| spread (min–max) | 30,179–30,672 | 31,450–31,493 | 39,057–39,920 | +| cold run (wall) | 30,466.3 | 34,395.1 | 43,067.6 | +| pages | 1013 | 1011 | 1011 | + +### Stacks Project (2 warm runs) + +| phase (ms) | pdflatex | +|---|---:| +| startup | 41.8 | +| preamble | 111.2 | +| begin-document | 180.1 | +| typesetting | 38,218.8 | +| shipout | 5,330.1 | +| end-document | 43.3 | +| finalization | 1,057.0 | +| **total (wall)** | **44,982.3** | +| spread (min–max) | 44,961–45,003 | +| cold run (wall) | 45,053.1 | +| pages | 7710 | +
+ +## What the probe costs + + + + For each document and engine, every build without prumo and every build with it, as a difference from the median build without. + + +A build with prumo pays for the probe's hooks and records, and for `-recorder`'s list of +files. Here a build without prumo and the same build with it ran in turns in one container, +five rounds after a warm-up of each, both timed from spawn to exit. The one-line document pays +7 ms on every engine. The papers and the deck pay 20 to 80 ms, under 6%. The books pay 1.5% or +less, except Open Logic on LuaLaTeX at 3.4%, where the builds without prumo alone spread over +1.2 s. The probe writes 18 records on the one-line document and 15,669 on the Stacks Project. +The phase numbers above are of builds with the probe, so they carry this cost. -pdflatex: 8,669 records (164 KiB); `prumo time --json` on the cached run 3 ms; `--html` 8 ms, 1,279 KiB. +| document | engine | records | plain (ms) | with prumo (ms) | cost | +|---|---|---:|---:|---:|---:| +| minimal | pdflatex | 18 | 133.1 ± 5.1 | 140.3 ± 7.9 | +7.2 ms (+5.4%) | +| minimal | xelatex | 18 | 267.4 ± 13.1 | 275.2 ± 13.9 | +7.8 ms (+2.9%) | +| minimal | lualatex | 18 | 316.2 ± 5.1 | 323.6 ± 8.3 | +7.3 ms (+2.3%) | +| paper | pdflatex | 555 | 838.8 ± 15.4 | 887.2 ± 8.7 | +48.4 ms (+5.8%) | +| paper | xelatex | 547 | 1,144.9 ± 53.7 | 1,195.1 ± 36.4 | +50.2 ms (+4.4%) | +| paper | lualatex | 556 | 1,368.4 ± 53.0 | 1,419.4 ± 44.9 | +50.9 ms (+3.7%) | +| generated book | pdflatex | 8,669 | 16,631.1 ± 261.1 | 16,767.1 ± 221.0 | +135.9 ms (+0.8%) | +| generated book | xelatex | 8,661 | 19,926.3 ± 137.5 | 20,046.6 ± 113.2 | +120.4 ms (+0.6%) | +| generated book | lualatex | 8,668 | 38,694.7 ± 2,050.2 | 38,702.6 ± 2,231.5 | +7.9 ms (+0.0%) | +| florilegium manual | lualatex | 502 | 1,621.4 ± 27.6 | 1,648.2 ± 25.4 | +26.8 ms (+1.7%) | +| ACM paper | pdflatex | 585 | 2,524.2 ± 94.6 | 2,571.0 ± 284.4 | +46.8 ms (+1.9%) | +| ACM paper | xelatex | 562 | 3,017.8 ± 64.3 | 3,083.8 ± 53.1 | +66.1 ms (+2.2%) | +| ACM paper | lualatex | 589 | 3,744.5 ± 108.8 | 3,766.3 ± 126.7 | +21.7 ms (+0.6%) | +| ACM paper, biblatex | pdflatex | 614 | 3,299.9 ± 166.1 | 3,351.9 ± 179.4 | +52.0 ms (+1.6%) | +| ACM paper, biblatex | xelatex | 586 | 3,964.5 ± 49.3 | 4,009.5 ± 62.9 | +45.1 ms (+1.1%) | +| ACM paper, biblatex | lualatex | 613 | 4,604.1 ± 149.0 | 4,682.9 ± 64.5 | +78.8 ms (+1.7%) | +| beamer deck | pdflatex | 795 | 1,098.5 ± 41.3 | 1,144.2 ± 36.6 | +45.7 ms (+4.2%) | +| beamer deck | xelatex | 803 | 1,428.2 ± 24.0 | 1,479.6 ± 38.0 | +51.4 ms (+3.6%) | +| beamer deck | lualatex | 812 | 1,739.5 ± 32.6 | 1,794.3 ± 12.7 | +54.9 ms (+3.2%) | +| HoTT book | pdflatex | 1,678 | 4,121.9 ± 41.8 | 4,185.2 ± 49.0 | +63.3 ms (+1.5%) | +| Open Logic | pdflatex | 10,756 | 29,502.2 ± 813.5 | 29,865.4 ± 511.2 | +363.2 ms (+1.2%) | +| Open Logic | xelatex | 10,737 | 31,370.5 ± 2,661.3 | 31,504.3 ± 910.8 | +133.8 ms (+0.4%) | +| Open Logic | lualatex | 10,746 | 38,005.7 ± 1,223.5 | 39,313.4 ± 1,831.3 | +1,307.7 ms (+3.4%) | +| Stacks Project | pdflatex | 15,669 | 44,712.6 ± 772.3 | 45,019.3 ± 518.8 | +306.8 ms (+0.7%) | -xelatex: 8,661 records (165 KiB); `prumo time --json` on the cached run 3 ms; `--html` 7 ms, 1,277 KiB. +## Package by package: pdfLaTeX against LuaLaTeX -lualatex: 8,668 records (169 KiB); `prumo time --json` on the cached run 3 ms; `--html` 7 ms, 1,279 KiB. + + + The paper's slowest packages on pdfLaTeX and LuaLaTeX. biblatex takes about 150 ms longer on LuaLaTeX and fontspec's LuaTeX code adds about 70 ms; the other packages take the same time on both. + + +On the paper, LuaLaTeX's preamble takes 780 ms and pdfLaTeX's 500. Package by package, most +of that difference is two rows. biblatex takes 289 ms on LuaLaTeX and 138 on pdfLaTeX, and +fontspec's LuaTeX code adds 67 ms that pdfLaTeX never loads. Every other package takes the +same time on both, within a few milliseconds. Each row is the package's own time, without +the packages it loads. biblatex's row includes its style and language +files, which it reads without the file hooks. The times are the medians of the five probed +builds in the overhead set. prumo stops at the package, so why biblatex is slower on LuaTeX +is a question for a macro profiler. + +| package | pdfLaTeX (ms) | LuaLaTeX (ms) | difference | +|---|---:|---:|---:| +| biblatex | 138.4 | 289.1 | +150.7 | +| fontspec-luatex | – | 67.1 | | +| pgfplots | 74.5 | 82.7 | +8.2 | +| siunitx | 33.6 | 35.4 | +1.9 | +| hyperref | 15.2 | 15.8 | +0.6 | +| xcolor | 8.6 | 9.0 | +0.4 | +| cleveref | 9.7 | 9.9 | +0.3 | +| translations | 8.3 | 8.6 | +0.3 | +| amsmath | 4.5 | 4.8 | +0.3 | +| csquotes | 5.7 | 5.8 | +0.1 | +| nameref | 4.8 | 4.9 | +0.1 | +| lipsum | 12.4 | 10.2 | -2.2 | + +## What `whatif` saves + + + + For each document and engine, the share of the build each whatif experiment saved, with the probe's bound and the one-time cost. + + +Each row is `prumo whatif --shell-escape` after the latexmk build, with the same number of +runs as the phases. + +- **The precompiled preamble** saves 49% on the paper and 25% on the beamer deck, both on + pdfLaTeX. Everywhere else it fails or saves nothing. XeTeX and LuaTeX cannot do it at all, + and on pdfTeX mylatexformat could not handle the ACM class, the HoTT book or Open Logic. On + the long books the preamble is under 2% of the build, so there was little to save. +- **Draft mode** saves what writing the PDF costs. That is the most on XeLaTeX, where it skips + xdvipdfmx: 38% on the one-line document, 13% on the paper, 16% on the generated book. On + pdfLaTeX and LuaLaTeX it is 1 to 7% or within noise, except on the HoTT book and the + florilegium manual, where writing the PDF is a third of the build and draft mode saves a + quarter. +- **Externalized pictures** save 15 to 32% on the beamer deck and the florilegium manual, + for a first run of 3 to 13 s. On the long books that first run makes a pass over the whole + book for every picture. It took 31 to 63 minutes on the generated book and 93 to 119 on + Open Logic, for savings of 1 to 13%. Open Logic saves about 4 s a build after that on + pdfLaTeX and LuaLaTeX, so it pays back after 1,400 to 1,800 builds. On XeLaTeX its runs + spread too much to tell. + On XeLaTeX and LuaLaTeX one page of the beamer deck renders a little differently (0.9% of its + pixels, the same words), so those rows say the output changed. + +| document | engine | pages | baseline | preamble | draft | externalize | +|---|---|---:|---:|---|---|---| +| minimal | pdflatex | 1 | 134 ms | noise | noise | – | +| minimal | xelatex | 1 | 271 ms | fails | 104 ms (38%) | – | +| minimal | lualatex | 1 | 317 ms | fails | noise | – | +| paper | pdflatex | 5 | 820 ms | 405 ms (49%) | 12 ms (2%) | 56 ms (7%) | +| paper | xelatex | 5 | 1,177 ms | fails | 150 ms (13%) | 59 ms (5%) | +| paper | lualatex | 5 | 1,333 ms | fails | 52 ms (4%) | 45 ms (3%) | +| generated book | pdflatex | 3,991 | 16,566 ms | noise | 1,112 ms (7%) | 745 ms (4%) | +| generated book | xelatex | 3,991 | 19,914 ms | fails | 3,156 ms (16%) | 286 ms (1%) | +| generated book | lualatex | 3,991 | 38,925 ms | fails | 2,495 ms (6%) | 672 ms (2%) | +| florilegium manual | lualatex | 35 | 1,632 ms | fails | 417 ms (26%) | 518 ms (32%) | +| ACM paper | pdflatex | 7 | 2,461 ms | fails | noise | – | +| ACM paper | xelatex | 7 | 3,012 ms | fails | 186 ms (6%) | – | +| ACM paper | lualatex | 6 | 3,846 ms | fails | noise | – | +| ACM paper, biblatex | pdflatex | 8 | 3,315 ms | fails | noise | – | +| ACM paper, biblatex | xelatex | 8 | 3,956 ms | fails | 183 ms (5%) | – | +| ACM paper, biblatex | lualatex | 7 | 4,641 ms | fails | noise | – | +| beamer deck | pdflatex | 33 | 1,098 ms | 278 ms (25%) | noise | 218 ms (20%) | +| beamer deck | xelatex | 33 | 1,398 ms | fails | 158 ms (11%) | 219 ms (16%), output changed | +| beamer deck | lualatex | 33 | 1,737 ms | fails | 70 ms (4%) | 264 ms (15%), output changed | +| HoTT book | pdflatex | 492 | 4,147 ms | fails | 1,043 ms (25%) | 171 ms (4%) | +| Open Logic | pdflatex | 1,013 | 31,190 ms | fails | 261 ms (1%) | 3,936 ms (13%) | +| Open Logic | xelatex | 1,011 | 32,550 ms | fails | noise | noise | +| Open Logic | lualatex | 1,011 | 41,563 ms | fails | 545 ms (1%) | 4,069 ms (10%) | +| Stacks Project | pdflatex | 7,710 | 45,605 ms | noise | 3,374 ms (7%) | – | + +- beamer deck, xelatex, externalize: output changed, page 19 looks different (96 of 11011 pixels at 24 dpi; same words) +- beamer deck, lualatex, externalize: output changed, page 19 looks different (96 of 11011 pixels at 24 dpi; same words) + +Every row in full, with the spreads, the bounds and the one-time costs, is in +[results/whatif.md](results/whatif.md). ## What they say -- **Where LuaLaTeX's time goes.** On the one-line document, LuaLaTeX spends - 235 ms of its 315 ms starting up, 22 ms on the preamble and 54 ms writing the PDF - (shipout plus finalization). On the five-page paper it is 236, 763 and 135 ms of 1,392 ms. -- **Startup is the engine's fixed cost.** pdfLaTeX starts in 42 ms, XeLaTeX in 83, LuaLaTeX - in 235, the same on every document here. It is most of a short document's time on +- **Startup is the engine's fixed cost.** pdfLaTeX starts in 42 ms, XeLaTeX in 83, LuaLaTeX in + 242, the same on every document here. It is three quarters of the one-line document on LuaLaTeX. -- **The paper's preamble is its biggest phase** on every engine: 502 of 855 ms on pdfLaTeX, - 763 of 1,392 on LuaLaTeX. The sample in the top-level README has biblatex at 39% of LuaLaTeX's. -- **Writing the PDF** (shipout plus finalization) is 111 ms on pdfLaTeX, 76 on XeLaTeX and - 135 on LuaLaTeX for the paper. On the 3,991-page book it is 2.9 s, 4.0 s and - 11.1 s: LuaTeX's shipout costs 2.7 ms a page against pdfTeX's 0.7. -- **Cold font caches cost seconds once.** The first LuaLaTeX run builds luaotfload's font - name database: 4.4 s instead of 0.31 s on the one-line document. XeLaTeX's first run - builds the fontconfig cache: 3.2 s instead of 0.27 s. pdfLaTeX has no cache to build. -- **Scale.** The book's 8,669 records are 164 KiB. prumo reads a cached run in - 3 ms and writes its 1,279 KiB HTML page in 8 ms. +- **Short documents spend their time in the preamble, long ones in the body.** The paper's + preamble is 500 of 860 ms on pdfLaTeX, the ACM paper's 1.5 of 3.7 s on LuaLaTeX. Open + Logic is 95% body, the Stacks Project 85%. +- **LuaLaTeX is the slowest engine on every document here,** 1.3 to 2.3 times pdfLaTeX's time. + On the generated book it takes 13 s longer in the body and 8 s longer writing the PDF: its + shipout is 2.7 ms a page against pdfTeX's 0.7. +- **XeLaTeX is close to pdfLaTeX on long documents** (Open Logic: 31.5 s against 30.5) and 1.2 + to 2 times on short ones, where its startup and xdvipdfmx show. +- **Cold font caches cost seconds once.** The first LuaLaTeX run of the beamer deck takes + 10.9 s instead of 1.75, the ACM paper 10.3 s instead of 3.7, while luaotfload builds its font + name database. XeLaTeX's first run is about 3 s longer on every document, for fontconfig. + pdfLaTeX has no cache to build. +- **The probe costs 7 ms on the one-line document and under 6% everywhere here.** +- **Scale.** The Stacks Project's 7,710 pages build in 45 s on pdfLaTeX and leave 15,669 + records. The generated book's 8,669 records are 164 KiB; prumo reads a cached run of it in + 3 ms and writes its 1,280 KiB HTML page in 8 ms. + +## Running it + +``` +docs/benchmarks/run.sh # the phases, every document +docs/benchmarks/run.sh --docs minimal,standard # just these +docs/benchmarks/run.sh --overhead # builds with and without prumo, in turns +docs/benchmarks/run.sh --whatif --docs hott # prumo whatif --shell-escape +docs/benchmarks/run.sh --report # the markdown again, from the JSON +python3 docs/benchmarks/charts.py # the charts, from the JSON +``` + +`BENCH_IMAGE`, `BENCH_CPUS` and `BENCH_REPOS` change the image, the core and where the clones +are kept. The first run fetches the seven documents that live elsewhere; after that it +needs no network. The results go to `results/`: `results.json`, `overhead/` and `whatif/` +hold every run, and the `.md` files beside them are the tables above. + +Some runs are long. The phases of every document take about an hour on one core. +`--whatif --docs open-logic` takes several hours, most of it in externalize's first run, +which builds every picture with a pass over the whole 1,013-page book. diff --git a/docs/benchmarks/bench.py b/docs/benchmarks/bench.py index f6183a6..f0602dc 100755 --- a/docs/benchmarks/bench.py +++ b/docs/benchmarks/bench.py @@ -1,22 +1,35 @@ #!/usr/bin/env python3 -"""Benchmarks: prumo's phase table for four documents on three engines. +"""Benchmarks: prumo's phase table, and `prumo whatif`, for the documents in corpus.toml. Runs inside a TeX Live image with prumo on PATH and the documents under /bench/docs (run.sh -sets that up). For each document and engine: +lays them out with fetch.py). For each document and engine: - 1. Copy the document to a scratch directory and build it there with latexmk, so the .aux, - .toc and .bbl exist. prumo copies the directory it is given, so every timed run is a - rerun with those files present, the way an editor rebuilds. + 1. Copy the document to a scratch directory, run its `prepare` commands, and build it + there with latexmk, so the .aux, .toc and .bbl exist. prumo copies the directory it is + given, so every timed run is a rerun with those files present, the way an editor + rebuilds. 2. One cold run: a fresh HOME, so no luaotfload font name database and no fontconfig cache. The operating system's file cache is not dropped (that needs root on the host). - 3. --runs warm runs with that HOME. + 3. The document's `runs` warm runs (--runs by default) with that HOME. Every timed run is `prumo time --rebuild --json`. The phases come from prumo, the wall time is prumo's own measure of the engine process. Writes results.json (every run, plus the medians) and results.md (the tables) to --out. -Usage: bench.py [--runs 5] [--docs minimal,standard,florilegium,large] - [--engines pdflatex,xelatex,lualatex] [--out results] +With --overhead, step 1 is followed by --runs rounds of two builds in turns, after a +warm-up of each: the plain engine command, and the same with what prumo adds (-recorder, +-jobname and the probe, from /probe). Both are timed here the same way, spawn to exit, so the +difference is the probe's cost. Each probed run's records are read with `prumo time --json +.prumo` for the per-package times. Results go to overhead/.json. + +With --whatif, step 1 is followed by `prumo whatif --json --shell-escape` with the same +number of runs, in the HOME latexmk warmed. Each document's reports go to +whatif/.json under --out, so several containers can run at once on different +documents, and whatif.md is written from every file there. --report rewrites the two +markdown files from the JSON without building anything. + +Usage: bench.py [--runs 5] [--docs minimal,standard,...] [--engines pdflatex,xelatex,lualatex] + [--whatif | --overhead | --report] [--out results] """ import argparse import json @@ -26,21 +39,23 @@ import statistics import subprocess import time +import tomllib from pathlib import Path -DOCS = { - # name: (directory under /bench/docs, main file, engines it supports, warm runs or None) - "minimal": ("minimal", "main.tex", None, None), - "standard": ("standard", "main.tex", None, None), - "florilegium": ("florilegium", "florilegium-doc.tex", ["lualatex"], None), - "large": ("large", "main.tex", None, 3), -} +DOCS = tomllib.loads((Path(__file__).resolve().parent / "corpus.toml").read_text()) +ENGINES = ["pdflatex", "xelatex", "lualatex"] + + +def title(doc): + """What the tables call a document: its title in corpus.toml.""" + return DOCS[doc].get("title", doc) PHASES = ["startup", "preamble", "begin_document", "typesetting", "shipout", "end_document", "finalization"] LATEXMK_FLAG = {"pdflatex": "-pdf", "xelatex": "-xelatex", "lualatex": "-lualatex"} def sh(cmd, **kw): - return subprocess.run(cmd, capture_output=True, text=True, **kw) + # TeX output is not always UTF-8 (a log quotes the bytes a file had). + return subprocess.run(cmd, capture_output=True, text=True, errors="replace", **kw) def version_line(cmd): @@ -63,21 +78,33 @@ def machine(): "xetex": version_line(["xetex", "--version"]), "luahbtex": version_line(["luahbtex", "--version"]), "prumo": version_line(["prumo", "--version"]), - "florilegium_commit": os.environ.get("FLORILEGIUM_COMMIT", "?"), + "commits": {n: d["commit"] for n, d in DOCS.items() if "commit" in d}, } +def doc_dir(doc, scratch): + """The document's own directory inside its scratch copy.""" + return scratch / DOCS[doc].get("dir", ".") + + def prepare(doc, engine, scratch): - """Copy the document and build it once with latexmk so the aux files exist.""" - src = Path("/bench/docs") / DOCS[doc][0] + """Copy the document, run its prepare commands, build it once with latexmk so the aux + files exist. Returns (ok, log tail, HOME used).""" + src = Path("/bench/docs") / doc if scratch.exists(): shutil.rmtree(scratch) shutil.copytree(src, scratch) - env = dict(os.environ, HOME=str(scratch.parent / f"home-prep-{doc}-{engine}")) - main = DOCS[doc][1] - r = sh(["latexmk", LATEXMK_FLAG[engine], "-interaction=nonstopmode", main], cwd=scratch, env=env) - pdf = scratch / main.replace(".tex", ".pdf") - return r.returncode == 0 and pdf.is_file(), r.stdout[-2000:] + r.stderr[-2000:] + home = scratch.parent / f"home-prep-{doc}-{engine}" + env = dict(os.environ, HOME=str(home)) + if DOCS[doc].get("prepare"): + r = sh(["bash", "-ec", DOCS[doc]["prepare"]], cwd=scratch, env=env) + if r.returncode != 0: + return False, "prepare: " + r.stdout[-1000:] + r.stderr[-1000:], home + main = DOCS[doc]["main"] + d = doc_dir(doc, scratch) + r = sh(["latexmk", LATEXMK_FLAG[engine], "-interaction=nonstopmode", main], cwd=d, env=env) + pdf = d / main.replace(".tex", ".pdf") + return r.returncode == 0 and pdf.is_file(), r.stdout[-2000:] + r.stderr[-2000:], home def timed_run(scratch, main, engine, home, out): @@ -164,69 +191,282 @@ def table(results, doc, engines): return "\n".join(lines) + "\n" -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--runs", type=int, default=5) - ap.add_argument("--docs", default="minimal,standard,florilegium,large") - ap.add_argument("--engines", default="pdflatex,xelatex,lualatex") - ap.add_argument("--out", default="results") - a = ap.parse_args() - out = Path(a.out) - out.mkdir(parents=True, exist_ok=True) - engines = a.engines.split(",") - work = Path("/tmp/bench") +def phases_md(results): + m = results["machine"] + md = [f"Machine: {m['cpu']}, pinned to {m['cpus_allowed']} CPU; {m['pdftex']}; image {m['image']}. " + f"Warm medians of {results['runs']} runs unless the heading says otherwise; the cold run has a fresh HOME.\n"] + for doc in (d for d in DOCS if d in results): + engines = [e for e in ENGINES if e in results[doc]] + runs = DOCS[doc].get("runs") + md.append(f"\n### {title(doc)}{f' ({runs} warm runs)' if runs else ''}\n\n" + table(results, doc, engines)) + for e in engines: + entry = results[doc][e] + if "failed" in entry: + md.append(f"\n{e}: the latexmk build failed, so nothing was timed.\n") + sc = entry.get("scale") + if sc: + md.append( + f"\n{e}: {sc['record_lines']:,} records, {sc['record_bytes'] / 1024:,.0f} KiB. " + f"`prumo time --json` reads the cached run in {sc['analyze_cached_ms']:,.0f} ms, and " + f"`--html` writes {sc['html_bytes'] / 1024:,.0f} KiB in {sc['html_ms']:,.0f} ms.\n" + ) + return "".join(md) + + +def run_phases(a, out, work): results = {"machine": machine(), "runs": a.runs} for doc in a.docs.split(","): - _, main_file, supported, runs_override = DOCS[doc] - runs = runs_override or a.runs + runs = DOCS[doc].get("runs") or a.runs results[doc] = {} - for engine in engines: - if supported and engine not in supported: - results[doc][engine] = {"skipped": f"the document needs {', '.join(supported)}"} + for engine in a.engines.split(","): + if engine not in DOCS[doc].get("engines", ENGINES): continue print(f"== {doc} on {engine}", flush=True) scratch = work / f"{doc}-{engine}" - ok, log = prepare(doc, engine, scratch) + ok, log, _ = prepare(doc, engine, scratch) if not ok: results[doc][engine] = {"failed": "latexmk", "log": log} print(f" latexmk failed:\n{log[-800:]}", flush=True) continue + d = doc_dir(doc, scratch) + main_file = DOCS[doc]["main"] home = work / f"home-{doc}-{engine}" cache = work / f"cache-{doc}-{engine}" - for d in (home, cache): - shutil.rmtree(d, ignore_errors=True) - d.mkdir(parents=True) - cold = summary(*timed_run(scratch, main_file, engine, home, cache)) + for p in (home, cache): + shutil.rmtree(p, ignore_errors=True) + p.mkdir(parents=True) + cold = summary(*timed_run(d, main_file, engine, home, cache)) print(f" cold {cold['wall_ms']:9.1f} ms wall, {cold['pages']} pages", flush=True) warm = [] for i in range(runs): - s = summary(*timed_run(scratch, main_file, engine, home, cache)) + s = summary(*timed_run(d, main_file, engine, home, cache)) warm.append(s) print(f" warm {i + 1} {s['wall_ms']:9.1f} ms wall", flush=True) entry = {"cold": cold, "warm": warm, "median": median_of(warm)} if doc == "large": - entry["scale"] = scale(scratch, main_file, engine, home, cache) + entry["scale"] = scale(d, main_file, engine, home, cache) if (cache / f"{engine}.html").is_file(): shutil.move(str(cache / f"{engine}.html"), str(out / f"large-{engine}.html")) results[doc][engine] = entry (out / "results.json").write_text(json.dumps(results, indent=1)) - md = [f"Machine: {results['machine']['cpu']}, pinned to {results['machine']['cpus_allowed']} CPU; " - f"{results['machine']['pdftex']}; image {results['machine']['image']}. " - f"Warm medians of {a.runs} runs (3 for the large document); the cold run has a fresh HOME.\n"] + (out / "results.json").write_text(json.dumps(results, indent=1)) + (out / "results.md").write_text(phases_md(results)) + print(phases_md(results)) + + +def run_whatif(a, out, work): + """`prumo whatif` on each document and engine; one JSON file per document.""" + (out / "whatif").mkdir(exist_ok=True) + for doc in a.docs.split(","): + runs = DOCS[doc].get("runs") or a.runs + entry = {"machine": machine(), "runs": runs} + for engine in a.engines.split(","): + if engine not in DOCS[doc].get("engines", ENGINES): + continue + print(f"== whatif {doc} on {engine}", flush=True) + scratch = work / f"{doc}-{engine}" + ok, log, home = prepare(doc, engine, scratch) + if not ok: + entry[engine] = {"failed": "latexmk", "log": log} + print(f" latexmk failed:\n{log[-800:]}", flush=True) + continue + cache = work / f"cache-whatif-{doc}-{engine}" + shutil.rmtree(cache, ignore_errors=True) + env = dict(os.environ, HOME=str(home), PRUMO_CACHE=str(cache)) + main = doc_dir(doc, scratch) / DOCS[doc]["main"] + cmd = ["prumo", "whatif", "--json", "--shell-escape", "--engine", engine, "--runs", str(runs), str(main)] + t0 = time.monotonic() + r = sh(cmd, env=env) + elapsed = round(time.monotonic() - t0, 1) + if r.returncode != 0: + entry[engine] = {"failed": "prumo whatif", "log": r.stderr[-2000:]} + print(f" prumo whatif failed:\n{r.stderr[-800:]}", flush=True) + else: + entry[engine] = {"report": json.loads(r.stdout), "elapsed_s": elapsed} + for row in entry[engine]["report"]["experiments"]: + print(f" {row['experiment']:<12} {cell(row)}", flush=True) + (out / "whatif" / f"{doc}.json").write_text(json.dumps(entry, indent=1)) + (out / "whatif.md").write_text(whatif_md(out)) + + +# The environments prumo watches by default (run::DEFAULT_ENVIRONMENTS), as it loads the probe. +PROBE_ENVS = "tikzpicture,axis,tcolorbox,tblr,longtable,tabularx,minted,lstlisting,forest" + + +def run_overhead(a, out, work): + """Plain and probed builds in turns, timed alike: what prumo's probe costs.""" + (out / "overhead").mkdir(exist_ok=True) for doc in a.docs.split(","): - md.append(f"\n### {doc}\n\n" + table(results, doc, engines)) - if doc == "large": - for e in engines: - sc = results[doc].get(e, {}).get("scale") - if sc: - md.append( - f"\n{e}: {sc['record_lines']:,} records ({sc['record_bytes'] / 1024:,.0f} KiB); " - f"`prumo time --json` on the cached run {sc['analyze_cached_ms']:,.0f} ms; " - f"`--html` {sc['html_ms']:,.0f} ms, {sc['html_bytes'] / 1024:,.0f} KiB.\n" + # --runs for every document: the cost is a few percent, and a long document's two or + # three runs cannot tell that from one slow run. + runs = a.runs + entry = {"machine": machine(), "runs": runs} + for engine in a.engines.split(","): + if engine not in DOCS[doc].get("engines", ENGINES): + continue + print(f"== overhead {doc} on {engine}", flush=True) + scratch = work / f"{doc}-{engine}" + ok, log, home = prepare(doc, engine, scratch) + if not ok: + entry[engine] = {"failed": "latexmk", "log": log} + print(f" latexmk failed:\n{log[-800:]}", flush=True) + continue + d = doc_dir(doc, scratch) + main = DOCS[doc]["main"] + job = main.removesuffix(".tex") + plain = [engine, "-interaction=nonstopmode", main] + probed = [engine, "-interaction=nonstopmode", "-recorder", f"-jobname={job}", + f"\\RequirePackage{{prumo}}\\PrumoEnvironments{{{PROBE_ENVS}}}\\input{{{job}}}"] + env = dict(os.environ, HOME=str(home)) + env_probed = dict(env, TEXINPUTS="/probe:") + + def timed(cmd, e): + t0 = time.monotonic() + r = subprocess.run(cmd, cwd=d, env=e, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return round((time.monotonic() - t0) * 1000, 3), r.returncode + + timed(plain, env) + timed(probed, env_probed) + res = {"plain_ms": [], "probed_ms": [], "units": {}, "exit_codes": []} + for i in range(runs): + p, pc = timed(plain, env) + q, qc = timed(probed, env_probed) + res["plain_ms"].append(p) + res["probed_ms"].append(q) + res["exit_codes"].append([pc, qc]) + r = sh(["prumo", "time", "--json", str(d / f"{job}.prumo")], env=env) + if r.returncode == 0: + j = json.loads(r.stdout) + res["pages"] = len(j["pages"]) + for u in j["units"]: + res["units"].setdefault(f"{u['kind']} {u['name']}", []).append(u["exclusive_ms"]) + print(f" run {i + 1}: plain {p:9.1f} ms, probed {q:9.1f} ms", flush=True) + res["records"] = sum(1 for _ in open(d / f"{job}.prumo")) + entry[engine] = res + (out / "overhead" / f"{doc}.json").write_text(json.dumps(entry, indent=1)) + (out / "overhead.md").write_text(overhead_md(out)) + + +def overhead_md(out): + """The probe's cost per document and engine.""" + md = ["| document | engine | records | plain (ms) | with prumo (ms) | cost |", "|---|---|---:|---:|---:|---:|"] + for doc in DOCS: + f = out / "overhead" / f"{doc}.json" + if not f.is_file(): + continue + entry = json.loads(f.read_text()) + for e in ENGINES: + r = entry.get(e) + if not r or "plain_ms" not in r: + continue + p, q = statistics.median(r["plain_ms"]), statistics.median(r["probed_ms"]) + sp = max(r["plain_ms"]) - min(r["plain_ms"]) + sq = max(r["probed_ms"]) - min(r["probed_ms"]) + md.append(f"| {title(doc)} | {e} | {r['records']:,} | {p:,.1f} ± {sp:,.1f} | {q:,.1f} ± {sq:,.1f} | " + f"{q - p:+,.1f} ms ({(q - p) / p * 100:+.1f}%) |") + return "\n".join(md) + "\n" + + +def cell(row): + """One experiment in the summary table.""" + st = row["status"] + if st == "not-run": + return "–" + if st == "failed": + return "fails" + if row["within_noise"]: + text = "noise" + elif row["saved_ms"] >= 0: + text = f"{row['saved_ms']:,.0f} ms ({row['saved_pct']:.0f}%)" + else: + text = f"{-row['saved_ms']:,.0f} ms slower" + if row["output"]["status"] == "changed": + text += ", output changed" + return text + + +def whatif_md(out): + """The summary table, the notes it needs, and every row in full.""" + reports = {} + for doc in DOCS: + f = out / "whatif" / f"{doc}.json" + if f.is_file(): + reports[doc] = json.loads(f.read_text()) + if not reports: + return "" + exps = ["preamble", "draft", "externalize"] + md = ["| document | engine | pages | baseline | " + " | ".join(exps) + " |", + "|---|---|---:|---:|" + "---|" * len(exps)] + notes, full = [], [] + for doc, entry in reports.items(): + for e in ENGINES: + if e not in entry: + continue + if "report" not in entry[e]: + md.append(f"| {title(doc)} | {e} | | | {entry[e]['failed']} failed | | |") + continue + r = entry[e]["report"] + rows = {x["experiment"]: x for x in r["experiments"]} + md.append( + f"| {title(doc)} | {e} | {r['baseline_pages']:,} | {r['baseline']['median_ms']:,.0f} ms | " + + " | ".join(cell(rows[x]) if x in rows else "" for x in exps) + " |" + ) + for x in exps: + row = rows.get(x) + if not row: + continue + if row["status"] == "failed": + notes.append(f"- {title(doc)}, {e}, {x}: {row['error']}") + elif row["status"] == "measured" and row["output"]["status"] == "changed": + notes.append(f"- {title(doc)}, {e}, {x}: output changed, {row['output']['evidence']}") + bound = "" if row.get("bound_ms") is None else f"{row['bound_ms']:,.0f}" + if row["status"] == "measured": + v = row["variant"] + one = "" if row["one_time_ms"] is None else f"{row['one_time_ms']:,.0f}" + full.append( + f"| {title(doc)} | {e} | {x} | {r['baseline']['median_ms']:,.1f} ± {r['baseline']['spread_ms']:,.1f} | " + f"{v['median_ms']:,.1f} ± {v['spread_ms']:,.1f} | {row['saved_ms']:,.1f} | {bound} | {one} | " + f"{row['output']['status']} |" ) - (out / "results.md").write_text("".join(md)) - (out / "results.json").write_text(json.dumps(results, indent=1)) - print("".join(md)) + else: + why = row.get("reason") or row.get("error", "") + full.append(f"| {title(doc)} | {e} | {x} | | | {row['status']}: {why.replace('|', '/')} | {bound} | | |") + md.append("") + if notes: + md += ["Failures and changed output:", ""] + notes + [""] + md += ["Every row (ms; ± is the spread, slowest run minus fastest):", "", + "| document | engine | experiment | baseline | variant | saved | bound | one-time | output |", + "|---|---|---|---:|---:|---:|---:|---:|---|"] + full + return "\n".join(md) + "\n" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--runs", type=int, default=5) + ap.add_argument("--docs", default=",".join(DOCS)) + ap.add_argument("--engines", default=",".join(ENGINES)) + ap.add_argument("--whatif", action="store_true") + ap.add_argument("--overhead", action="store_true") + ap.add_argument("--report", action="store_true") + ap.add_argument("--out", default="results") + a = ap.parse_args() + unknown = [d for d in a.docs.split(",") if d not in DOCS] + if unknown: + raise SystemExit(f"not in corpus.toml: {', '.join(unknown)}") + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + work = Path("/tmp/bench") + if a.report: + if (out / "results.json").is_file(): + (out / "results.md").write_text(phases_md(json.loads((out / "results.json").read_text()))) + (out / "whatif.md").write_text(whatif_md(out)) + (out / "overhead.md").write_text(overhead_md(out)) + elif a.overhead: + run_overhead(a, out, work) + elif a.whatif: + run_whatif(a, out, work) + else: + run_phases(a, out, work) if __name__ == "__main__": diff --git a/docs/benchmarks/charts.py b/docs/benchmarks/charts.py new file mode 100644 index 0000000..8af07bb --- /dev/null +++ b/docs/benchmarks/charts.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Charts for docs/benchmarks/README.md, as SVG, from the JSON under results/. Standard +library only. Each chart is written twice, for a light and a dark page; the README picks one +with , and the exact numbers are in the tables under each chart. + + phases each document in its own panel and on its own axis: one bar per engine, cut + into startup, preamble, body and writing the PDF, with every run's wall time + overhead every plain and every probed run, against the plain runs' median + packages the paper's package load times on pdfLaTeX and LuaLaTeX, by the difference + whatif what each experiment saved, as a share of the build, with the probe's bound + +Color means phase, in every chart: startup is a neutral gray (no experiment changes it), +dark on a light page and light on a dark one, the preamble violet, the body magenta, writing +the PDF yellow. The same hue marks the experiment that shrinks that phase. Engines are told +apart by position and label, and in the package chart by a hollow or filled dot. The colors +are the dataviz reference palette's. Every pair of the four, the gray included, is at least +OKLab ΔE 13 apart under simulated protan and deutan vision and 19 apart under normal vision, +in both modes (measured with its validate_palette.js; the mid gray it had before sat 1.1 +from the body's magenta for deutan readers). + +Usage: charts.py [--results DIR] [--out DIR] +""" +import argparse +import json +import statistics +import tomllib +from html import escape +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CORPUS = tomllib.loads((HERE / "corpus.toml").read_text()) + +THEMES = { + "light": { + "surface": "#fcfcfb", "ink": "#0b0b0b", "ink2": "#52514e", "muted": "#898781", + "grid": "#e1e0d9", "axis": "#c3c2b7", + "startup": "#52514e", "preamble": "#6250d6", "body": "#e87ba4", "pdf": "#eda100", + "inside": {"pdf": "#0b0b0b"}, + }, + "dark": { + "surface": "#1a1a19", "ink": "#f0efec", "ink2": "#c3c2b7", "muted": "#898781", + "grid": "#2c2c2a", "axis": "#383835", + "startup": "#c3c2b7", "preamble": "#9085e9", "body": "#d55181", "pdf": "#c98500", + "inside": {"pdf": "#0b0b0b", "startup": "#0b0b0b"}, + }, +} +ENGINES = ["pdflatex", "xelatex", "lualatex"] +NAME = {"pdflatex": "pdfLaTeX", "xelatex": "XeLaTeX", "lualatex": "LuaLaTeX"} +GROUPS = [ + ("startup", "startup", ["startup"]), + ("preamble", "preamble", ["preamble"]), + ("body", "body", ["begin_document", "typesetting", "end_document"]), + ("pdf", "writing the PDF", ["shipout", "finalization"]), +] +EXPERIMENTS = [("preamble", "preamble", "precompiled preamble"), ("draft", "pdf", "draft mode"), + ("externalize", "body", "externalized pictures")] +FONT = "system-ui, -apple-system, 'Segoe UI', sans-serif" +W = 860 + + +def title(doc): + """What a reader calls the document: the title in corpus.toml, else its key.""" + return CORPUS.get(doc, {}).get("title", doc) + + +def text_width(s, size=12): + """Rough width of system-ui text; there are no font metrics here, so per-class widths.""" + w = 0.0 + for ch in s: + if ch in "il.,:;|!'·": + w += 0.28 + elif ch in "fjrt() -": + w += 0.36 + elif ch.isdigit() or ch in "%×+−": + w += 0.58 + elif ch.isupper() or ch in "mw": + w += 0.74 + else: + w += 0.55 + return w * size + + +def fmt_time(ms): + if ms < 1000: + return f"{ms:,.0f} ms" + if ms >= 120_000: + return f"{ms / 60_000:,.0f} min" + return f"{ms / 1000:,.2f} s" if ms < 10_000 else f"{ms / 1000:,.1f} s" + + +def nice_ticks(hi, n=4): + """Round tick values from 0 to at least hi.""" + raw = hi / n + mag = 10 ** len(str(int(raw))) / 10 if raw >= 1 else 1 + step = next(s * mag for s in (1, 2, 2.5, 5, 10) if s * mag >= raw) + ticks, v = [], 0.0 + while v < hi + step * 0.999: + ticks.append(v) + v += step + return ticks + + +class Svg: + def __init__(self, height, t, title, desc): + self.t = t + self.parts = [ + f'', + f"{escape(title)}{escape(desc)}", + f'', + ] + + def add(self, s): + self.parts.append(s) + + def text(self, x, y, s, size=12, fill="ink", anchor="start", weight=None): + w = f' font-weight="{weight}"' if weight else "" + nums = ' font-variant-numeric="tabular-nums"' if any(c.isdigit() for c in s) else "" + self.add(f'{escape(s)}') + + def line(self, x1, y1, x2, y2, stroke, width=1, tip=None): + t = f"{escape(tip)}" if tip else "" + self.add(f'{t}') + + def bar(self, x, y, w, h, fill, tip, round_end=True): + """A bar from a baseline at x: square there, 4 px rounded at the data end.""" + if w <= 0: + return + r = min(4, w / 2, h / 2) if round_end else 0 + d = (f"M{x:.1f},{y:.1f} h{w - r:.1f} a{r},{r} 0 0 1 {r},{r} v{h - 2 * r:.1f} " + f"a{r},{r} 0 0 1 -{r},{r} h-{w - r:.1f} z") + self.add(f'{escape(tip)}') + + def dot(self, cx, cy, fill, tip, hollow=False, r=4): + t = self.t + if hollow: + self.add(f'{escape(tip)}') + else: + self.add(f'{escape(tip)}') + + def swatch(self, x, y, key, name): + self.add(f'') + self.text(x + 15, y, name, fill="ink2") + return x + 15 + text_width(name) + 18 + + def header(self, title, subtitle): + self.text(24, 34, title, size=17, weight="600") + self.text(24, 56, subtitle, fill="ink2") + + def done(self): + return "\n".join(self.parts + [""]) + "\n" + + +def load(results_dir): + def per_doc(sub): + out = {} + for doc in CORPUS: + f = results_dir / sub / f"{doc}.json" + if f.is_file(): + out[doc] = json.loads(f.read_text()) + return out + + f = results_dir / "results.json" + return { + "phases": json.loads(f.read_text()) if f.is_file() else {}, + "whatif": per_doc("whatif"), + "overhead": per_doc("overhead"), + } + + +def caption(data): + """Machine, TeX Live and runs, for the foot of every chart.""" + p = data["phases"] + m = p.get("machine") or next(iter(data["whatif"].values()), {}).get("machine", {}) + cpu = m.get("cpu", "?").replace("13th Gen Intel(R) Core(TM) ", "") + fmt = next((p[d][e]["cold"]["format_date"] for d in CORPUS if d in p for e in ENGINES + if "cold" in p[d].get(e, {})), None) + runs = sorted({(CORPUS[d].get("runs") or p.get("runs", 5)) for d in CORPUS}) + by_runs = {} + for d in CORPUS: + if CORPUS[d].get("runs"): + by_runs.setdefault(CORPUS[d]["runs"], []).append(title(d)) + fewer = "; ".join(f"{' and '.join(names)} {n}" for n, names in sorted(by_runs.items(), reverse=True)) + return (f"{cpu}, one P-core per container · TeX Live 2026{f', LaTeX {fmt}' if fmt else ''} · " + f"warm runs: {max(runs)}{f'; {fewer}' if fewer else ''}") + + +def walls(entry): + return [r["wall_ms"] for r in entry["warm"] if r["wall_ms"] is not None] + + +def doc_order(data): + """Documents from the quickest build to the slowest (the first engine's median).""" + p = data["phases"] + + def key(doc): + for e in ENGINES: + if "median" in p.get(doc, {}).get(e, {}): + return p[doc][e]["median"]["wall"] + return float("inf") + + return sorted((d for d in CORPUS if d in p or d in data["whatif"] or d in data["overhead"]), key=key) + + +def phases_chart(data, t): + p = data["phases"] + docs = [d for d in doc_order(data) if any("median" in p.get(d, {}).get(e, {}) for e in ENGINES)] + pw = (W - 48 - 24) / 2 + head, row_h, axis_h, gap = 30, 22, 26, 22 + + def panel_h(doc): + n = sum("median" in p[doc].get(e, {}) for e in ENGINES) + return head + n * row_h + axis_h + + top = 100 + heights = [] + for i in range(0, len(docs), 2): + heights.append(max(panel_h(d) for d in docs[i:i + 2])) + height = top + sum(heights) + gap * (len(heights) - 1) + 40 + svg = Svg(height, t, "Where the time goes", + "One panel per benchmark document, each on its own time axis. For each engine a bar cut into " + "startup, preamble, body and writing the PDF (medians of the warm runs), with every run's wall " + "time as a dot.") + svg.header("Where the time goes", "Median of the warm runs, cut into phases. The dots are each run's wall " + "time. Each panel has its own axis.") + x = 24 + for key, name, _ in GROUPS: + x = svg.swatch(x, 84, key, name) + svg.dot(x + 4, 80, "ink", "one run's wall time") + svg.text(x + 14, 84, "one run", fill="ink2") + y = top + for i in range(0, len(docs), 2): + for j, doc in enumerate(docs[i:i + 2]): + px = 24 + j * (pw + 24) + engines = [e for e in ENGINES if "median" in p[doc].get(e, {})] + pages = p[doc][engines[0]]["cold"]["pages"] + svg.text(px, y + 14, title(doc), size=13, weight="600") + svg.text(px + text_width(title(doc), 13) * 1.2 + 10, y + 14, f"{pages:,} page{'s' if pages != 1 else ''}", + size=11, fill="muted") + label_w, val_w = 70, 62 + bx, bw = px + label_w, pw - label_w - val_w + hi = max(max(walls(p[doc][e])) for e in engines) + ticks = nice_ticks(hi, 3) + scale = bw / ticks[-1] + y0, y1 = y + head - 4, y + head + len(engines) * row_h + for tk in ticks: + gx = bx + tk * scale + svg.line(gx, y0, gx, y1, "axis" if tk == 0 else "grid") + svg.text(gx, y1 + 14, fmt_time(tk) if tk else "0", size=10, fill="muted", + anchor="start" if tk == 0 else "middle") + for k, e in enumerate(engines): + m = p[doc][e]["median"] + ry = y + head + k * row_h + cy = ry + row_h / 2 + svg.text(bx - 8, cy + 4, NAME[e], fill="ink2", anchor="end") + parts = [(key, name, sum(m[ph] or 0 for ph in phs)) for key, name, phs in GROUPS] + total = sum(v for *_, v in parts) + cx = bx + shown = [(key, name, v) for key, name, v in parts if v * scale >= 1] + for n, (key, name, v) in enumerate(shown): + w = v * scale - (2 if n < len(shown) - 1 else 0) + svg.bar(cx, ry + 4, max(w, 1), row_h - 8, key, + f"{doc}, {NAME[e]}: {name} {v:,.1f} ms ({v / total * 100:.0f}%)", + round_end=n == len(shown) - 1) + pct = f"{v / total * 100:.0f}%" + # The last segment ends where the run dots are: leave them room. + room = 8 if n < len(shown) - 1 else 22 + if w >= text_width(pct, 10) + room: + svg.text(cx + 4, cy + 3.5, pct, size=10, fill=t["inside"].get(key, "#ffffff"), weight="600") + cx += w + 2 + for wv in walls(p[doc][e]): + svg.dot(bx + wv * scale, cy, "ink", f"{doc}, {NAME[e]}: a run of {fmt_time(wv)}", r=3) + svg.text(px + pw, cy + 4, fmt_time(m["wall"]), fill="ink2", anchor="end") + y += heights[i // 2] + gap + svg.text(24, height - 16, caption(data), size=11, fill="muted") + return svg.done() + + +def grouped_rows(data, source): + """(document or "", engine, entry) rows, the document named on its first row only.""" + rows = [] + for doc in doc_order(data): + entry = data[source].get(doc) + if not entry: + continue + engines = [e for e in ENGINES if e in entry and "failed" not in entry[e]] + for i, e in enumerate(engines): + rows.append((title(doc) if i == 0 else "", e, entry[e], i == 0)) + return rows + + +def overhead_chart(data, t): + rows = grouped_rows(data, "overhead") + if not rows: + return None + label_w, row_h, gap, top = 250, 22, 10, 100 + rel = [] + for *_, r, _ in rows: + base = statistics.median(r["plain_ms"]) + rel += [v / base * 100 - 100 for v in r["plain_ms"] + r["probed_ms"]] + lo, hi = min(min(rel), -1), max(max(rel), 1) + step = next(s for s in (1, 2, 5, 10, 20, 50) if (hi - lo) / s <= 8) + lo, hi = step * (lo // step), step * -(-hi // step) + plot_x, plot_w = label_w, W - label_w - 150 + xs = lambda v: plot_x + (v - lo) / (hi - lo) * plot_w # noqa: E731 + height = top + sum(row_h + (gap if first else 0) for *_, first in rows) + 64 + svg = Svg(height, t, "What the probe costs", + "For each benchmark document and engine, every build without prumo and every build with it " + "(the probe and -recorder), as a difference from the median build without it.") + svg.header("What the probe costs", "Builds with the probe and -recorder against builds without, in turns. " + "0 is the median without, the tick the median with.") + x = 24 + svg.dot(x + 4, 80, "ink2", "a build without prumo", hollow=True) + svg.text(x + 14, 84, "without prumo", fill="ink2") + x += 14 + text_width("without prumo") + 22 + svg.dot(x + 4, 80, "ink", "a build with prumo") + svg.text(x + 14, 84, "with prumo", fill="ink2") + y0 = top + y = top + for doc, e, r, first in rows: + if first: + y += gap + y += row_h + y1 = y + v = lo + while v <= hi + 1e-9: + svg.line(xs(v), y0 + gap - 4, xs(v), y1, "axis" if v == 0 else "grid") + svg.text(xs(v), y1 + 16, f"{v:+.0f}%" if v else "0", size=10, fill="muted", anchor="middle") + v += step + y = top + for doc, e, r, first in rows: + if first: + y += gap + if y > top + gap: + svg.line(24, y - gap / 2, label_w - 8, y - gap / 2, "grid") + cy = y + row_h / 2 + if doc: + svg.text(24, cy + 4, doc, weight="600") + svg.text(label_w - 12, cy + 4, NAME[e], fill="ink2", anchor="end") + base = statistics.median(r["plain_ms"]) + for val in r["plain_ms"]: + svg.dot(xs(val / base * 100 - 100), cy, "ink2", f"{doc}, {NAME[e]}: {fmt_time(val)} without prumo", + hollow=True) + for val in r["probed_ms"]: + svg.dot(xs(val / base * 100 - 100), cy, "ink", f"{doc}, {NAME[e]}: {fmt_time(val)} with prumo") + med = statistics.median(r["probed_ms"]) + d = med - base + svg.line(xs(d / base * 100), y + 3, xs(d / base * 100), y + row_h - 3, "ink", 2, + f"{doc}, {NAME[e]}: median with prumo {fmt_time(med)}, without {fmt_time(base)}") + svg.text(plot_x + plot_w + 14, cy + 4, f"{d:+,.0f} ms ({d / base * 100:+.1f}%)", fill="ink2") + y += row_h + svg.text(plot_x + plot_w + 14, top + 2, "median cost", size=11, fill="muted") + svg.text(24, height - 16, caption(data), size=11, fill="muted") + return svg.done() + + +def packages_chart(data, t, doc="standard", a="pdflatex", b="lualatex", n=12): + entry = data["overhead"].get(doc, {}) + if a not in entry or b not in entry or "units" not in entry[a]: + return None, None + med = {e: {k: statistics.median(v) for k, v in entry[e]["units"].items() if k.split()[0] in ("pkg", "cls")} + for e in (a, b)} + keys = set(med[a]) | set(med[b]) + rows = sorted(keys, key=lambda k: -max(med[a].get(k, 0), med[b].get(k, 0)))[:n] + rows.sort(key=lambda k: -(med[b].get(k, 0) - med[a].get(k, 0))) + label_w, row_h, top = 150, 24, 100 + hi = max(max(med[a].get(k, 0), med[b].get(k, 0)) for k in rows) + ticks = nice_ticks(hi, 5) + plot_w = W - label_w - 140 + xs = lambda v: label_w + v / ticks[-1] * plot_w # noqa: E731 + pre = data["phases"].get(doc, {}) + pa = pre.get(a, {}).get("median", {}).get("preamble") + pb = pre.get(b, {}).get("median", {}).get("preamble") + height = top + len(rows) * row_h + 66 + svg = Svg(height, t, f"Package load times on {NAME[a]} and {NAME[b]}", + f"The {doc} document's slowest packages and classes, exclusive time on {NAME[a]} (hollow) and " + f"{NAME[b]} (filled), sorted by how much longer {NAME[b]} takes.") + sub = f"The paper ({doc}): each package's own time, without the packages it loads, median of the runs." + svg.header(f"Package load times, {NAME[a]} against {NAME[b]}", sub) + x = 24 + svg.dot(x + 4, 80, "preamble", NAME[a], hollow=True) + svg.text(x + 14, 84, NAME[a], fill="ink2") + x += 14 + text_width(NAME[a]) + 22 + svg.dot(x + 4, 80, "preamble", NAME[b]) + svg.text(x + 14, 84, NAME[b], fill="ink2") + if pa and pb: + svg.text(x + 14 + text_width(NAME[b]) + 22, 84, + f"whole preamble: {fmt_time(pa)} and {fmt_time(pb)}", fill="muted") + y1 = top + len(rows) * row_h + for tk in ticks: + svg.line(xs(tk), top, xs(tk), y1, "axis" if tk == 0 else "grid") + svg.text(xs(tk), y1 + 16, f"{tk:g} ms", size=10, fill="muted", anchor="middle") + svg.text(label_w + plot_w + 18, top - 6, "difference", size=11, fill="muted") + table = [f"| package | {NAME[a]} (ms) | {NAME[b]} (ms) | difference |", "|---|---:|---:|---:|"] + for i, k in enumerate(rows): + kind, name = k.split(" ", 1) + cy = top + i * row_h + row_h / 2 + svg.text(label_w - 12, cy + 4, name + (" (class)" if kind == "cls" else ""), anchor="end") + va, vb = med[a].get(k), med[b].get(k) + if va is not None and vb is not None: + svg.line(xs(min(va, vb)), cy, xs(max(va, vb)), cy, "axis", 2) + svg.dot(xs(va), cy, "preamble", f"{name}: {va:,.1f} ms on {NAME[a]}", hollow=True) + svg.dot(xs(vb), cy, "preamble", f"{name}: {vb:,.1f} ms on {NAME[b]}") + svg.text(label_w + plot_w + 18, cy + 4, f"{vb - va:+,.0f} ms", fill="ink2") + elif vb is not None: + svg.dot(xs(vb), cy, "preamble", f"{name}: {vb:,.1f} ms, {NAME[b]} only") + svg.text(label_w + plot_w + 18, cy + 4, f"{NAME[b]} only", fill="muted") + else: + svg.dot(xs(va), cy, "preamble", f"{name}: {va:,.1f} ms, {NAME[a]} only", hollow=True) + svg.text(label_w + plot_w + 18, cy + 4, f"{NAME[a]} only", fill="muted") + fa = "–" if va is None else f"{va:,.1f}" + fb = "–" if vb is None else f"{vb:,.1f}" + diff = "" if va is None or vb is None else f"{vb - va:+,.1f}" + table.append(f"| {name}{' (class)' if kind == 'cls' else ''} | {fa} | {fb} | {diff} |") + svg.text(24, height - 16, caption(data), size=11, fill="muted") + return svg.done(), "\n".join(table) + "\n" + + +def whatif_chart(data, t): + rows = [(d, e, r["report"], f) for d, e, r, f in grouped_rows(data, "whatif") if "report" in r] + if not rows: + return None + label_w, gap, row_h, top = 250, 10, 22, 104 + panel_gap = 16 + panel_w = (W - label_w - 24 - 2 * panel_gap) / 3 + plot_w = panel_w - 70 + height = top + sum(row_h + (gap if first else 0) for *_, first in rows) + 88 + svg = Svg(height, t, "What prumo whatif saves", + "For each benchmark document and engine, the share of the build each whatif experiment saved, " + "with the probe's bound for it and the one-time cost.") + svg.header("What prumo whatif saves", "Share of the build saved (medians). The tick is the probe's bound: the " + "most the change can save.") + y_end = top + sum(row_h + (gap if first else 0) for *_, first in rows) + for j, (name, key, title) in enumerate(EXPERIMENTS): + px = label_w + j * (panel_w + panel_gap) + svg.swatch(px, 84, key, title) + for pct in (0, 50, 100): + gx = px + pct / 100 * plot_w + svg.line(gx, top + gap - 4, gx, y_end, "axis" if pct == 0 else "grid") + svg.text(gx, y_end + 16, f"{pct}%", size=10, fill="muted", anchor="start" if pct == 0 else "middle") + y = top + for doc, e, r, first in rows: + if first: + y += gap + if y > top + gap: + svg.line(24, y - gap / 2, label_w - 8, y - gap / 2, "grid") + cy = y + row_h / 2 + if doc: + svg.text(24, cy + 4, doc, weight="600") + svg.text(label_w - 12, cy + 4, NAME[e], fill="ink2", anchor="end") + base = r["baseline"]["median_ms"] + by = {x["experiment"]: x for x in r["experiments"]} + for j, (name, key, title) in enumerate(EXPERIMENTS): + px = label_w + j * (panel_w + panel_gap) + row = by.get(name) + if not row: + continue + label = f"{doc or ''} {NAME[e]}, {title}".strip() + right = px + if row.get("bound_ms") is not None and row["status"] == "measured" and base > 0: + bx = px + min(row["bound_ms"] / base, 1) * plot_w + svg.line(bx, y + 4, bx, y + row_h - 4, "ink2", 2, f"{label}: bound {fmt_time(row['bound_ms'])}") + right = bx + if row["status"] != "measured": + note = "–" if row["status"] == "not-run" else "fails" + svg.text(max(px + 4, right + 5), cy + 4, note, size=11, fill="muted") + continue + noise = row["within_noise"] + if row["saved_ms"] > 0: + w = min(row["saved_pct"], 100) / 100 * plot_w + changed = row["output"]["status"] == "changed" + tip = (f"{label}: saved {fmt_time(row['saved_ms'])} of {fmt_time(base)} ({row['saved_pct']:.0f}%)" + + (", within noise" if noise else "") + + (f"; output changed: {row['output']['evidence']}" if changed else "")) + svg.bar(px, y + 6, max(w, 2), row_h - 12, "muted" if noise else key, tip) + right = max(right, px + max(w, 2)) + if noise: + svg.text(right + 5, cy + 4, "noise", size=11, fill="muted") + continue + if row["saved_ms"] < 0: + svg.text(right + 5, cy + 4, f"{fmt_time(-row['saved_ms'])} slower", size=11, fill="ink2") + continue + lab = f"{row['saved_pct']:.0f}%" + (" ≠" if row["output"]["status"] == "changed" else "") + once = f" · once {fmt_time(row['one_time_ms'])}" if row.get("one_time_ms") else "" + room = px + panel_w - (right + 5) + if once and text_width(lab + once, 11) <= room: + lab += once + svg.text(right + 5, cy + 4, lab, size=11, fill="ink2") + y += row_h + svg.text(24, height - 54, "gray: within noise, no larger than the runs' spread. fails: the variant did not " + "build. –: not tried here. ≠: the output changed.", size=11, fill="muted") + svg.text(24, height - 38, "once: paid on the first build only, building the format or every picture.", + size=11, fill="muted") + svg.text(24, height - 16, caption(data), size=11, fill="muted") + return svg.done() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", type=Path, default=HERE / "results") + ap.add_argument("--out", type=Path, default=HERE / "img") + a = ap.parse_args() + a.out.mkdir(parents=True, exist_ok=True) + data = load(a.results) + for mode, t in THEMES.items(): + charts = {"phases": phases_chart(data, t) if data["phases"] else None, + "overhead": overhead_chart(data, t), "whatif": whatif_chart(data, t)} + charts["packages"], table = packages_chart(data, t) + for name, svg in charts.items(): + if svg: + (a.out / f"{name}-{mode}.svg").write_text(svg) + if table: + (a.results / "packages.md").write_text(table) + print(f"charts in {a.out}") + + +if __name__ == "__main__": + main() diff --git a/docs/benchmarks/corpus.toml b/docs/benchmarks/corpus.toml new file mode 100644 index 0000000..3bc130e --- /dev/null +++ b/docs/benchmarks/corpus.toml @@ -0,0 +1,116 @@ +# The benchmark documents. fetch.py lays them out under one directory and bench.py builds +# them. Each document comes from one of three places: +# +# source = "docs/" this repository +# generate = "