diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index f1690eae1..2bbe2581b 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -79,6 +79,17 @@ jobs: if [ -f "$PLUGIN" ]; then yq -i '.project.plugins = [env(PLUGIN)]' myst.yml fi + # The lecture sources still carry Sphinx `{raw}` blocks, whose source + # mystmd shows to the reader as escaped text under the page title. They + # are rewritten out of the sources at cutover, so the preview runs the + # same script and shows post-cutover content. It exits non-zero on a + # block it does not recognise, which fails the preview rather than + # publishing a page with literal markup on it. + - name: Rewrite `{raw}` blocks out of the lecture sources + if: github.event.action != 'closed' + working-directory: preview-content/${{ env.CONTENT_DIR }} + run: node ${{ github.workspace }}/scripts/rewrite-raw-blocks.mjs . + - name: Build static site if: github.event.action != 'closed' working-directory: preview-content/${{ env.CONTENT_DIR }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5e00b8e..142d4fc2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 that declares the part replaces the whole default, credit included, which is how a site states other terms ([#203](https://github.com/QuantEcon/quantecon-theme.mystmd/issues/203)). +- `scripts/rewrite-raw-blocks.mjs`, which rewrites Sphinx `{raw}` directives + out of lecture sources: it deletes the notebook logo header in both its + `{raw} jupyter` and `{raw} html` forms, turns an Our World in Data chart into + the native `{iframe}` directive, and unfences a `colspan`/`rowspan` table so + mystmd's HTML transform renders it. Any other `{raw}` block is reported with + its file and line and nothing is written. mystmd renders no `raw` node, so + until now a block's own source reached the reader as escaped text under the + page title. The PR preview runs the script over the lecture content it + builds, so previews show post-cutover sources and every theme PR exercises it + ([#204](https://github.com/QuantEcon/quantecon-theme.mystmd/issues/204)) ([#222](https://github.com/QuantEcon/quantecon-theme.mystmd/pull/222)). ## [2.7.0] - 2026-09-11 diff --git a/scripts/rewrite-raw-blocks.mjs b/scripts/rewrite-raw-blocks.mjs new file mode 100644 index 000000000..d675f5d9e --- /dev/null +++ b/scripts/rewrite-raw-blocks.mjs @@ -0,0 +1,247 @@ +#!/usr/bin/env node +// +// Rewrites `{raw}` directives out of QuantEcon lecture sources. +// +// `{raw}` is a Sphinx construct. mystmd parses the directive but renders no +// `raw` node, so a block's own source reaches the reader as escaped text under +// the page title, and the ipynb export drops it. Three shapes appear across the +// lecture repositories, and each has a mystmd-native equivalent or no reason to +// exist: +// +// * the notebook logo header (`
`), in both +// `{raw} jupyter` and `{raw} html` form -- deleted, because the header +// belongs to the ipynb export rather than to a copy in every lecture file +// (QuantEcon/mystmd#108); +// * an Our World in Data chart -- the `{iframe}` directive; +// * a table carrying `colspan`/`rowspan` -- the same HTML with its fence +// removed, which mystmd's HTML transform turns into a real table. +// +// Any other `{raw}` block is reported and nothing is written at all: an +// unrecognised block is a content decision rather than something to guess at. +// +// Usage: node scripts/rewrite-raw-blocks.mjs [path...] (default: ".") + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const SKIP_DIRS = new Set(['.git', 'node_modules', '_build', '.deploy']); + +// A directive fence opens with three or more backticks, colons or tildes, +// optionally indented, and names its directive in braces. A plain code fence +// (no braces) matches too, which is the point: the scanner has to step over one +// to avoid rewriting a `{raw}` block quoted as an example inside it. +// +// The tail is `(.*?)\s*$` rather than `(.*)$` so that a line carrying a +// directive argument still matches when the file uses CRLF: `.` excludes CR, +// and an unanchored `$` only matches at the very end, so a greedy tail would +// leave the CR unmatched and the whole block invisible. +const FENCE_OPEN = /^(\s*)(`{3,}|:{3,}|~{3,})\s*(?:\{([a-zA-Z0-9_-]+)\})?\s*(.*?)\s*$/; + +// The closing fence repeats the opening character at least as many times, which +// is how a longer fence nests a shorter one inside its body. +const closingFence = (marker) => + new RegExp(`^\\s*\\${marker[0]}{${marker.length},}\\s*$`); + +// Directives whose body is a listing rather than content. A `{raw}` block shown +// as an example inside one is sample text, so the scanner steps over these +// whole, exactly as it does over an unnamed code fence. +const LISTING_DIRECTIVES = new Set(['code', 'code-block', 'code-cell', 'literalinclude']); + +const joined = (body) => body.join('\n').trim(); +const isHeader = (body) => /id\s*=\s*["']qe-notebook-header["']/.test(joined(body)); +// Both shapes are recognised only under `html`, and only when the body is the +// whole element and nothing else: a body that merely starts with one may carry +// anything after it, and converting that would drop content silently. +const isIframe = (body) => /^]*>\s*<\/iframe>$/.test(joined(body)); +const isTable = (body) => /^][\s\S]*<\/table>$/.test(joined(body)); + +/** + * Rewrites a run of lines, recursing into the body of every other directive. + * + * Recursion is what catches a `{raw}` block written inside an `{exercise}` or + * `{solution}`; `base` is the 1-based line number of `lines[0]` in the file, so + * a block reported from inside one still names its real line. + */ +function rewriteLines(lines, base, state) { + const out = []; + let i = 0; + + while (i < lines.length) { + const match = FENCE_OPEN.exec(lines[i]); + if (!match) { + out.push(lines[i]); + i += 1; + continue; + } + + const [, indent, marker, name, argument] = match; + const close = closingFence(marker); + let end = i + 1; + while (end < lines.length && !close.test(lines[end])) end += 1; + if (end >= lines.length) { + // Unterminated, so not a fence whose extent this script can trust. + out.push(lines[i]); + i += 1; + continue; + } + + const body = lines.slice(i + 1, end); + const startLine = base + i; + + if (name !== 'raw') { + // Recurse into another directive, which is how a `{raw}` block written + // inside an `{exercise}` or `{solution}` is reached -- but never into a + // listing, whose content is sample text that may well quote one. + const listing = !name || LISTING_DIRECTIVES.has(name); + const inner = listing ? body : rewriteLines(body, startLine + 1, state); + out.push(lines[i], ...inner, lines[end]); + i = end + 1; + continue; + } + + if (isHeader(body)) { + state.counts.headers += 1; + i = end + 1; + // The blank line below the block goes with it. Lecture files put the + // header between a target label and the page title, so leaving the blank + // line would strand the label a paragraph above the heading it names. + if (lines[i] !== undefined && lines[i].trim() === '') i += 1; + continue; + } + + const html = argument === 'html'; + + if (html && isIframe(body)) { + const src = /src\s*=\s*["']([^"']+)["']/.exec(joined(body)); + if (src) { + state.counts.iframes += 1; + // `:width:` carries over the only sizing the raw markup set that the + // directive also expresses; its own aspect-ratio box supplies the rest. + out.push( + `${indent}${marker}{iframe} ${src[1]}`, + `${indent}:width: 100%`, + `${indent}${marker}` + ); + i = end + 1; + continue; + } + state.unknown.push({ line: startLine, reason: '{raw} iframe with no src' }); + out.push(...lines.slice(i, end + 1)); + i = end + 1; + continue; + } + + if (html && isTable(body)) { + state.counts.tables += 1; + // A bare HTML block runs on until a blank line, where the fence ended the + // block by itself. Without blank lines around it, the Markdown on either + // side is absorbed into the raw HTML and never parsed -- a heading after + // the table renders as literal text. + if (out.length && out[out.length - 1].trim() !== '') out.push(''); + out.push(...body); + i = end + 1; + if (lines[i] !== undefined && lines[i].trim() !== '') out.push(''); + continue; + } + + state.unknown.push({ line: startLine, reason: `{raw} ${argument || '(no format)'}` }); + out.push(...lines.slice(i, end + 1)); + i = end + 1; + } + + return out; +} + +/** + * Rewrites one document. Never throws on content: unclassified blocks come back + * in `unknown` and are left exactly as they were. + */ +export function rewriteDocument(source) { + // Split on either ending and rejoin with the one the file uses, so a CRLF + // source keeps its line endings instead of picking up a mixture. + const eol = source.includes('\r\n') ? '\r\n' : '\n'; + const lines = source.split(/\r?\n/); + const state = { counts: { headers: 0, iframes: 0, tables: 0 }, unknown: [] }; + + // YAML frontmatter is delimited by `---` rather than a fence, and is passed + // through untouched. + let start = 0; + if (lines[0]?.trim() === '---') { + const end = lines.indexOf('---', 1); + if (end > 0) start = end + 1; + } + + const head = lines.slice(0, start); + const body = rewriteLines(lines.slice(start), start + 1, state); + return { text: [...head, ...body].join(eol), ...state }; +} + +function* markdownFiles(root) { + if (fs.statSync(root).isFile()) { + if (root.endsWith('.md')) yield root; + return; + } + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const full = path.join(root, entry.name); + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) yield* markdownFiles(full); + } else if (entry.isFile() && entry.name.endsWith('.md')) { + yield full; + } + } +} + +function main(argv) { + const totals = { files: 0, headers: 0, iframes: 0, tables: 0 }; + const problems = []; + const pending = []; + + for (const root of argv.length ? argv : ['.']) { + for (const file of markdownFiles(root)) { + const source = fs.readFileSync(file, 'utf8'); + if (!source.includes('{raw}')) continue; + const { text, counts, unknown } = rewriteDocument(source); + for (const u of unknown) problems.push(`${file}:${u.line}: ${u.reason}`); + if (text !== source) { + pending.push([file, text]); + totals.files += 1; + totals.headers += counts.headers; + totals.iframes += counts.iframes; + totals.tables += counts.tables; + } + } + } + + // All or nothing: a half-rewritten tree is harder to reason about than one + // that still has every block in it. + if (problems.length) { + console.error('Unhandled `{raw}` blocks -- nothing was written:'); + for (const problem of problems) console.error(` ${problem}`); + return 1; + } + + console.log( + `rewrite-raw-blocks: ${totals.files} file(s) changed; ${totals.headers} header(s) deleted, ` + + `${totals.iframes} iframe(s) and ${totals.tables} table(s) converted` + ); + for (const [file, text] of pending) fs.writeFileSync(file, text); + return 0; +} + +// `import.meta.url` is already resolved through symlinks, so the invoked path +// has to be too: comparing it raw makes the script a silent no-op whenever any +// component of the path it was called by is a link. +function invokedDirectly() { + if (!process.argv[1]) return false; + try { + return fileURLToPath(import.meta.url) === fs.realpathSync(path.resolve(process.argv[1])); + } catch { + return false; + } +} + +if (invokedDirectly()) { + process.exit(main(process.argv.slice(2))); +} diff --git a/tests/unit/rewrite-raw-blocks.test.mjs b/tests/unit/rewrite-raw-blocks.test.mjs new file mode 100644 index 000000000..41be19427 --- /dev/null +++ b/tests/unit/rewrite-raw-blocks.test.mjs @@ -0,0 +1,263 @@ +/** + * Unit tests for the `{raw}` block rewriter (scripts/rewrite-raw-blocks.mjs). + * + * The fixtures below are the shapes the lecture sources actually carry, taken + * from `short_path.md` (a `{raw} html` header), `need_for_speed.md` (a + * `{raw} jupyter` header), `simple_linear_regression.md` (the Our World in Data + * chart, on a colon fence) and `opt_transport.md` (the factory table). + * + * Run with: npm run test:unit + */ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { rewriteDocument } from '../../scripts/rewrite-raw-blocks.mjs'; + +const FRONTMATTER = ['---', 'jupytext:', ' text_representation:', ' extension: .md', '---'].join( + '\n' +); + +const HEADER_BODY = [ + '
', + ' ', + ' QuantEcon', + ' ', + '
', +].join('\n'); + +const headerDoc = (lang, fence = '```') => + `${FRONTMATTER}\n\n(short_path)=\n${fence}{raw} ${lang}\n${HEADER_BODY}\n${fence}\n\n# Shortest Paths\n\nText.\n`; + +test('deletes a `{raw} jupyter` header, leaving the label on the heading', () => { + const { text, counts, unknown } = rewriteDocument(headerDoc('jupyter')); + assert.deepEqual(unknown, []); + assert.equal(counts.headers, 1); + assert.ok(!text.includes('{raw}')); + assert.ok(!text.includes('qe-notebook-header')); + // The blank line below the block goes with it, so the target still names the + // heading rather than sitting a paragraph above it. + assert.match(text, /\(short_path\)=\n# Shortest Paths/); +}); + +test('deletes a `{raw} html` header too', () => { + const { text, counts } = rewriteDocument(headerDoc('html')); + assert.equal(counts.headers, 1); + assert.ok(!text.includes('qe-notebook-header')); +}); + +test('deletes a header on a longer fence', () => { + const { counts } = rewriteDocument(headerDoc('html', '````')); + assert.equal(counts.headers, 1); +}); + +test('converts the Our World in Data chart to `{iframe}`', () => { + const source = [ + '**Q2:** Gather some data', + '', + ':::{raw} html', + '', + ':::', + '', + 'After.', + '', + ].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.deepEqual(unknown, []); + assert.equal(counts.iframes, 1); + assert.equal( + text, + [ + '**Q2:** Gather some data', + '', + ':::{iframe} https://ourworldindata.org/grapher/life-expectancy-vs-gdp-per-capita', + ':width: 100%', + ':::', + '', + 'After.', + '', + ].join('\n') + ); +}); + +test('unwraps a table, keeping its colspan and rowspan', () => { + const source = [ + 'Before.', + '', + '```{raw} html', + '', + ' ', + ' ', + '\t ', + '\t', + '
Factory
Requirement
', + '```', + '', + 'After.', + '', + ].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.deepEqual(unknown, []); + assert.equal(counts.tables, 1); + assert.ok(!text.includes('{raw}')); + assert.ok(text.includes('')); + assert.ok(text.includes('colspan="3"')); + assert.ok(text.includes('rowspan="2"')); + // The HTML itself is untouched, tabs and all. + assert.ok(text.includes('\t ')); +}); + +test('a second pass changes nothing', () => { + for (const source of [headerDoc('jupyter'), headerDoc('html')]) { + const once = rewriteDocument(source).text; + const twice = rewriteDocument(once); + assert.equal(twice.text, once); + assert.deepEqual(twice.counts, { headers: 0, iframes: 0, tables: 0 }); + } +}); + +test('reports any other `{raw}` block by line, and leaves it alone', () => { + const source = ['# Title', '', '```{raw} latex', '\\newpage', '```', '', 'After.', ''].join('\n'); + const { text, unknown } = rewriteDocument(source); + assert.equal(text, source); + assert.deepEqual(unknown, [{ line: 3, reason: '{raw} latex' }]); +}); + +test('reports a block nested inside another directive at its real line', () => { + const source = [ + '# Title', + '', + '::::{exercise}', + 'Body.', + '', + ':::{raw} html', + '

something else

', + ':::', + '::::', + '', + ].join('\n'); + const { text, unknown } = rewriteDocument(source); + assert.equal(text, source); + assert.deepEqual(unknown, [{ line: 6, reason: '{raw} html' }]); +}); + +test('rewrites a header nested inside another directive', () => { + const source = ['::::{note}', ':::{raw} jupyter', HEADER_BODY, ':::', '::::', ''].join('\n'); + const { text, counts } = rewriteDocument(source); + assert.equal(counts.headers, 1); + assert.equal(text, ['::::{note}', '::::', ''].join('\n')); +}); + +test('leaves a `{raw}` block quoted inside a code fence alone', () => { + const source = [ + 'How the old sources looked:', + '', + '````markdown', + '```{raw} jupyter', + HEADER_BODY, + '```', + '````', + '', + ].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.equal(text, source); + assert.deepEqual(counts, { headers: 0, iframes: 0, tables: 0 }); + assert.deepEqual(unknown, []); +}); + +test('a document with no `{raw}` block is returned unchanged', () => { + const source = ['# Title', '', '```{note}', 'Hello.', '```', ''].join('\n'); + assert.equal(rewriteDocument(source).text, source); +}); + +test('a CRLF source is rewritten, and keeps its line endings', () => { + const source = headerDoc('jupyter').replace(/\n/g, '\r\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.deepEqual(unknown, []); + assert.equal(counts.headers, 1, 'a trailing CR must not hide the fence'); + assert.ok(!text.includes('qe-notebook-header')); + assert.ok(text.includes('\r\n')); + assert.ok(!/[^\r]\n/.test(text), 'no bare LF should be left in a CRLF file'); +}); + +test('an unhandled block in a CRLF source is still reported', () => { + const source = ['# Title', '', '```{raw} latex', '\\newpage', '```', ''].join('\r\n'); + const { unknown } = rewriteDocument(source); + assert.deepEqual(unknown, [{ line: 3, reason: '{raw} latex' }]); +}); + +test('a table gets blank lines around it, so following Markdown still parses', () => { + // An HTML block runs to the next blank line; without one the heading below + // would be swallowed into it and render as literal text. + const source = [ + 'Intro.', + '```{raw} html', + '
Requirement
A
', + '```', + '## A Real Heading', + '', + ].join('\n'); + const { text, counts } = rewriteDocument(source); + assert.equal(counts.tables, 1); + assert.equal( + text, + ['Intro.', '', '
A
', '', '## A Real Heading', ''].join('\n') + ); +}); + +test('a body that only starts with a table is reported, not unfenced', () => { + const source = [ + '```{raw} html', + '
a
', + '', + '```', + '', + ].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.equal(text, source); + assert.equal(counts.tables, 0); + assert.equal(unknown.length, 1); +}); + +test('only `html` blocks are converted; another format is reported', () => { + for (const lang of ['latex', 'tex']) { + const source = ['```{raw} ' + lang, '
a
', '```', ''].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.equal(text, source, `${lang} must not be unfenced as HTML`); + assert.equal(counts.tables, 0); + assert.deepEqual(unknown, [{ line: 1, reason: `{raw} ${lang}` }]); + } +}); + +test('a block quoted inside a code-listing directive is left alone', () => { + // Spelled as MyST spells them, with the argument after the closing brace: the + // scanner only reads a directive name from `{...}`, so an argument written + // inside the braces leaves the name unread and the fence is stepped over as + // an unnamed one -- which passes this test without ever consulting + // LISTING_DIRECTIVES. (An unnamed fence has its own test above.) + for (const directive of [ + '{code} python', + '{code-cell} ipython3', + '{code-block} md', + '{literalinclude}', + ]) { + const source = [ + '```' + directive, + ':::{raw} jupyter', + HEADER_BODY, + ':::', + '```', + '', + ].join('\n'); + const { text, counts, unknown } = rewriteDocument(source); + assert.equal(text, source, `${directive} body must not be rewritten`); + assert.deepEqual(counts, { headers: 0, iframes: 0, tables: 0 }); + assert.deepEqual(unknown, []); + } +}); + +test('a tilde code fence is stepped over like a backtick one', () => { + const source = ['~~~markdown', '```{raw} latex', '\\newpage', '```', '~~~', ''].join('\n'); + const { text, unknown } = rewriteDocument(source); + assert.equal(text, source); + assert.deepEqual(unknown, []); +});