From e6a960dcb274b30866a0e50db34dc9acae3fdab5 Mon Sep 17 00:00:00 2001 From: openhands Date: Wed, 19 Aug 2026 01:01:24 +0000 Subject: [PATCH] feat: link contents to .ipynb and expand LaTeX macros before conversion - Point the "Contents" and "Exercise Answers" tables in basic_lessons/README.md at the generated .ipynb files instead of the .md sources, so readers land on the executable notebooks. - Add convert_to_ipynb.py: a two-step .md -> intermediate.md -> .ipynb pipeline that first expands the project.math LaTeX macros (myvec/mymatrix/quat/dual) into an intermediate .md, then runs jupytext on that file. Most .ipynb renderers cannot draw custom macros, so the generated notebooks now contain only standard LaTeX and render anywhere. The intermediate file is a transient build artifact (deleted after conversion; --keep retains it). Expansion mirrors KaTeX/TeX macro semantics (braced args incl. nested braces, single space-separated tokens, no-arg macros) and leaves {code-cell} bodies untouched so Python source is never corrupted. - Wire the script into the CI "Generate downloadable notebooks" step. - Document the new pipeline in AGENTS.md. Co-authored-by: openhands --- .github/workflows/notebook_to_html.yml | 14 +- AGENTS.md | 42 +++-- basic_lessons/README.md | 22 +-- convert_to_ipynb.py | 250 +++++++++++++++++++++++++ 4 files changed, 294 insertions(+), 34 deletions(-) create mode 100644 convert_to_ipynb.py diff --git a/.github/workflows/notebook_to_html.yml b/.github/workflows/notebook_to_html.yml index 582e1b8..888befa 100644 --- a/.github/workflows/notebook_to_html.yml +++ b/.github/workflows/notebook_to_html.yml @@ -33,18 +33,16 @@ jobs: # Checks out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - # Convert basic_lessons/*.md → basic_lessons/*.ipynb so the site provides "Download notebook" buttons + # Convert basic_lessons/*.md → basic_lessons/*.ipynb so the site provides "Download notebook" buttons. + # The .md uses custom LaTeX macros (see `project.math` in myst.yml) that most .ipynb + # renderers cannot draw, so convert_to_ipynb.py first expands them into an intermediate + # .md (which is deleted afterwards) and then runs jupytext on that. # https://jupytext.readthedocs.io/ — supports md:myst format natively # Must run BEFORE build so myst/jupyter-book can pick up the generated notebooks - name: Generate downloadable notebooks run: | - pip install jupytext - for f in basic_lessons/lesson*_tutorial.md basic_lessons/lesson*_exercise_answers.md; do - [ -f "$f" ] || continue - out="${f%.md}.ipynb" - python -m jupytext --from md:myst --to notebook --output "$out" "$f" - echo "Generated: $out" - done + pip install jupytext pyyaml + python convert_to_ipynb.py # Runs a set of commands using the runner's shell # https://mystmd.org/guide/deployment-github-pages#fn-except-custom-domains diff --git a/AGENTS.md b/AGENTS.md index f279cbc..b0f778f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ |------|---------| | `basic_lessons/` | Canonical source: MyST text notebooks (`.md` with `{code-cell}` directives) — 6 tutorials + 5 exercise answer keys | | `basic_lessons/.gitignore` | Excludes generated `.ipynb` files (produced at build time) | +| `convert_to_ipynb.py` | Builds the downloadable `.ipynb` files: `.md` → intermediate `.md` (LaTeX macros expanded) → `.ipynb` | | `other/` | Supplementary content (e.g. `dqrobotics.md`) | | `myst.yml` | MyST project config (root): LaTeX macros, TOC, site options | | `build_html.sh` | Build script for `jupyter-book` | @@ -59,18 +60,24 @@ x = np.array([1, 2, 3]) ### Downloadable `.ipynb` from `.md` notebooks -The `basic_lessons/` `.md` files are the canonical source. `.ipynb` files are generated at build time so visitors can download them: +The `basic_lessons/` `.md` files are the canonical source. `.ipynb` files are generated at build time so visitors can download them. The conversion is a **two-step pipeline** run by `convert_to_ipynb.py`: -1. **CI pipeline** (`.github/workflows/notebook_to_html.yml`) runs `jupytext --from md:myst --to notebook` before the MyST build, converting each `basic_lessons/*.md` → `basic_lessons/*.ipynb`. -2. **`myst.yml` TOC** references the generated `.ipynb` for the lesson section — MyST renders these identically to the `.md` but provides native "Download notebook" buttons. -3. **`basic_lessons/.gitignore`** excludes `.ipynb` so only `.md` is tracked in git. +``` +basic_lessons/*.md -> _expanded.md (LaTeX macros expanded) -> basic_lessons/*.ipynb +``` + +1. **CI pipeline** (`.github/workflows/notebook_to_html.yml`) runs `python convert_to_ipynb.py` before the MyST build. +2. **Why the intermediate step?** The lessons use custom LaTeX macros (e.g. `\myvec{q}`) that MyST expands at build time by passing them to KaTeX. Most standalone `.ipynb` renderers (JupyterLab, VS Code, nbviewer, ...) do **not** know these macros and would show them literally. So `convert_to_ipynb.py` first expands every macro to its definition from `myst.yml` (`project.math`), writing a temporary `_expanded.md`; `jupytext` then converts that expanded file to `.ipynb`. The generated `.ipynb` therefore contains **only standard LaTeX** and renders anywhere. You keep writing the original `.md` with the convenient macros. +3. **`myst.yml` TOC** references the generated `.ipynb` for the lesson section — MyST renders these identically to the `.md` but provides native "Download notebook" buttons. +4. **`basic_lessons/.gitignore`** excludes `.ipynb` so only `.md` is tracked in git. + +The intermediate `_expanded.md` files are also build artifacts — deleted after conversion (use `--keep` to inspect them). They are not tracked and not referenced in `myst.yml`. To generate locally (e.g. for testing): ```bash -pip install jupytext -for f in basic_lessons/lesson*_tutorial.md basic_lessons/lesson*_exercise_answers.md; do - python -m jupytext --from md:myst --to notebook --output "${f%.md}.ipynb" "$f" -done +pip install jupytext pyyaml +python convert_to_ipynb.py # all lessons +python convert_to_ipynb.py --keep # keep the intermediate _expanded.md files ``` ### Image references @@ -101,10 +108,8 @@ pip install jupyter-book --pre **jupyter-book build (CI pipeline):** ```bash # Step 1: Generate .ipynb from .md (required for download buttons) -pip install jupytext -for f in basic_lessons/lesson*_tutorial.md basic_lessons/lesson*_exercise_answers.md; do - python -m jupytext --from md:myst --to notebook --output "${f%.md}.ipynb" "$f" -done +pip install jupytext pyyaml +python convert_to_ipynb.py # Step 2: Build the site chmod +x build_html.sh @@ -142,7 +147,7 @@ warning classes seen so far (all fixed in #11): ### CI/CD The GitHub Actions workflow (`.github/workflows/notebook_to_html.yml`) runs on pushes to `main` and on pull requests: -1. Generates `.ipynb` from `.md` using jupytext +1. Generates `.ipynb` from `.md` via `python convert_to_ipynb.py` (expanding LaTeX macros into an intermediate `.md` first) 2. Runs `./build_html.sh` (jupyter-book pipeline) 3. Uploads `_build/html/` as Pages artifact 4. Deploys to GitHub Pages @@ -174,6 +179,12 @@ Thank you! Please report it at https://github.com/MarinhoLab/OpenExecutableBooks - `\quat{}` for quaternions - `\dual{}` for dual numbers +These are defined in `myst.yml` under `project.math` (KaTeX `#1` substitutions) and +are expanded by MyST when building the site. Use them freely in the `.md` source — +`convert_to_ipynb.py` expands them into the downloadable `.ipynb` so the notebooks +also render in standalone viewers. **Code cells are left untouched** (a macro name +in a Python comment/variable is never expanded). + ### Image references: - Use relative paths: `![alt](Lesson4.png)` (relative to `basic_lessons/`) - Images live alongside the lesson files in `basic_lessons/`. @@ -193,6 +204,7 @@ Thank you! Please report it at https://github.com/MarinhoLab/OpenExecutableBooks ### Files excluded from version control: - `basic_lessons/*.ipynb` — Generated at build time from `.md` files (via `basic_lessons/.gitignore`) +- `basic_lessons/*_expanded.md` — transient intermediate files from `convert_to_ipynb.py` (deleted after conversion; `--keep` retains them for inspection) There is intentionally no root `.gitignore`. If you create local `venv/` or `_build/` directories, keep them out of commits (e.g. via `.git/info/exclude`). @@ -207,6 +219,6 @@ directories, keep them out of commits (e.g. via `.git/info/exclude`). 4. Use `{code-cell}` directives for Python code blocks 5. Use `%%capture` on `%pip install` cells to suppress output 6. Update `myst.yml` — add new file(s) to `project.toc` list as `.ipynb` (generated at build time) -7. Update `basic_lessons/README.md` — add the new lesson to the contents table -8. Test: `./build_html.sh` from the repository root +7. Update `basic_lessons/README.md` — add the new lesson to the contents table, linking to the generated `.ipynb` (not the `.md`) +8. Test: run `python convert_to_ipynb.py` to regenerate the `.ipynb`, then `./build_html.sh` from the repository root 9. Open PR with descriptive title and body diff --git a/basic_lessons/README.md b/basic_lessons/README.md index feab50e..7d2e6a2 100644 --- a/basic_lessons/README.md +++ b/basic_lessons/README.md @@ -15,19 +15,19 @@ The reader is expected to follow it sequentially. | Number | Title and Link | Content | |--------|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0 | [](./lesson0_tutorial.md) | Setting up the virtual environment and installing all required dependencies. | -| 1 | [](./lesson1_tutorial.md) | Basic operations in Python and `numpy` | -| 2 | [](./lesson2_tutorial.md) | Learn about elements and operations in $\mathbb{R}^n$, $SO(n)$, and $SE(n)$ with $n\in{\{2,3\}}$ related to positions, orientations, and poses, respectively. | -| 3 | [](./lesson3_tutorial.md) | Learn about the composition of rigid body motion in series to obtain the forward kinematics model of a robotic manipulator, mapping their configuration space $\myvec{q}\in\mathbb{R}^n$ into their task space $\myvec{x}\in\mathbb{R}^m$. | -| 4 | [](./lesson4_tutorial.md) | Learn about the first-order differential mapping $\dot{\myvec{x}}=\mymatrix{J}\dot{\myvec{q}}$ between joint space and task space velocities through the calculation of the Jacobian $\mymatrix{J}$. | -| 5 | [](./lesson5_tutorial.md) | Employ the previous knowledge in all previous lessons to employ a Lyapunov-stable control law to move a manipulator in task space using configuration-space signals. | +| 0 | [](./lesson0_tutorial.ipynb) | Setting up the virtual environment and installing all required dependencies. | +| 1 | [](./lesson1_tutorial.ipynb) | Basic operations in Python and `numpy` | +| 2 | [](./lesson2_tutorial.ipynb) | Learn about elements and operations in $\mathbb{R}^n$, $SO(n)$, and $SE(n)$ with $n\in{\{2,3\}}$ related to positions, orientations, and poses, respectively. | +| 3 | [](./lesson3_tutorial.ipynb) | Learn about the composition of rigid body motion in series to obtain the forward kinematics model of a robotic manipulator, mapping their configuration space $\myvec{q}\in\mathbb{R}^n$ into their task space $\myvec{x}\in\mathbb{R}^m$. | +| 4 | [](./lesson4_tutorial.ipynb) | Learn about the first-order differential mapping $\dot{\myvec{x}}=\mymatrix{J}\dot{\myvec{q}}$ between joint space and task space velocities through the calculation of the Jacobian $\mymatrix{J}$. | +| 5 | [](./lesson5_tutorial.ipynb) | Employ the previous knowledge in all previous lessons to employ a Lyapunov-stable control law to move a manipulator in task space using configuration-space signals. | ### Exercise Answers | Lesson | Link | |--------|------| -| L1 | [](./lesson1_exercise_answers.md) | -| L2 | [](./lesson2_exercise_answers.md) | -| L3 | [](./lesson3_exercise_answers.md) | -| L4 | [](./lesson4_exercise_answers.md) | -| L5 | [](./lesson5_exercise_answers.md) | \ No newline at end of file +| L1 | [](./lesson1_exercise_answers.ipynb) | +| L2 | [](./lesson2_exercise_answers.ipynb) | +| L3 | [](./lesson3_exercise_answers.ipynb) | +| L4 | [](./lesson4_exercise_answers.ipynb) | +| L5 | [](./lesson5_exercise_answers.ipynb) | \ No newline at end of file diff --git a/convert_to_ipynb.py b/convert_to_ipynb.py new file mode 100644 index 0000000..1e6f2f9 --- /dev/null +++ b/convert_to_ipynb.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python +"""Convert MyST text notebooks (.md) into downloadable Jupyter notebooks (.ipynb). + +The conversion is a two-step pipeline: + + .md -> intermediate.md -> .ipynb + +1. **Expand LaTeX macros.** The lessons use custom LaTeX macros (e.g. ``\\myvec``, + ``\\mymatrix``, ``\\quat``, ``\\dual``) whose definitions live in ``myst.yml`` + under ``project.math``. MyST expands them at build time by passing them to KaTeX, + but most ``.ipynb`` renderers (JupyterLab, VS Code, nbviewer, ...) do not know + these macros, so they would render literally. The intermediate ``.md`` therefore + has every macro expanded inline to its definition, so the resulting ``.ipynb`` + only ever contains standard LaTeX that any renderer can draw. +2. **Convert to notebook.** ``jupytext`` reads the intermediate ``.md`` + (``md:myst`` format) and writes the standard ``.ipynb``. + +The intermediate file is written next to the source, named ``_expanded.md`` +(e.g. ``lesson1_tutorial_expanded.md``). It is a build artifact: it is not tracked +in git, is not referenced in ``myst.yml``, and is not listed in +``basic_lessons``. You only ever edit the original ``.md`` (which keeps the +convenient macros); the intermediate and ``.ipynb`` are regenerated on every build. + +Usage: + python convert_to_ipynb.py # convert every lesson in basic_lessons/ + python convert_to_ipynb.py --keep # also keep the intermediate .md files + python convert_to_ipynb.py path/to/file.md # convert a single file +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +import yaml + +# Matches a MyST code-cell directive fence, e.g. ````{code-cell}`` or ````{code-cell} python`. +CODE_CELL_RE = re.compile(r"^\s*````\s*\{code-cell\}[^`]*$") + +DEFAULT_GLOB = "basic_lessons/lesson*.md" + + +def load_macros(myst_path: Path) -> dict[str, str]: + """Read the ``project.math`` macro definitions from ``myst.yml``. + + Returns a dict mapping the macro *name* (without the leading backslash, e.g. + ``"myvec"``) to its LaTeX definition (e.g. ``"\\mathbf{\\boldsymbol{ #1 }}"``). + """ + with open(myst_path, "r", encoding="utf-8") as fh: + config = yaml.safe_load(fh) or {} + math_macros = (config.get("project") or {}).get("math") or {} + macros: dict[str, str] = {} + for key, value in math_macros.items(): + name = key.lstrip("\\") + macros[name] = value + return macros + + +def _brace_group(text: str, pos: int) -> tuple[str, int] | None: + """Read a balanced ``{...}`` group starting at ``text[pos] == '{'``. + + Returns the *inner* content and the index just past the closing brace, or + ``None`` if the group is not balanced. Handles nested braces, e.g. ``{H^{-1}}``. + """ + if pos >= len(text) or text[pos] != "{": + return None + depth = 0 + for i in range(pos, len(text)): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + return text[pos + 1 : i], i + 1 + return None + + +def _expand_at(text: str, pos: int, macros: dict[str, str]) -> tuple[str, int] | None: + """If a macro call starts at ``text[pos]`` (which must be ``\\``), expand it. + + Returns ``(expansion, index_just_past_the_call)`` or ``None`` if no macro is + called at ``pos``. Mirrors KaTeX/TeX macro semantics for the argument forms the + lessons use (see the ``#1`` definitions in ``myst.yml``): + + * ``\\name{arg}`` -- braced argument, braces may be nested. + * ``\\name arg`` -- a single space-separated token (e.g. ``\\myvec H`` grabs + the ``H`` and leaves any following ``_{...}^{...}`` intact). + * ``\\name`` -- no argument (the definition has no ``#1``). + + The macro name must not be followed by a letter (so ``\\dual`` never matches + ``\\dualfoo``). Like TeX, the macro is treated as a control sequence no matter + what token precedes it (e.g. ``\\mymatrix{B}_2\\mymatrix{A}_2``). + """ + if pos >= len(text) or text[pos] != "\\": + return None + for name in sorted(macros, key=len, reverse=True): + start = pos + 1 + end = start + len(name) + if text[start:end] != name: + continue + after = text[end] if end < len(text) else "" + if after.isalpha(): + continue # a longer command that merely shares this prefix + if "#" not in macros[name]: + # No-argument macro. + return macros[name], end + if after == "{": + group = _brace_group(text, end) + if group is None: + return None + arg, next_pos = group + return macros[name].replace("#1", arg.strip()), next_pos + if after == " ": + # Single-token argument: take the next ``{...}`` group or one char. + j = end + 1 + if j < len(text) and text[j] == "{": + group = _brace_group(text, j) + if group is None: + return None + arg, next_pos = group + elif j < len(text) and not text[j].isspace(): + arg, next_pos = text[j], j + 1 + else: + return None + return macros[name].replace("#1", arg.strip()), next_pos + return None + + +def expand_macros(text: str, macros: dict[str, str]) -> str: + """Expand every LaTeX macro in *text* to its definition. + + Scans left-to-right and never re-scans replaced text, so the process is + idempotent and terminates. See :func:`_expand_at` for the supported forms. + """ + if not macros: + return text + out: list[str] = [] + pos = 0 + while pos < len(text): + if text[pos] == "\\": + hit = _expand_at(text, pos, macros) + if hit is not None: + expansion, next_pos = hit + out.append(expansion) + pos = next_pos + continue + out.append(text[pos]) + pos += 1 + return "".join(out) + + +def expand_macros_outside_code_cells(text: str, macros: dict[str, str]) -> str: + """Expand LaTeX macros in the prose, leaving ``{code-cell}`` bodies untouched. + + Macros are only ever used in the surrounding prose (LaTeX math). If a macro name + ever appears inside a code cell (e.g. a variable named ``quat``), expanding it + would corrupt the Python source, so code-cell bodies are copied through verbatim. + """ + lines = text.splitlines(keepends=True) + out: list[str] = [] + in_code_cell = False + for line in lines: + if not in_code_cell and CODE_CELL_RE.match(line): + in_code_cell = True + out.append(line) + continue + if in_code_cell and line.strip() == "````": + in_code_cell = False + out.append(line) + continue + if in_code_cell: + out.append(line) + else: + out.append(expand_macros(line, macros)) + return "".join(out) + + +def convert(md_path: Path, myst_path: Path, keep_intermediate: bool) -> Path: + """Run the ``.md -> intermediate.md -> .ipynb`` pipeline for one file.""" + import jupytext # imported lazily so --help works without it installed + + text = md_path.read_text(encoding="utf-8") + macros = load_macros(myst_path) + expanded = expand_macros_outside_code_cells(text, macros) + + intermediate = md_path.with_name(md_path.stem + "_expanded.md") + intermediate.write_text(expanded, encoding="utf-8") + try: + nb = jupytext.read(str(intermediate), fmt="md:myst") + ipynb_path = md_path.with_suffix(".ipynb") + jupytext.write(nb, str(ipynb_path), fmt="ipynb") + finally: + if not keep_intermediate: + intermediate.unlink(missing_ok=True) + + return ipynb_path + + +def main(argv: list[str] | None = None) -> int: + repo_root = Path(__file__).resolve().parent + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "paths", + nargs="*", + help="Specific .md files to convert. If omitted, every file matching --glob is used.", + ) + parser.add_argument( + "--glob", + default=DEFAULT_GLOB, + help="Glob (relative to the repo root) for lesson files when no explicit paths are given.", + ) + parser.add_argument( + "--myst", + default=str(repo_root / "myst.yml"), + help="Path to myst.yml (source of the project.math macro definitions).", + ) + parser.add_argument( + "--keep", + action="store_true", + help="Keep the intermediate _expanded.md files instead of deleting them.", + ) + args = parser.parse_args(argv) + + myst_path = Path(args.myst) + if not myst_path.is_file(): + print(f"error: {myst_path} not found", file=sys.stderr) + return 1 + + if args.paths: + md_files = [Path(p) for p in args.paths] + else: + md_files = sorted(repo_root.glob(args.glob)) + if not md_files: + print(f"error: no .md files matched {args.paths or args.glob}", file=sys.stderr) + return 1 + + for md_path in md_files: + if not md_path.is_file(): + print(f"skip: {md_path} not found", file=sys.stderr) + continue + ipynb_path = convert(md_path, myst_path, keep_intermediate=args.keep) + print(f"generated: {ipynb_path} (from {md_path})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())