diff --git a/README.md b/README.md index 801451f..59aa0df 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,6 @@ Each chapter is an ablation: from bandits through PPO and SAC, then offline RL, BreakRL is a bilingual, failure-first RL textbook (English default, Chinese edition). Each chapter explains a mechanism, then removes it so you can see the failure. Chapters 12–14 use toy sequence tasks and two-digit addition; they are not a substitute for production RLHF, DPO, or GRPO training. -**Current textbook:** v1.2.x on `main`. A V2 rewrite is an unstarted internal spec; it will not replace or empty this tree. -

Minimum demo: at low data coverage, TD loss falls while actual return collapses

diff --git a/README.zh.md b/README.zh.md index 537e3dd..e37de1b 100644 --- a/README.zh.md +++ b/README.zh.md @@ -27,8 +27,6 @@ BreakRL 是一本双语、失败优先的强化学习教材（英文为默认语言，另有中文版）。每章先讲机制，再去掉它，让你看见失败。第 12–14 章用玩具序列和两位数加法讲机制，不能替代真实的 RLHF、DPO 或 GRPO 训练。 -**当前教材：** `main` 上的 v1.2.x。V2 仍是未开工的内部规格，不会替换或清空本仓库。 -

最小演示：低数据覆盖率时 TD loss 下降，实际回报塌缩

diff --git a/book/_config.yml b/book/_config.yml index 9149bba..d3e5321 100644 --- a/book/_config.yml +++ b/book/_config.yml @@ -31,7 +31,6 @@ sphinx: language: en exclude_patterns: - ".github/**" - - "PROJECT_AUDIT.md" html_css_files: - lang-toggle.css - breakrl.css diff --git a/book/_static/breakrl.js b/book/_static/breakrl.js index 8b711bb..34f5ec1 100644 --- a/book/_static/breakrl.js +++ b/book/_static/breakrl.js @@ -1,36 +1,7 @@ (function () { 'use strict'; - function pagename() { - if (window.DOCUMENTATION_OPTIONS && DOCUMENTATION_OPTIONS.pagename) { - return String(DOCUMENTATION_OPTIONS.pagename); - } - var path = String(window.location.pathname || ''); - var trimmed = path.replace(/\/+$/, ''); - if (!trimmed || /(?:^|\/)index\.html$/.test(trimmed) || /\/$/.test(path)) { - return 'index'; - } - var marker = '/BreakRL/'; - var start = trimmed.indexOf(marker); - var relative = start >= 0 ? trimmed.slice(start + marker.length) : trimmed.replace(/^.*\//, ''); - return relative.replace(/\.html$/, ''); - } - - function setSurface() { - var page = pagename(); - var home = page === 'index' || page === 'index-zh'; - var demo = page === 'demo'; - var atlas = page === 'failure-atlas' || page === 'failure-atlas-en'; - var pdf = page === 'offline-rl-text' || page === 'offline-rl-text-en'; - document.documentElement.classList.toggle('breakrl-home', home); - document.documentElement.classList.toggle('breakrl-demo-page', demo); - document.documentElement.classList.toggle('breakrl-atlas', atlas); - document.documentElement.classList.toggle('breakrl-reader', pdf); - document.body.classList.toggle('breakrl-home', home); - document.body.classList.toggle('breakrl-demo-page', demo); - document.body.classList.toggle('breakrl-atlas', atlas); - document.body.classList.toggle('breakrl-reader', pdf); - } + var CHINESE_FOOTER = 'BreakRL 是一本双语、失败优先的强化学习教材。正文 CC BY 4.0，代码 MIT。'; function brandHref() { var brand = document.querySelector('a.navbar-brand'); @@ -77,6 +48,20 @@ } } + // Give an element its Chinese label while remembering what the page itself + // says, so switching back to English restores the text `_toc.yml` or + // `_config.yml` holds instead of a copy kept here. + function swapToChinese(selector, chinese, lang) { + document.querySelectorAll(selector).forEach(function (node) { + var original = node.getAttribute('data-breakrl-original'); + if (original === null) { + original = (node.textContent || '').trim(); + node.setAttribute('data-breakrl-original', original); + } + node.textContent = lang === 'zh' ? chinese : original; + }); + } + function relabelToc() { var lang = document.documentElement.getAttribute('data-breakrl-lang') || 'en'; document.querySelectorAll('.bd-sidebar p.caption .caption-text').forEach(function (node) { @@ -84,38 +69,20 @@ if (!node.getAttribute('data-breakrl-caption')) { node.setAttribute('data-breakrl-caption', original); } - if (original === '从这里开始 / Start here' || original === 'Start here') { + if (original === 'Start here') { node.textContent = lang === 'zh' ? '从这里开始' : 'Start here'; - } else if (original === 'English' || original === 'Chapters') { - node.textContent = 'Chapters'; - } else if (original === '中文版' || original === '章节') { + } else if (original === '中文版') { node.textContent = lang === 'zh' ? '章节' : '中文版'; } }); - document.querySelectorAll('.bd-sidebar a[href$="demo.html"]').forEach(function (node) { - node.textContent = lang === 'zh' ? '最小演示' : 'Minimum demo'; - }); - document.querySelectorAll('.breakrl-footer').forEach(function (node) { - node.textContent = lang === 'zh' - ? 'BreakRL 是一本双语、失败优先的强化学习教材。正文 CC BY 4.0，代码 MIT。' - : 'BreakRL is a bilingual, failure-first RL textbook. Text CC BY 4.0. Code MIT.'; - }); - } - - function keepHomepageVisible() { - var article = document.getElementById('breakrl'); - var english = document.getElementById('breakrl-english'); - if (article && !english && article.hidden) { - article.hidden = false; - } + swapToChinese('.bd-sidebar a[href$="demo.html"]', '最小演示', lang); + swapToChinese('.breakrl-footer', CHINESE_FOOTER, lang); } function init() { - setSurface(); useTextBrand(); installHeaderBrand(); relabelToc(); - keepHomepageVisible(); } if (document.readyState === 'loading') { diff --git a/book/_static/lang-toggle.js b/book/_static/lang-toggle.js index 37475c4..ba37af2 100644 --- a/book/_static/lang-toggle.js +++ b/book/_static/lang-toggle.js @@ -5,129 +5,27 @@ var LANG_ZH = 'zh'; var LANG_EN = 'en'; - // Only book pages have language counterparts. Shared utility pages stay put. - var PAGE_MAP = { - 'index.html': { zh: 'index-zh.html', en: 'index.html' }, - 'index-zh.html': { zh: 'index-zh.html', en: 'index.html' }, - 'search.html': { zh: 'search.html', en: 'search.html' }, - 'genindex.html': { zh: 'genindex.html', en: 'genindex.html' }, - 'failure-atlas.html': { zh: 'failure-atlas.html', en: 'failure-atlas-en.html' }, - 'failure-atlas-en.html': { zh: 'failure-atlas.html', en: 'failure-atlas-en.html' }, - 'offline-rl-text.html': { zh: 'offline-rl-text.html', en: 'offline-rl-text-en.html' }, - 'offline-rl-text-en.html': { zh: 'offline-rl-text.html', en: 'offline-rl-text-en.html' }, - 'notes/multi-armed-bandit/multi-armed-bandit_experiments.html': { - zh: 'notes/multi-armed-bandit/multi-armed-bandit_experiments.html', - en: 'notes/multi-armed-bandit/multi-armed-bandit_experiments_en.html' - }, - 'notes/multi-armed-bandit/multi-armed-bandit_experiments_en.html': { - zh: 'notes/multi-armed-bandit/multi-armed-bandit_experiments.html', - en: 'notes/multi-armed-bandit/multi-armed-bandit_experiments_en.html' - }, - 'notes/mdp/mdp_experiments.html': { - zh: 'notes/mdp/mdp_experiments.html', - en: 'notes/mdp/mdp_experiments_en.html' - }, - 'notes/mdp/mdp_experiments_en.html': { - zh: 'notes/mdp/mdp_experiments.html', - en: 'notes/mdp/mdp_experiments_en.html' - }, - 'notes/temporal-difference-learning/temporal-difference-learning_experiments.html': { - zh: 'notes/temporal-difference-learning/temporal-difference-learning_experiments.html', - en: 'notes/temporal-difference-learning/temporal-difference-learning_experiments_en.html' - }, - 'notes/temporal-difference-learning/temporal-difference-learning_experiments_en.html': { - zh: 'notes/temporal-difference-learning/temporal-difference-learning_experiments.html', - en: 'notes/temporal-difference-learning/temporal-difference-learning_experiments_en.html' - }, - 'notes/dqn/dqn_experiments.html': { - zh: 'notes/dqn/dqn_experiments.html', - en: 'notes/dqn/dqn_experiments_en.html' - }, - 'notes/dqn/dqn_experiments_en.html': { - zh: 'notes/dqn/dqn_experiments.html', - en: 'notes/dqn/dqn_experiments_en.html' - }, - 'notes/policy-gradient/pg_experiments.html': { - zh: 'notes/policy-gradient/pg_experiments.html', - en: 'notes/policy-gradient/pg_experiments_en.html' - }, - 'notes/policy-gradient/pg_experiments_en.html': { - zh: 'notes/policy-gradient/pg_experiments.html', - en: 'notes/policy-gradient/pg_experiments_en.html' - }, - 'notes/actor-critic/ac_experiments.html': { - zh: 'notes/actor-critic/ac_experiments.html', - en: 'notes/actor-critic/ac_experiments_en.html' - }, - 'notes/actor-critic/ac_experiments_en.html': { - zh: 'notes/actor-critic/ac_experiments.html', - en: 'notes/actor-critic/ac_experiments_en.html' - }, - 'notes/ppo/ppo_experiments.html': { - zh: 'notes/ppo/ppo_experiments.html', - en: 'notes/ppo/ppo_experiments_en.html' - }, - 'notes/ppo/ppo_experiments_en.html': { - zh: 'notes/ppo/ppo_experiments.html', - en: 'notes/ppo/ppo_experiments_en.html' - }, - 'notes/sac/sac_experiments.html': { - zh: 'notes/sac/sac_experiments.html', - en: 'notes/sac/sac_experiments_en.html' - }, - 'notes/sac/sac_experiments_en.html': { - zh: 'notes/sac/sac_experiments.html', - en: 'notes/sac/sac_experiments_en.html' - }, - 'notes/offline-rl/offline-rl_experiments.html': { - zh: 'notes/offline-rl/offline-rl_experiments.html', - en: 'notes/offline-rl/offline-rl_experiments_en.html' - }, - 'notes/offline-rl/offline-rl_experiments_en.html': { - zh: 'notes/offline-rl/offline-rl_experiments.html', - en: 'notes/offline-rl/offline-rl_experiments_en.html' - }, - 'notes/model-based-rl/model-based-rl_experiments.html': { - zh: 'notes/model-based-rl/model-based-rl_experiments.html', - en: 'notes/model-based-rl/model-based-rl_experiments_en.html' - }, - 'notes/model-based-rl/model-based-rl_experiments_en.html': { - zh: 'notes/model-based-rl/model-based-rl_experiments.html', - en: 'notes/model-based-rl/model-based-rl_experiments_en.html' - }, - 'notes/decision-transformer/decision-transformer_experiments.html': { - zh: 'notes/decision-transformer/decision-transformer_experiments.html', - en: 'notes/decision-transformer/decision-transformer_experiments_en.html' - }, - 'notes/decision-transformer/decision-transformer_experiments_en.html': { - zh: 'notes/decision-transformer/decision-transformer_experiments.html', - en: 'notes/decision-transformer/decision-transformer_experiments_en.html' - }, - 'notes/rlhf/rlhf_experiments.html': { - zh: 'notes/rlhf/rlhf_experiments.html', - en: 'notes/rlhf/rlhf_experiments_en.html' - }, - 'notes/rlhf/rlhf_experiments_en.html': { - zh: 'notes/rlhf/rlhf_experiments.html', - en: 'notes/rlhf/rlhf_experiments_en.html' - }, - 'notes/dpo/dpo_experiments.html': { - zh: 'notes/dpo/dpo_experiments.html', - en: 'notes/dpo/dpo_experiments_en.html' - }, - 'notes/dpo/dpo_experiments_en.html': { - zh: 'notes/dpo/dpo_experiments.html', - en: 'notes/dpo/dpo_experiments_en.html' - }, - 'notes/grpo/grpo_experiments.html': { - zh: 'notes/grpo/grpo_experiments.html', - en: 'notes/grpo/grpo_experiments_en.html' - }, - 'notes/grpo/grpo_experiments_en.html': { - zh: 'notes/grpo/grpo_experiments.html', - en: 'notes/grpo/grpo_experiments_en.html' - } - }; + // Pages pair by filename. Chapter notebooks follow the convention + // `scripts/check_consistency.py` enforces — the English edition of + // `notes//_experiments.html` is its `_en` sibling — so a new + // chapter's toggle works without an edit here. The site's own text pages are + // listed instead, because their names do not say which edition they are. + // Sphinx's search and genindex pages, and the single-language demo, pair with + // nothing and stay put. + var TEXT_PAGES = indexPairs([ + ['index-zh.html', 'index.html'], + ['failure-atlas.html', 'failure-atlas-en.html'], + ['offline-rl-text.html', 'offline-rl-text-en.html'] + ]); + + function indexPairs(pairs) { + var index = {}; + pairs.forEach(function (pair) { + index[pair[0]] = { zh: pair[0], en: pair[1] }; + index[pair[1]] = { zh: pair[0], en: pair[1] }; + }); + return index; + } var UI_LABELS = { zh: { @@ -164,46 +62,41 @@ var originalTitle = null; - function pageKey(pathname) { + // A pathname split into the directory that holds the page and the page's own + // filename. A URL ending in `/` serves that directory's index page. + function pageLocation(pathname) { var path = String(pathname || '').split(/[?#]/)[0]; - var trimmed = path.replace(/\/+$/, ''); - if (!trimmed || /(?:^|\/)index\.html$/.test(trimmed) || /\/$/.test(path)) { - return 'index.html'; - } - var keys = Object.keys(PAGE_MAP); - for (var i = 0; i < keys.length; i += 1) { - var key = keys[i]; - if (trimmed === key || trimmed.slice(-(key.length + 1)) === '/' + key) { - return key; - } - } - return null; + var cut = path.lastIndexOf('/') + 1; + return { directory: path.slice(0, cut), name: path.slice(cut) || 'index.html' }; } - function parallelPath(pathname, lang) { - var key = pageKey(pathname); - var page = key ? PAGE_MAP[key] : null; - if (!page || !page[lang] || page[lang] === key) { + function pagePair(location) { + if (TEXT_PAGES[location.name]) { + return TEXT_PAGES[location.name]; + } + if (!/(^|\/)notes\//.test(location.directory)) { return null; } - var path = String(pathname || '').split(/[?#]/)[0]; - if (path === key) { - return page[lang]; + if (/_experiments_en\.html$/.test(location.name)) { + return { zh: location.name.replace(/_en\.html$/, '.html'), en: location.name }; } - if (key === 'index.html') { - if (/\/$/.test(path)) { - return path + page[lang]; - } - if (/(?:^|\/)index\.html$/.test(path)) { - return path.replace(/index\.html$/, page[lang]); - } + if (/_experiments\.html$/.test(location.name)) { + return { zh: location.name, en: location.name.replace(/\.html$/, '_en.html') }; } - var marker = '/' + key; - var start = path.lastIndexOf(marker); - if (start < 0) { + return null; + } + + function pageName(pathname) { + return pageLocation(pathname).name; + } + + function parallelPath(pathname, lang) { + var location = pageLocation(pathname); + var pair = pagePair(location); + if (!pair || !pair[lang] || location.name === pair[lang]) { return null; } - return path.slice(0, start + 1) + page[lang]; + return location.directory + pair[lang]; } function parallelTarget(lang) { @@ -214,13 +107,12 @@ return target + (window.location.search || '') + (window.location.hash || ''); } - function pageLanguage(pathname) { - var key = pageKey(pathname); - var page = key ? PAGE_MAP[key] : null; - if (!page || page.zh === page.en) { - return null; - } - return page.en === key ? LANG_EN : LANG_ZH; + // The build states each page's own language in the HTML it generates + // (`scripts/breakrl_locale.py` owns that rule), so the toggle reads it rather + // than deriving the same answer a second time. + function pageLanguage() { + var lang = document.documentElement.getAttribute('lang') || ''; + return /^zh\b/i.test(lang) ? LANG_ZH : LANG_EN; } function storedLang() { @@ -231,10 +123,6 @@ return value === LANG_EN || value === LANG_ZH ? value : null; } - function currentLang() { - return pageLanguage(window.location.pathname) || storedLang() || LANG_EN; - } - function setDirectText(selector, text) { document.querySelectorAll(selector).forEach(function (node) { var textNode = null; @@ -292,13 +180,13 @@ } }); - var key = pageKey(window.location.pathname); + var name = pageName(window.location.pathname); if (originalTitle === null) { originalTitle = document.title; } - if (lang === LANG_EN && key === 'search.html') { + if (lang === LANG_EN && name === 'search.html') { document.title = 'Search - BreakRL'; - } else if (lang === LANG_EN && key === 'genindex.html') { + } else if (lang === LANG_EN && name === 'genindex.html') { document.title = 'Index - BreakRL'; } else { document.title = originalTitle; @@ -350,7 +238,6 @@ }); setUiLanguage(lang); filterToc(lang); - filterIndex(lang); } function filterToc(lang) { @@ -373,34 +260,6 @@ }); } - function filterIndex(lang) { - var cn = document.getElementById('breakrl'); - var en = document.getElementById('breakrl-english'); - // Split homepages only have one of these ids. Hiding #breakrl would - // conceal the entire English (or Chinese) article body. - if (!cn || !en) { - return; - } - cn.hidden = lang === LANG_EN; - en.hidden = lang === LANG_ZH; - - document.querySelectorAll('.bd-sidebar-secondary a').forEach(function (link) { - var href = link.getAttribute('href'); - var isEn = href === '#' || href === '#breakrl-english'; - var isZh = href === '#breakrl'; - if (!isZh && !isEn) { - return; - } - var entry = link.parentElement; - while (entry && entry.tagName !== 'LI') { - entry = entry.parentElement; - } - if (entry) { - entry.hidden = (lang === LANG_EN && isZh) || (lang === LANG_ZH && isEn); - } - }); - } - function applyLang(lang) { if (lang !== LANG_EN && lang !== LANG_ZH) { return; @@ -429,9 +288,9 @@ } function init() { + var pageLang = pageLanguage(); var saved = storedLang(); - var pageLang = pageLanguage(window.location.pathname); - if (saved && pageLang && saved !== pageLang) { + if (saved && saved !== pageLang) { var target = parallelTarget(saved); if (target) { window.location.replace(target); @@ -455,15 +314,13 @@ } } hosts.forEach(installButton); - setLangState(saved || pageLang || LANG_EN); + setLangState(saved || pageLang); } window.BreakRLLanguageToggle = { - pageKey: pageKey, - pageLanguage: pageLanguage, + textPages: TEXT_PAGES, parallelPath: parallelPath, - parallelTarget: parallelTarget, - pageMap: PAGE_MAP + parallelTarget: parallelTarget }; if (document.readyState === 'loading') { diff --git a/book/index-zh.md b/book/index-zh.md index 0b7f170..e5ff3aa 100644 --- a/book/index-zh.md +++ b/book/index-zh.md @@ -4,8 +4,6 @@ BreakRL 是一本双语、失败优先的强化学习教材：每章先讲机制，再去掉它，让你看见失败。 -**当前教材：** v1.2.x。V2 仍是未开工的内部规格，不会替换或清空 `main`。 - ## 三分钟开始 1. 打开[最小演示](demo)，拖动覆盖率滑块（教学示意，不训练模型）； @@ -26,20 +24,20 @@ BreakRL 是一本双语、失败优先的强化学习教材：每章先讲机制 | 章节 | 正文 | 实验 | 运行 | | --- | --- | --- | --- | -| 1. 多臂老虎机：探索与利用 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit.pdf) | [在线阅读](notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb) | -| 2. 马尔可夫决策过程 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp.pdf) | [在线阅读](notes/mdp/mdp_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp_experiments.ipynb) | -| 3. 时序差分学习 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning.pdf) | [在线阅读](notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb) | -| 4. DQN：神经网络价值学习 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn.pdf) | [在线阅读](notes/dqn/dqn_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn_experiments.ipynb) | -| 5. Policy Gradient / REINFORCE | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg.pdf) | [在线阅读](notes/policy-gradient/pg_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg_experiments.ipynb) | -| 6. Actor-Critic / A2C | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac.pdf) | [在线阅读](notes/actor-critic/ac_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac_experiments.ipynb) | -| 7. PPO：约束策略更新 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo.pdf) | [在线阅读](notes/ppo/ppo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo_experiments.ipynb) | -| 8. SAC：最大熵连续控制 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac.pdf) | [在线阅读](notes/sac/sac_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac_experiments.ipynb) | +| 1. 多臂老虎机：探索与利用 | [PDF](notes/multi-armed-bandit/multi-armed-bandit.pdf) | [在线阅读](notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb) | +| 2. 马尔可夫决策过程 | [PDF](notes/mdp/mdp.pdf) | [在线阅读](notes/mdp/mdp_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp_experiments.ipynb) | +| 3. 时序差分学习 | [PDF](notes/temporal-difference-learning/temporal-difference-learning.pdf) | [在线阅读](notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb) | +| 4. DQN：神经网络价值学习 | [PDF](notes/dqn/dqn.pdf) | [在线阅读](notes/dqn/dqn_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn_experiments.ipynb) | +| 5. Policy Gradient / REINFORCE | [PDF](notes/policy-gradient/pg.pdf) | [在线阅读](notes/policy-gradient/pg_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg_experiments.ipynb) | +| 6. Actor-Critic / A2C | [PDF](notes/actor-critic/ac.pdf) | [在线阅读](notes/actor-critic/ac_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac_experiments.ipynb) | +| 7. PPO：约束策略更新 | [PDF](notes/ppo/ppo.pdf) | [在线阅读](notes/ppo/ppo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo_experiments.ipynb) | +| 8. SAC：最大熵连续控制 | [PDF](notes/sac/sac.pdf) | [在线阅读](notes/sac/sac_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac_experiments.ipynb) | | 9. 离线强化学习：CQL 与 IQL | [PDF](offline-rl-text) | [在线阅读](notes/offline-rl/offline-rl_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/offline-rl/offline-rl_experiments.ipynb) | -| 10. 模型式强化学习：Dyna-Q | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl.pdf) | [在线阅读](notes/model-based-rl/model-based-rl_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl_experiments.ipynb) | -| 11. Decision Transformer | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer.pdf) | [在线阅读](notes/decision-transformer/decision-transformer_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer_experiments.ipynb) | -| 12. RLHF：从偏好到奖励 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf.pdf) | [在线阅读](notes/rlhf/rlhf_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf_experiments.ipynb) | -| 13. DPO：不训奖励模型的偏好优化 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo.pdf) | [在线阅读](notes/dpo/dpo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo_experiments.ipynb) | -| 14. GRPO 与 RLVR：可验证奖励 | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo.pdf) | [在线阅读](notes/grpo/grpo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo_experiments.ipynb) | +| 10. 模型式强化学习：Dyna-Q | [PDF](notes/model-based-rl/model-based-rl.pdf) | [在线阅读](notes/model-based-rl/model-based-rl_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl_experiments.ipynb) | +| 11. Decision Transformer | [PDF](notes/decision-transformer/decision-transformer.pdf) | [在线阅读](notes/decision-transformer/decision-transformer_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer_experiments.ipynb) | +| 12. RLHF：从偏好到奖励 | [PDF](notes/rlhf/rlhf.pdf) | [在线阅读](notes/rlhf/rlhf_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf_experiments.ipynb) | +| 13. DPO：不训奖励模型的偏好优化 | [PDF](notes/dpo/dpo.pdf) | [在线阅读](notes/dpo/dpo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo_experiments.ipynb) | +| 14. GRPO 与 RLVR：可验证奖励 | [PDF](notes/grpo/grpo.pdf) | [在线阅读](notes/grpo/grpo_experiments.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo_experiments.ipynb) | 第 12–14 章用玩具序列和两位数加法讲机制，不能替代真实的 RLHF、DPO 或 GRPO 训练。 diff --git a/book/index.md b/book/index.md index c864602..326e1df 100644 --- a/book/index.md +++ b/book/index.md @@ -4,8 +4,6 @@ BreakRL is a bilingual, failure-first textbook. Each chapter explains a mechanism, then removes it so you can see the failure. -**Current textbook:** v1.2.x. A V2 rewrite is an unstarted internal spec; it will not replace or empty `main`. - ## Start in three minutes 1. Open the [minimum demo](demo) and drag the coverage slider (a teaching plot; it does not train a model); @@ -26,20 +24,20 @@ When training does not converge, look up the symptom in the [RL Failure Atlas](f | Chapter | Text | Experiments | Run | | --- | --- | --- | --- | -| 1. Multi-armed bandits | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit_en.pdf) | [Read online](notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb) | -| 2. Markov decision processes | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp_en.pdf) | [Read online](notes/mdp/mdp_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp_experiments_en.ipynb) | -| 3. Temporal-difference learning | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning_en.pdf) | [Read online](notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb) | -| 4. DQN: neural value learning | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn_en.pdf) | [Read online](notes/dqn/dqn_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn_experiments_en.ipynb) | -| 5. Policy gradient / REINFORCE | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg_en.pdf) | [Read online](notes/policy-gradient/pg_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg_experiments_en.ipynb) | -| 6. Actor-Critic / A2C | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac_en.pdf) | [Read online](notes/actor-critic/ac_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac_experiments_en.ipynb) | -| 7. PPO: constrained policy updates | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo_en.pdf) | [Read online](notes/ppo/ppo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo_experiments_en.ipynb) | -| 8. SAC: maximum-entropy continuous control | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac_en.pdf) | [Read online](notes/sac/sac_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac_experiments_en.ipynb) | +| 1. Multi-armed bandits | [PDF](notes/multi-armed-bandit/multi-armed-bandit_en.pdf) | [Read online](notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb) | +| 2. Markov decision processes | [PDF](notes/mdp/mdp_en.pdf) | [Read online](notes/mdp/mdp_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/mdp/mdp_experiments_en.ipynb) | +| 3. Temporal-difference learning | [PDF](notes/temporal-difference-learning/temporal-difference-learning_en.pdf) | [Read online](notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb) | +| 4. DQN: neural value learning | [PDF](notes/dqn/dqn_en.pdf) | [Read online](notes/dqn/dqn_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dqn/dqn_experiments_en.ipynb) | +| 5. Policy gradient / REINFORCE | [PDF](notes/policy-gradient/pg_en.pdf) | [Read online](notes/policy-gradient/pg_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/policy-gradient/pg_experiments_en.ipynb) | +| 6. Actor-Critic / A2C | [PDF](notes/actor-critic/ac_en.pdf) | [Read online](notes/actor-critic/ac_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/actor-critic/ac_experiments_en.ipynb) | +| 7. PPO: constrained policy updates | [PDF](notes/ppo/ppo_en.pdf) | [Read online](notes/ppo/ppo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/ppo/ppo_experiments_en.ipynb) | +| 8. SAC: maximum-entropy continuous control | [PDF](notes/sac/sac_en.pdf) | [Read online](notes/sac/sac_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/sac/sac_experiments_en.ipynb) | | 9. Offline RL: CQL and IQL | [PDF](offline-rl-text-en) | [Read online](notes/offline-rl/offline-rl_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/offline-rl/offline-rl_experiments_en.ipynb) | -| 10. Model-based RL: Dyna-Q | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl_en.pdf) | [Read online](notes/model-based-rl/model-based-rl_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb) | -| 11. Decision Transformer | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer_en.pdf) | [Read online](notes/decision-transformer/decision-transformer_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb) | -| 12. RLHF: from preferences to rewards | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf_en.pdf) | [Read online](notes/rlhf/rlhf_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf_experiments_en.ipynb) | -| 13. DPO: preference optimization without a reward model | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo_en.pdf) | [Read online](notes/dpo/dpo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo_experiments_en.ipynb) | -| 14. GRPO and RLVR: verifiable rewards | [PDF](https://github.com/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo_en.pdf) | [Read online](notes/grpo/grpo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo_experiments_en.ipynb) | +| 10. Model-based RL: Dyna-Q | [PDF](notes/model-based-rl/model-based-rl_en.pdf) | [Read online](notes/model-based-rl/model-based-rl_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb) | +| 11. Decision Transformer | [PDF](notes/decision-transformer/decision-transformer_en.pdf) | [Read online](notes/decision-transformer/decision-transformer_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb) | +| 12. RLHF: from preferences to rewards | [PDF](notes/rlhf/rlhf_en.pdf) | [Read online](notes/rlhf/rlhf_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/rlhf/rlhf_experiments_en.ipynb) | +| 13. DPO: preference optimization without a reward model | [PDF](notes/dpo/dpo_en.pdf) | [Read online](notes/dpo/dpo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/dpo/dpo_experiments_en.ipynb) | +| 14. GRPO and RLVR: verifiable rewards | [PDF](notes/grpo/grpo_en.pdf) | [Read online](notes/grpo/grpo_experiments_en.ipynb) | [Colab](https://colab.research.google.com/github/Powfu-zwx/BreakRL/blob/main/book/notes/grpo/grpo_experiments_en.ipynb) | Chapters 12–14 use toy sequence tasks and two-digit addition. They show the mechanisms; they are not a substitute for production RLHF, DPO, or GRPO training. diff --git a/book/notes/actor-critic/ac_experiments.ipynb b/book/notes/actor-critic/ac_experiments.ipynb index 0884155..f5b0acf 100644 --- a/book/notes/actor-critic/ac_experiments.ipynb +++ b/book/notes/actor-critic/ac_experiments.ipynb @@ -53,7 +53,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/actor-critic/ac_experiments_en.ipynb b/book/notes/actor-critic/ac_experiments_en.ipynb index f5d774e..dd1226b 100644 --- a/book/notes/actor-critic/ac_experiments_en.ipynb +++ b/book/notes/actor-critic/ac_experiments_en.ipynb @@ -53,7 +53,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/decision-transformer/decision-transformer_experiments.ipynb b/book/notes/decision-transformer/decision-transformer_experiments.ipynb index 869c8e5..4060ddd 100644 --- a/book/notes/decision-transformer/decision-transformer_experiments.ipynb +++ b/book/notes/decision-transformer/decision-transformer_experiments.ipynb @@ -45,7 +45,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb b/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb index 7caa0b3..85b682e 100644 --- a/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb +++ b/book/notes/decision-transformer/decision-transformer_experiments_en.ipynb @@ -45,7 +45,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/dpo/dpo_experiments.ipynb b/book/notes/dpo/dpo_experiments.ipynb index e123dca..d86a754 100644 --- a/book/notes/dpo/dpo_experiments.ipynb +++ b/book/notes/dpo/dpo_experiments.ipynb @@ -45,7 +45,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/dpo/dpo_experiments_en.ipynb b/book/notes/dpo/dpo_experiments_en.ipynb index b3c40b5..9e9d829 100644 --- a/book/notes/dpo/dpo_experiments_en.ipynb +++ b/book/notes/dpo/dpo_experiments_en.ipynb @@ -45,7 +45,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/dqn/dqn_experiments.ipynb b/book/notes/dqn/dqn_experiments.ipynb index 1b04ef8..fe9efe2 100644 --- a/book/notes/dqn/dqn_experiments.ipynb +++ b/book/notes/dqn/dqn_experiments.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/dqn/dqn_experiments_en.ipynb b/book/notes/dqn/dqn_experiments_en.ipynb index 0c9b128..92f0523 100644 --- a/book/notes/dqn/dqn_experiments_en.ipynb +++ b/book/notes/dqn/dqn_experiments_en.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/grpo/grpo_experiments.ipynb b/book/notes/grpo/grpo_experiments.ipynb index 88047f9..b5ac830 100644 --- a/book/notes/grpo/grpo_experiments.ipynb +++ b/book/notes/grpo/grpo_experiments.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/grpo/grpo_experiments_en.ipynb b/book/notes/grpo/grpo_experiments_en.ipynb index 9e4ca8d..b509e5b 100644 --- a/book/notes/grpo/grpo_experiments_en.ipynb +++ b/book/notes/grpo/grpo_experiments_en.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/mdp/mdp_experiments.ipynb b/book/notes/mdp/mdp_experiments.ipynb index c0d64f6..efe5c1a 100644 --- a/book/notes/mdp/mdp_experiments.ipynb +++ b/book/notes/mdp/mdp_experiments.ipynb @@ -46,7 +46,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/mdp/mdp_experiments_en.ipynb b/book/notes/mdp/mdp_experiments_en.ipynb index b26f279..70e6e0a 100644 --- a/book/notes/mdp/mdp_experiments_en.ipynb +++ b/book/notes/mdp/mdp_experiments_en.ipynb @@ -45,7 +45,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/model-based-rl/model-based-rl_experiments.ipynb b/book/notes/model-based-rl/model-based-rl_experiments.ipynb index ed2b894..f1aa615 100644 --- a/book/notes/model-based-rl/model-based-rl_experiments.ipynb +++ b/book/notes/model-based-rl/model-based-rl_experiments.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb b/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb index 09b1d68..1e90a01 100644 --- a/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb +++ b/book/notes/model-based-rl/model-based-rl_experiments_en.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb b/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb index 7855bb7..f9dd854 100644 --- a/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb +++ b/book/notes/multi-armed-bandit/multi-armed-bandit_experiments.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb b/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb index 39a37e5..4246f91 100644 --- a/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb +++ b/book/notes/multi-armed-bandit/multi-armed-bandit_experiments_en.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/offline-rl/offline-rl_experiments.ipynb b/book/notes/offline-rl/offline-rl_experiments.ipynb index 4795aef..7c7da16 100644 --- a/book/notes/offline-rl/offline-rl_experiments.ipynb +++ b/book/notes/offline-rl/offline-rl_experiments.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/offline-rl/offline-rl_experiments_en.ipynb b/book/notes/offline-rl/offline-rl_experiments_en.ipynb index 5da97a7..5d7aa6b 100644 --- a/book/notes/offline-rl/offline-rl_experiments_en.ipynb +++ b/book/notes/offline-rl/offline-rl_experiments_en.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/policy-gradient/pg_experiments.ipynb b/book/notes/policy-gradient/pg_experiments.ipynb index e8cf1c1..ad4abac 100644 --- a/book/notes/policy-gradient/pg_experiments.ipynb +++ b/book/notes/policy-gradient/pg_experiments.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/policy-gradient/pg_experiments_en.ipynb b/book/notes/policy-gradient/pg_experiments_en.ipynb index 537ec33..d3706fc 100644 --- a/book/notes/policy-gradient/pg_experiments_en.ipynb +++ b/book/notes/policy-gradient/pg_experiments_en.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/ppo/ppo_experiments.ipynb b/book/notes/ppo/ppo_experiments.ipynb index 1e015a7..96e471a 100644 --- a/book/notes/ppo/ppo_experiments.ipynb +++ b/book/notes/ppo/ppo_experiments.ipynb @@ -42,7 +42,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/ppo/ppo_experiments_en.ipynb b/book/notes/ppo/ppo_experiments_en.ipynb index 709acf5..586d603 100644 --- a/book/notes/ppo/ppo_experiments_en.ipynb +++ b/book/notes/ppo/ppo_experiments_en.ipynb @@ -42,7 +42,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/rlhf/rlhf_experiments.ipynb b/book/notes/rlhf/rlhf_experiments.ipynb index 055e99b..a7abf8e 100644 --- a/book/notes/rlhf/rlhf_experiments.ipynb +++ b/book/notes/rlhf/rlhf_experiments.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/rlhf/rlhf_experiments_en.ipynb b/book/notes/rlhf/rlhf_experiments_en.ipynb index e8237f6..51fe725 100644 --- a/book/notes/rlhf/rlhf_experiments_en.ipynb +++ b/book/notes/rlhf/rlhf_experiments_en.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/sac/sac_experiments.ipynb b/book/notes/sac/sac_experiments.ipynb index cea1f15..f6c12ff 100644 --- a/book/notes/sac/sac_experiments.ipynb +++ b/book/notes/sac/sac_experiments.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/sac/sac_experiments_en.ipynb b/book/notes/sac/sac_experiments_en.ipynb index 4d1c322..67cbccf 100644 --- a/book/notes/sac/sac_experiments_en.ipynb +++ b/book/notes/sac/sac_experiments_en.ipynb @@ -43,7 +43,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb b/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb index 99b487d..6c25519 100644 --- a/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb +++ b/book/notes/temporal-difference-learning/temporal-difference-learning_experiments.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb b/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb index 7d059c9..43c3092 100644 --- a/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb +++ b/book/notes/temporal-difference-learning/temporal-difference-learning_experiments_en.ipynb @@ -47,7 +47,7 @@ "\n", " _setup = Path('/content/_breakrl_colab_setup.py')\n", " _req = urllib.request.Request(\n", - " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/main/scripts/colab_setup.py' + '?v=2',\n", + " 'https://raw.githubusercontent.com/Powfu-zwx/BreakRL/v1.2.2/scripts/colab_setup.py' + '?v=v1.2.2',\n", " headers={'Cache-Control': 'no-cache'},\n", " )\n", " _setup.write_bytes(urllib.request.urlopen(_req).read())\n", diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index ed4d7bf..a249e93 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -18,17 +18,21 @@ python -m ipykernel install --user --name rl_env --display-name rl_env python scripts/run_jupyter.py # Windows certificate-store workaround; otherwise python -m jupyter lab ``` +`requirements.txt` keeps lower bounds so Colab resolves current packages. The committed figures and saved outputs were generated with Python 3.10.21, numpy 2.2.6, matplotlib 3.10.9, torch 2.9.0, and gymnasium 1.3.0, on the site toolchain jupyter-book 1.0.4.post1 / myst-nb 1.4.0 / Sphinx 7.4.7; a newer stack can move the last digits of an experiment. + ## Chapter layout -Each chapter lives in `book/notes//` and contains: +Each chapter lives in `book/notes//` and follows this convention: - one `.tex` file and the compiled `.pdf`; - one `_experiments.ipynb`; - three `fig*.pdf` experiment figures generated by the notebook. +That shape is a convention, not a schema. `scripts/check_consistency.py` only checks that every `.tex` has a compiled PDF beside it and that every experiment notebook parses; what a chapter holds beyond that is the author's call. + **Bilingual pair:** every chapter has a Chinese edition (`.tex` / `.pdf` / `_experiments.ipynb`) and an English edition (`_en.tex` / `_en.pdf` / `_experiments_en.ipynb`). English notebooks translate markdown cells only; code and saved outputs must match the Chinese notebook (`scripts/check_consistency.py` enforces this). Register both editions in the English and 中文版 parts of `book/_toc.yml`, in `README.md`, in `README.zh.md`, and in the `book/index.md` / `book/index-zh.md` tables. **If you change Chinese conclusions, numbers, or figures, update the English edition in the same change.** -The site language toggle (`book/_static/lang-toggle.js`) depends on filename pairs: `*_experiments` ↔ `*_experiments_en`, `failure-atlas` ↔ `failure-atlas-en`, `index.html` ↔ `index-zh.html`, and `offline-rl-text` ↔ `offline-rl-text-en`. Keep those names if you add or rename a page. The homepage is one language per page (English default). +The site language toggle (`book/_static/lang-toggle.js`) pairs pages by filename. Chapter notebooks pair automatically (`*_experiments` ↔ `*_experiments_en`, and `scripts/check_consistency.py` fails when one edition is missing); the site's own text pages are listed in the toggle's `TEXT_PAGES`, so a new one needs an entry there: `failure-atlas` ↔ `failure-atlas-en`, `index.html` ↔ `index-zh.html`, `offline-rl-text` ↔ `offline-rl-text-en`. The homepage is one language per page (English default). Chapter text follows [`book/notes/rl_note_template.tex`](../book/notes/rl_note_template.tex): story → formalism → mechanism (three ablation figures) → algorithm → comparison → place in the family → references. Use `[!htbp]` on figure environments. Notebooks use three Figure sections plus a short recap. diff --git a/docs/CONTRIBUTING.zh.md b/docs/CONTRIBUTING.zh.md index 3858195..b2cd80d 100644 --- a/docs/CONTRIBUTING.zh.md +++ b/docs/CONTRIBUTING.zh.md @@ -18,17 +18,21 @@ python -m ipykernel install --user --name rl_env --display-name rl_env python scripts/run_jupyter.py # Windows 证书库规避；无此问题可直接 python -m jupyter lab ``` +`requirements.txt` 只写下界，以便 Colab 解析到当前可用版本。已提交的图表与 notebook 输出由 Python 3.10.21、numpy 2.2.6、matplotlib 3.10.9、torch 2.9.0、gymnasium 1.3.0 生成，站点工具链为 jupyter-book 1.0.4.post1 / myst-nb 1.4.0 / Sphinx 7.4.7；换成更新的版本可能让实验的末位数字发生偏移。 + ## 章节结构约定 -每章一个目录 `book/notes//`，包含且仅包含： +每章一个目录 `book/notes//`，约定如下： - 一个 `.tex` 与同名编译产物 `.pdf`； - 一个 `_experiments.ipynb`； - 三张 `fig*.pdf` 实验图（由 notebook 生成）。 +这是写作约定而非结构校验：`scripts/check_consistency.py` 只保证每个 `.tex` 旁边有编译好的同名 PDF、每个实验 notebook 能被解析，章节里放什么由作者决定。 + **英文版（双语对齐）**：每章均含中文版（`.tex` / `.pdf` / `_experiments.ipynb`）与英文版（`_en.tex` / `_en.pdf` / `_experiments_en.ipynb`）。英文版只翻译 markdown 单元格，代码与输出和中文版保持一致（`scripts/check_consistency.py` 会强制校验英文版 Notebook 的代码单元与中文版一致），并登记进 `book/_toc.yml` 的 English 与中文版分区、`README.md`、`README.zh.md` 与 `book/index.md` / `book/index-zh.md` 的章节表。**修改中文版内容（正文结论、实验数值、图）时必须同步英文版**——两份不一致视为未完成。 -站点的「中文 / EN」语言切换按钮（`book/_static/lang-toggle.js`）依赖文件名约定：`*_experiments` ↔ `*_experiments_en`、`failure-atlas` ↔ `failure-atlas-en`、`index.html` ↔ `index-zh.html`、`offline-rl-text` ↔ `offline-rl-text-en` 互为平行页。首页每种语言单独一页（默认英文）。新增或改名页面需保持该约定，否则切换按钮失效。 +站点的「中文 / EN」语言切换按钮（`book/_static/lang-toggle.js`）按文件名配对。章节 notebook 自动配对（`*_experiments` ↔ `*_experiments_en`，缺一版时 `scripts/check_consistency.py` 报错）；站点自己的文字页需登记在切换按钮的 `TEXT_PAGES` 里，新增时要在那里加一条：`failure-atlas` ↔ `failure-atlas-en`、`index.html` ↔ `index-zh.html`、`offline-rl-text` ↔ `offline-rl-text-en`。首页每种语言单独一页（默认英文）。 正文遵循模板 [`book/notes/rl_note_template.tex`](../book/notes/rl_note_template.tex) 的结构：先讲故事 → 形式化 → 机制（配三图消融）→ 算法流程 → 对比 → 体系位置 → 参考资料。figure 环境统一用 `[!htbp]`。notebook 按三个 Figure 小节 + 小结组织。 diff --git a/scripts/breakrl_locale.py b/scripts/breakrl_locale.py index c9f36fe..8de240d 100644 --- a/scripts/breakrl_locale.py +++ b/scripts/breakrl_locale.py @@ -1,10 +1,13 @@ """Set the generated HTML locale for Chinese book pages. The site default language is English. Chinese experiment notebooks and the -Chinese failure atlas still need zh_CN so screen readers, search, and the -language toggle see the correct page locale. +Chinese failure atlas still need a Chinese tag so screen readers, search, and +the language toggle see the correct page locale. Sphinx writes this value into +`` and `docsearch:language` verbatim, so it has to be a BCP 47 tag. """ +CHINESE_LANGUAGE = "zh-CN" + def is_chinese_page(pagename): if pagename in {"failure-atlas", "index-zh", "offline-rl-text"}: @@ -14,7 +17,7 @@ def is_chinese_page(pagename): def set_page_language(app, pagename, templatename, context, doctree): if is_chinese_page(pagename): - context["language"] = "zh_CN" + context["language"] = CHINESE_LANGUAGE def setup(app): diff --git a/scripts/check_consistency.py b/scripts/check_consistency.py index f84e00f..0f58407 100644 --- a/scripts/check_consistency.py +++ b/scripts/check_consistency.py @@ -13,6 +13,19 @@ FIGURE_ENVIRONMENT = re.compile(r"\\begin\{figure\}\[([^]]+)\]") INCLUDE_GRAPHIC = re.compile(r"\\includegraphics(?:\[[^]]*\])?\{([^}]+)\}") TOC_FILE_ENTRY = re.compile(r"^\s*-\s+file:\s+(\S+)", re.MULTILINE) +MARKDOWN_LINK = re.compile(r"\]\(([^)\s]+)") +HTML_HREF = re.compile(r'href="([^"]+)"') + +# The reader's entry path: the minimum demo, Failure Atlas #8, then the Offline +# RL chapter. Each of these pages restates it in its own markup, so the check +# asks for the link targets and leaves the wording around them to the author. +# It is a page-level check: it says the path is still offered, not where. +ENTRY_PAGES = ( + ("README.md", "failure-atlas-en.html"), + ("README.zh.md", "failure-atlas.html"), + ("book/index.md", "failure-atlas-en.html"), + ("book/index-zh.md", "failure-atlas.html"), +) def chapter_dirs() -> list[Path]: @@ -28,6 +41,30 @@ def _cell_source(cell: Any) -> str: return source.replace("\r\n", "\n") +def _strip_tex_comments(text: str) -> str: + """Drop TeX comments so commented-out source never drives a check. + + A percent sign starts a comment only when it is not escaped, so the + backslashes immediately before it are counted: ``\\%`` renders a literal + percent, while ``\\\\%`` (a line break followed by a comment) does not. + """ + lines = [] + for line in text.splitlines(): + for index, char in enumerate(line): + if char != "%": + continue + backslashes = 0 + cursor = index - 1 + while cursor >= 0 and line[cursor] == "\\": + backslashes += 1 + cursor -= 1 + if backslashes % 2 == 0: + line = line[:index] + break + lines.append(line) + return "\n".join(lines) + + def _load_notebook(path: Path, nbformat: Any, errors: list[str], label: str) -> Any: try: return nbformat.read(path, as_version=4) @@ -47,30 +84,13 @@ def check_chapters() -> list[str]: for chapter in chapter_dirs(): rel = chapter.relative_to(REPO_ROOT) - tex_files = sorted(chapter.glob("*.tex")) - en_tex = [path for path in tex_files if path.stem.endswith("_en")] - main_tex = [path for path in tex_files if not path.stem.endswith("_en")] - if len(main_tex) != 1: - errors.append(f"{rel}: expected exactly one main .tex, found {len(main_tex)}") - if len(en_tex) > 1: - errors.append(f"{rel}: expected at most one *_en.tex, found {len(en_tex)}") - - for tex in main_tex + en_tex: + for tex in sorted(chapter.glob("*.tex")): pdf = tex.with_suffix(".pdf") if not pdf.exists(): errors.append(f"{rel}: missing chapter PDF {pdf.name}") - notebooks = sorted(chapter.glob("*_experiments.ipynb")) - if len(notebooks) != 1: - errors.append( - f"{rel}: expected exactly one *_experiments.ipynb, found {len(notebooks)}" - ) - for path in notebooks: + for path in sorted(chapter.glob("*_experiments*.ipynb")): _load_notebook(path, nbformat, errors, str(path.relative_to(REPO_ROOT))) - - figures = sorted(chapter.glob("fig*.pdf")) - if len(figures) != 3: - errors.append(f"{rel}: expected exactly three figure PDFs, found {len(figures)}") return errors @@ -102,6 +122,18 @@ def _compare_notebooks(cn: Any, en: Any, rel: Path, errors: list[str]) -> None: break +def _paired_edition(path: Path) -> Path: + """Return the same asset in the other language edition. + + Chapters ship two editions of every tex and notebook and name them + ```` (Chinese) and ``_en`` (English). The site's language toggle + derives its links from that convention, so both editions must exist. + """ + stem = path.stem + twin = stem.removesuffix("_en") if stem.endswith("_en") else f"{stem}_en" + return path.with_name(f"{twin}{path.suffix}") + + def check_bilingual() -> list[str]: errors = [] try: @@ -111,42 +143,20 @@ def check_bilingual() -> list[str]: for chapter in chapter_dirs(): rel = chapter.relative_to(REPO_ROOT) - cn_notebooks = sorted(chapter.glob("*_experiments.ipynb")) - en_tex = sorted(chapter.glob("*_en.tex")) - en_notebooks = sorted(chapter.glob("*_experiments_en.ipynb")) - - if not en_tex and not en_notebooks: - continue - - for path in en_tex: - base = path.stem.removesuffix("_en") - cn_path = chapter / f"{base}_experiments.ipynb" - en_path = chapter / f"{base}_experiments_en.ipynb" - if not cn_path.exists(): - errors.append(f"{rel}: missing Chinese notebook {cn_path.name}") + for pattern in ("*.tex", "*_experiments*.ipynb"): + for path in sorted(chapter.glob(pattern)): + twin = _paired_edition(path) + if not twin.exists(): + errors.append(f"{rel}: {path.name} has no paired edition {twin.name}") + + for cn_path in sorted(chapter.glob("*_experiments.ipynb")): + en_path = _paired_edition(cn_path) if not en_path.exists(): - errors.append(f"{rel}: missing English notebook {en_path.name}") - if not cn_path.exists() or not en_path.exists(): continue cn = _load_notebook(cn_path, nbformat, errors, str(cn_path.relative_to(REPO_ROOT))) en = _load_notebook(en_path, nbformat, errors, str(en_path.relative_to(REPO_ROOT))) if cn is not None and en is not None: _compare_notebooks(cn, en, rel, errors) - - for path in en_notebooks: - base = path.stem.removesuffix("_experiments_en") - expected_tex = chapter / f"{base}_en.tex" - if not expected_tex.exists(): - errors.append(f"{rel}: English notebook has no paired TeX {expected_tex.name}") - - if len(en_tex) > 1: - errors.append(f"{rel}: expected at most one *_en.tex, found {len(en_tex)}") - if len(en_notebooks) > 1: - errors.append( - f"{rel}: expected at most one *_experiments_en.ipynb, found {len(en_notebooks)}" - ) - if not cn_notebooks: - errors.append(f"{rel}: English assets exist but the Chinese notebook is missing") return errors @@ -156,7 +166,7 @@ def check_tex() -> list[str]: return [f"missing {NOTES.relative_to(REPO_ROOT).as_posix()} directory"] for path in sorted(NOTES.rglob("*.tex")): try: - text = path.read_text(encoding="utf-8") + text = _strip_tex_comments(path.read_text(encoding="utf-8")) except OSError as error: errors.append(f"{path.relative_to(REPO_ROOT)}: cannot read TeX: {error}") continue @@ -167,8 +177,6 @@ def check_tex() -> list[str]: ) for match in INCLUDE_GRAPHIC.finditer(text): graphic = Path(match.group(1)) - if graphic.name.startswith("fig1_xxx"): - continue if graphic.suffix: candidates = [path.parent / graphic] else: @@ -200,21 +208,36 @@ def check_toc() -> list[str]: else: errors.append(f"{toc_label}: missing root entry") - toc_notebooks = set() for entry in entries: source = _resolve_toc_entry(entry) - if entry.startswith("notes/"): - toc_notebooks.add(source) if not source.exists(): errors.append(f"{toc_label}: missing source for entry {entry}") + return errors - repo_notebooks = { - notebook - for chapter in chapter_dirs() - for notebook in chapter.glob("*_experiments.ipynb") - } - for notebook in sorted(repo_notebooks - toc_notebooks): - errors.append(f"{toc_label}: chapter notebook not listed: {notebook.relative_to(REPO_ROOT)}") + +def _link_targets(text: str) -> list[str]: + """Every link target in a markdown document, raw HTML anchors included.""" + return MARKDOWN_LINK.findall(text) + HTML_HREF.findall(text) + + +def check_entry_path() -> list[str]: + errors = [] + for name, atlas in ENTRY_PAGES: + path = REPO_ROOT / name + if not path.is_file(): + errors.append(f"missing entry page {name}") + continue + targets = _link_targets(path.read_text(encoding="utf-8")) + wanted = { + "the minimum demo": [t for t in targets if t.rstrip("/").endswith(("demo", "demo.html"))], + f"{atlas}#atlas-8-offline-loss": [ + t for t in targets if t.endswith(f"{atlas}#atlas-8-offline-loss") + ], + "the Offline RL chapter": [t for t in targets if "offline-rl" in t], + } + for label, matches in wanted.items(): + if not matches: + errors.append(f"{name}: entry path no longer links {label}") return errors @@ -235,11 +258,7 @@ def check_colab_entry_points() -> list[str]: index_zh_label = (BOOK / "index-zh.md").relative_to(REPO_ROOT).as_posix() for chapter in chapter_dirs(): wanted = bootstrap_source(chapter.name) - notebooks = sorted(chapter.glob("*_experiments*.ipynb")) - if not notebooks: - errors.append(f"{chapter.relative_to(REPO_ROOT)}: missing experiment notebooks") - continue - for path in notebooks: + for path in sorted(chapter.glob("*_experiments*.ipynb")): rel = path.relative_to(REPO_ROOT) notebook = _load_notebook(path, nbformat, errors, str(rel)) if notebook is None: @@ -265,111 +284,14 @@ def check_colab_entry_points() -> list[str]: return errors -def _section_between(text: str, start: str, end: str, label: str, errors: list[str]) -> str: - start_at = text.find(start) - end_at = text.find(end) - if start_at < 0 or end_at < 0 or end_at <= start_at: - errors.append(f"{label}: missing section {start!r} .. {end!r}") - return "" - return text[start_at:end_at] - - -def check_flagship_ring() -> list[str]: - errors = [] - anchor = "#atlas-8-offline-loss" - readme_en = (REPO_ROOT / "README.md").read_text(encoding="utf-8") - readme_zh = (REPO_ROOT / "README.zh.md").read_text(encoding="utf-8") - index_en = (BOOK / "index.md").read_text(encoding="utf-8") - index_zh = (BOOK / "index-zh.md").read_text(encoding="utf-8") - demo = (BOOK / "demo.md").read_text(encoding="utf-8") - atlas_en = (BOOK / "failure-atlas-en.md").read_text(encoding="utf-8") - atlas_zh = (BOOK / "failure-atlas.md").read_text(encoding="utf-8") - - en_start = _section_between( - readme_en, "## Start in three minutes", "## Learning path", "README.md", errors - ) - zh_start = _section_between( - readme_zh, "## 三分钟开始", "## 学习路线", "README.zh.md", errors - ) - if en_start: - for needle in ("demo.html", f"failure-atlas-en.html{anchor}", "offline-rl"): - if needle not in en_start: - errors.append(f"README.md three-minute start is missing {needle}") - if "and start with Chapter 1" in en_start: - errors.append("README.md three-minute start still sends readers to Chapter 1") - if zh_start: - for needle in ("demo.html", f"failure-atlas.html{anchor}", "offline-rl"): - if needle not in zh_start: - errors.append(f"README.zh.md three-minute start is missing {needle}") - if "打开[在线教材]" in zh_start: - errors.append("README.zh.md three-minute start still sends readers to Chapter 1") - - if "三分钟开始" in index_en or "用失败学强化学习" in index_en: - errors.append("book/index.md still stacks the Chinese homepage body") - if "Start in three minutes" in index_zh or "Learn reinforcement learning through failure" in index_zh: - errors.append("book/index-zh.md still stacks the English homepage body") - for label, text, needles in ( - ("book/index.md", index_en, ("demo", "atlas-8-offline-loss", "offline-rl")), - ("book/index-zh.md", index_zh, ("demo", "atlas-8-offline-loss", "offline-rl")), - ): - for needle in needles: - if needle not in text: - errors.append(f"{label}: missing flagship link {needle}") - if "github.com/Powfu-zwx/BreakRL/blob/" in text and "offline-rl" in text: - if re.search( - r"github\.com/Powfu-zwx/BreakRL/blob/[^)\s]*offline-rl[^)\s]*\.pdf", - text, - ): - errors.append(f"{label}: Offline RL PDF still points at a GitHub blob") - - if anchor not in demo or "offline-rl" not in demo: - errors.append("book/demo.md must link Failure Atlas #8 and the Offline RL chapter") - if 'id="atlas-8-offline-loss"' not in atlas_en or "demo" not in atlas_en or "offline-rl" not in atlas_en: - errors.append("book/failure-atlas-en.md #8 must keep the demo → atlas → chapter loop") - if 'id="atlas-8-offline-loss"' not in atlas_zh or "demo" not in atlas_zh or "offline-rl" not in atlas_zh: - errors.append("book/failure-atlas.md #8 must keep the demo → atlas → chapter loop") - return errors - - -def check_flagship_look() -> list[str]: - """Keep the site wordmark recolorable and the demo h1 in the accessibility tree.""" - from check_site import css_hides_h1 - - errors = [] - config = (BOOK / "_config.yml").read_text(encoding="utf-8") - if re.search(r"(?m)^\s*logo\s*:", config): - errors.append("book/_config.yml must not set an image logo; navbar brand is CSS text / currentColor") - logo = REPO_ROOT / "assets" / "breakrl-logo.svg" - if logo.is_file(): - text = logo.read_text(encoding="utf-8", errors="ignore") - if "#171513" in text: - errors.append( - "assets/breakrl-logo.svg uses hardcoded fill #171513; " - "do not ship an image SVG the site cannot recolor" - ) - else: - errors.append("assets/breakrl-logo.svg must not ship; navbar brand is CSS text / currentColor") - demo = (BOOK / "demo.md").read_text(encoding="utf-8") - if not re.search(r"(?m)^# ", demo): - errors.append("book/demo.md must keep a markdown level-one heading") - for css_name in ("breakrl.css", "breakrl-demo.css", "lang-toggle.css"): - css_path = BOOK / "_static" / css_name - if not css_path.is_file(): - continue - if css_hides_h1(css_path.read_text(encoding="utf-8")): - errors.append(f"book/_static/{css_name}: must not display:none an h1") - return errors - - def main() -> int: errors = ( check_chapters() + check_bilingual() + check_tex() + check_toc() + + check_entry_path() + check_colab_entry_points() - + check_flagship_ring() - + check_flagship_look() ) if errors: for error in errors: diff --git a/scripts/check_site.py b/scripts/check_site.py index 41d045d..76a630c 100644 --- a/scripts/check_site.py +++ b/scripts/check_site.py @@ -9,9 +9,14 @@ if str(_SCRIPTS) not in sys.path: sys.path.insert(0, str(_SCRIPTS)) +from breakrl_locale import is_chinese_page # noqa: E402 from paths import BOOK_BUILD # noqa: E402 DEFAULT_BUILD = BOOK_BUILD +# Sphinx writes the language into `` and `docsearch:language` +# verbatim, so these are BCP 47 tags. `zh_CN` is a Python locale name, not a +# tag, and belongs here only as a failure. +CHINESE_LANG_VALUES = {"zh-CN", "zh"} _H1_IN_SELECTOR = re.compile(r"(^|[\s,>+~])h1($|[\s,:+.\[#>~])", re.I) _DISPLAY_NONE = re.compile(r"display\s*:\s*none", re.I) @@ -25,18 +30,17 @@ def __init__(self) -> None: self.html_lang: str | None = None self.docsearch_language: str | None = None - def _collect(self, attrs: list[tuple[str, str | None]]) -> None: - for name, value in attrs: - if name in {"href", "src"} and value: - self.links.append(value) - def _process_tag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attributes = dict(attrs) if tag == "html": self.html_lang = attributes.get("lang") if tag == "meta" and attributes.get("name") == "docsearch:language": self.docsearch_language = attributes.get("content") - self._collect(attrs) + # A page embeds its own PDFs with ``, and Jupyter Book + # neither rewrites nor copies those. Any other `data` attribute is a + # value the site scripts read, not a file. + names = {"href", "src"} | ({"data"} if tag in {"object", "embed"} else set()) + self.links.extend(value for name, value in attrs if name in names and value) def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: self._process_tag(tag, attrs) @@ -74,24 +78,31 @@ def _local_target(build_root: Path, source: Path, value: str) -> Path | None: return target +def page_files(build_root: Path) -> list[Path]: + """Every HTML page a reader can reach, in a stable order. + + Jupyter Book ships a few HTML macro templates in ``_static``; they contain + unresolved Sphinx expressions by design and are not public pages. Anything + under ``.github`` is repository metadata that leaked into the build. + """ + return sorted( + path + for path in build_root.rglob("*.html") + if "_static" not in path.relative_to(build_root).parts + and ".github" not in path.relative_to(build_root).parts + ) + + def check_site(build_root: Path = DEFAULT_BUILD) -> list[str]: errors = [] if not build_root.is_dir(): return [f"missing site build directory: {build_root}"] - all_html_files = sorted(build_root.rglob("*.html")) - for path in all_html_files: + for path in sorted(build_root.rglob("*.html")): if ".github" in path.relative_to(build_root).parts: errors.append(f"stray non-book page in build: {path.relative_to(build_root)}") - # Jupyter Book ships a few HTML macro templates in _static. They contain - # unresolved Sphinx expressions by design and are not public pages. - html_files = [ - path - for path in all_html_files - if "_static" not in path.relative_to(build_root).parts - and ".github" not in path.relative_to(build_root).parts - ] + html_files = page_files(build_root) if not html_files: return [f"no HTML pages found under {build_root}"] @@ -103,31 +114,21 @@ def check_site(build_root: Path = DEFAULT_BUILD) -> list[str]: except (OSError, UnicodeError) as error: errors.append(f"cannot read {relative}: {error}") continue - is_english_page = relative.name == "failure-atlas-en.html" or relative.name.endswith( - "_en.html" - ) - is_chinese_page = relative.name in {"failure-atlas.html", "index-zh.html", "offline-rl-text.html"} or ( - relative.name.endswith("_experiments.html") - and not relative.name.endswith("_experiments_en.html") - ) - if is_english_page and parser.html_lang != "en": - errors.append(f"{relative}: English page must declare html lang=\"en\"") - if is_english_page and parser.docsearch_language != "en": - errors.append(f"{relative}: English page must declare docsearch:language=en") - if is_chinese_page and parser.html_lang not in {"zh-CN", "zh_CN", "zh"}: - errors.append(f"{relative}: Chinese page must declare html lang=\"zh-CN\"") - if is_chinese_page and parser.docsearch_language not in {"zh-CN", "zh_CN", "zh"}: - errors.append(f"{relative}: Chinese page must declare docsearch:language=zh-CN") + if is_chinese_page(relative.name.removesuffix(".html")): + if parser.html_lang not in CHINESE_LANG_VALUES: + errors.append(f"{relative}: Chinese page must declare html lang=\"zh-CN\"") + if parser.docsearch_language not in CHINESE_LANG_VALUES: + errors.append(f"{relative}: Chinese page must declare docsearch:language=zh-CN") + else: + if parser.html_lang != "en": + errors.append(f"{relative}: English page must declare html lang=\"en\"") + if parser.docsearch_language != "en": + errors.append(f"{relative}: English page must declare docsearch:language=en") for value in parser.links: target = _local_target(build_root, path, value) if target is not None and not target.exists(): errors.append(f"{relative}: missing local target {value}") - for name in ("offline-rl.pdf", "offline-rl_en.pdf"): - pdf = build_root / "notes" / "offline-rl" / name - if not pdf.is_file(): - errors.append(f"missing flagship chapter PDF: notes/offline-rl/{name}") - for atlas_name in ("failure-atlas-en.html", "failure-atlas.html"): atlas = build_root / atlas_name if not atlas.is_file(): @@ -135,24 +136,16 @@ def check_site(build_root: Path = DEFAULT_BUILD) -> list[str]: elif 'id="atlas-8-offline-loss"' not in atlas.read_text(encoding="utf-8"): errors.append(f"{atlas_name}: missing id=\"atlas-8-offline-loss\"") - blob_pdf = re.compile( - r"github\.com/Powfu-zwx/BreakRL/blob/[^\"']*offline-rl[^\"']*\.pdf" - ) - homepage_checks = ( - (build_root / "index.html", "三分钟开始", "Chinese homepage body"), - (build_root / "index-zh.html", "Start in three minutes", "English homepage body"), - ) - for path, stacked_marker, label in homepage_checks: + # The homepage links the atlas across pages. The language toggle must not + # rewrite that into an in-page hash, which resolves to nothing. + for homepage in ("index.html", "index-zh.html"): + path = build_root / homepage if not path.is_file(): - errors.append(f"missing homepage: {path.relative_to(build_root)}") - continue - text = path.read_text(encoding="utf-8") - if stacked_marker in text: - errors.append(f"{path.name}: still stacks the {label}") - if blob_pdf.search(text): - errors.append(f"{path.name}: Offline RL PDF still points at a GitHub blob") - if 'href="#failure-atlas' in text: - errors.append(f"{path.name}: Failure Atlas #8 link was rewritten into a broken in-page hash") + errors.append(f"missing homepage: {homepage}") + elif 'href="#failure-atlas' in path.read_text(encoding="utf-8"): + errors.append( + f"{homepage}: Failure Atlas #8 link was rewritten into a broken in-page hash" + ) demo = build_root / "demo.html" if not demo.is_file(): @@ -166,16 +159,6 @@ def check_site(build_root: Path = DEFAULT_BUILD) -> list[str]: for css_path in sorted(static.glob("breakrl*.css")) + sorted(static.glob("lang-toggle.css")): if css_hides_h1(css_path.read_text(encoding="utf-8")): errors.append(f"{css_path.relative_to(build_root)}: must not display:none an h1") - - for svg in static.glob("breakrl-logo.svg"): - errors.append( - f"{svg.relative_to(build_root)}: do not ship an image wordmark SVG; navbar brand is CSS text / currentColor" - ) - for path in html_files: - if "breakrl-logo.svg" in path.read_text(encoding="utf-8"): - errors.append( - f"{path.relative_to(build_root)}: must not ship an image wordmark SVG; navbar brand is CSS text" - ) return errors @@ -186,12 +169,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}") return 1 - pages = sum( - 1 - for path in build_root.rglob("*.html") - if "_static" not in path.relative_to(build_root).parts - and ".github" not in path.relative_to(build_root).parts - ) + pages = len(page_files(build_root)) print(f"checked generated site: {pages} HTML pages and local links OK") return 0 diff --git a/scripts/colab_setup.py b/scripts/colab_setup.py index ded3d3f..a9785b3 100644 --- a/scripts/colab_setup.py +++ b/scripts/colab_setup.py @@ -8,15 +8,21 @@ import sys from pathlib import Path -from paths import NOTES_PREFIX - GITHUB_REPO = "Powfu-zwx/BreakRL" +# The bootstrap a notebook carries must be the one that shipped with it. Readers +# open `main`, so the clone below still tracks `main`; only the fetched setup +# module is pinned, and pinning it to the tag also busts the HTTP cache. +RELEASE_TAG = "v1.2.2" REPO_URL = f"https://github.com/{GITHUB_REPO}.git" RAW_SETUP_URL = ( - f"https://raw.githubusercontent.com/{GITHUB_REPO}/main/scripts/colab_setup.py" + f"https://raw.githubusercontent.com/{GITHUB_REPO}/{RELEASE_TAG}/scripts/colab_setup.py" ) COLAB_ROOT = Path("/content/BreakRL") -NOTES_DIR = Path(NOTES_PREFIX) +# This module is delivered alone, by raw URL, into a Colab runtime that has no +# checkout yet, so it must not import repo-internal helpers such as +# `scripts.paths`. The layout below is therefore restated here on purpose; +# `_repo_is_ready` fails loudly if it ever drifts from the repository. +NOTES_DIR = Path("book/notes") COLAB_NOTEBOOK_BASE = ( f"https://colab.research.google.com/github/{GITHUB_REPO}/blob/main" ) @@ -87,7 +93,7 @@ def bootstrap_source(chapter: str) -> str: "\n" " _setup = Path('/content/_breakrl_colab_setup.py')\n" " _req = urllib.request.Request(\n" - f" {RAW_SETUP_URL!r} + '?v=2',\n" + f" {RAW_SETUP_URL!r} + '?v={RELEASE_TAG}',\n" " headers={'Cache-Control': 'no-cache'},\n" " )\n" " _setup.write_bytes(urllib.request.urlopen(_req).read())\n" diff --git a/scripts/paths.py b/scripts/paths.py index 434d624..f740e96 100644 --- a/scripts/paths.py +++ b/scripts/paths.py @@ -5,4 +5,3 @@ BOOK = REPO_ROOT / "book" NOTES = BOOK / "notes" BOOK_BUILD = BOOK / "_build" / "html" -NOTES_PREFIX = "book/notes" diff --git a/scripts/prepare_site_assets.py b/scripts/prepare_site_assets.py index aa03e86..34bf8be 100644 --- a/scripts/prepare_site_assets.py +++ b/scripts/prepare_site_assets.py @@ -1,10 +1,17 @@ -"""Materialize saved notebook image outputs for a no-execution book build.""" +"""Place the assets the site links but Jupyter Book does not copy. + +Saved notebook image outputs, which myst-nb resolves from disk by content hash +because the book is not executed, and the files pages embed as raw HTML, which +Jupyter Book only copies when a markdown link points at them. +""" import base64 import hashlib import json +import re import shutil import sys from pathlib import Path +from urllib.parse import urlsplit _SCRIPTS = Path(__file__).resolve().parent if str(_SCRIPTS) not in sys.path: @@ -13,36 +20,46 @@ from paths import BOOK, NOTES OUTPUT_FOLDER = BOOK / "_build" / "jupyter_execute" +# Each saved image MIME type carries both facts this module needs: the suffix of +# the file it writes, and whether nbformat stored the payload base64-encoded +# (everything binary) or as plain markup (``image/svg+xml``). IMAGE_TYPES = { - "image/png": ".png", - "image/jpeg": ".jpeg", - "image/gif": ".gif", - "image/svg+xml": ".svg", - "application/pdf": ".pdf", + "image/png": (".png", True), + "image/jpeg": (".jpeg", True), + "image/gif": (".gif", True), + "image/svg+xml": (".svg", False), + "application/pdf": (".pdf", True), } -def _output_bytes(mime_type: str, content: str | list[str]) -> bytes: +def _output_bytes(content: str | list[str], *, base64_encoded: bool) -> bytes: if isinstance(content, list): content = "".join(content) - if mime_type in {"image/png", "image/jpeg", "image/gif", "application/pdf"}: + if base64_encoded: return base64.b64decode(content) return content.replace("\r\n", "\n").encode("utf-8") def prepare_assets(output_folder: Path = OUTPUT_FOLDER) -> int: + """Write each saved image output as a content-addressed file. + + myst-nb runs with ``execute_notebooks: "off"`` and resolves notebook image + outputs from this folder by the SHA-256 of the decoded bytes. The naming is + an internal contract of the installed myst-nb version, so re-check it before + upgrading ``jupyter-book``. + """ output_folder.mkdir(parents=True, exist_ok=True) written = 0 for notebook in sorted(NOTES.rglob("*_experiments*.ipynb")): document = json.loads(notebook.read_text(encoding="utf-8")) for cell in document.get("cells", []): for output in cell.get("outputs", []): - for mime_type, extension in IMAGE_TYPES.items(): + for mime_type, (suffix, base64_encoded) in IMAGE_TYPES.items(): content = output.get("data", {}).get(mime_type) if not content: continue - data = _output_bytes(mime_type, content) - target = output_folder / f"{hashlib.sha256(data).hexdigest()}{extension}" + data = _output_bytes(content, base64_encoded=base64_encoded) + target = output_folder / f"{hashlib.sha256(data).hexdigest()}{suffix}" if not target.exists(): target.write_bytes(data) written += 1 @@ -53,33 +70,47 @@ def prepare_build_assets(app): prepare_assets(output_folder=Path(app.outdir).parent / "jupyter_execute") -FLAGSHIP_CHAPTER_PDFS = ( - NOTES / "offline-rl" / "offline-rl.pdf", - NOTES / "offline-rl" / "offline-rl_en.pdf", -) +# A page that embeds a file with `` gets no help from Jupyter +# Book: it rewrites markdown links into `_downloads/` and copies the target, but +# leaves raw HTML alone. The embedded files therefore have to be found in the +# pages and copied to the path the page names. +INLINE_OBJECT = re.compile(r"]*\sdata=\"([^\"]+)\"") + + +def inline_assets() -> list[Path]: + found = set() + for page in sorted(BOOK.rglob("*.md")): + if "_build" in page.parts: + continue + for target in INLINE_OBJECT.findall(page.read_text(encoding="utf-8")): + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or parsed.path.startswith("/"): + continue + asset = (page.parent / parsed.path).resolve() + if asset.is_file() and asset.is_relative_to(BOOK): + found.add(asset) + return sorted(found) -def copy_flagship_chapter_pdfs(outdir: Path) -> int: - dest_dir = Path(outdir) / "notes" / "offline-rl" - dest_dir.mkdir(parents=True, exist_ok=True) +def copy_inline_assets(outdir: Path) -> int: copied = 0 - for src in FLAGSHIP_CHAPTER_PDFS: - if not src.is_file(): - raise FileNotFoundError(src) - shutil.copy2(src, dest_dir / src.name) + for src in inline_assets(): + dest = Path(outdir) / src.relative_to(BOOK) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) copied += 1 return copied -def copy_flagship_chapter_pdfs_on_build(app, exception): +def copy_inline_assets_on_build(app, exception): if exception: return - copy_flagship_chapter_pdfs(Path(app.outdir)) + copy_inline_assets(Path(app.outdir)) def setup(app): app.connect("builder-inited", prepare_build_assets) - app.connect("build-finished", copy_flagship_chapter_pdfs_on_build) + app.connect("build-finished", copy_inline_assets_on_build) return { "version": "1.1", "parallel_read_safe": True, diff --git a/scripts/sync_colab_bootstrap.py b/scripts/sync_colab_bootstrap.py index 04de2ff..3a5365b 100644 --- a/scripts/sync_colab_bootstrap.py +++ b/scripts/sync_colab_bootstrap.py @@ -1,25 +1,27 @@ """Write the Colab bootstrap cell into every experiment notebook. The cell source is owned by ``colab_setup.bootstrap_source``. Re-run this -script after changing that template. Idempotent. Edits only the first code -cell so saved outputs keep their original JSON. +script after changing that template. Idempotent: a notebook already carrying +the wanted cell is not written at all, so the diff of a no-op run is empty. +Only the first code cell is touched, so saved outputs stay as the reader +published them. """ from __future__ import annotations -import json import sys -import uuid from pathlib import Path _SCRIPTS = Path(__file__).resolve().parent if str(_SCRIPTS) not in sys.path: sys.path.insert(0, str(_SCRIPTS)) +import nbformat + from check_consistency import chapter_dirs from colab_setup import bootstrap_source from paths import REPO_ROOT -FIRST_CODE_CELL = '\n {\n "cell_type": "code",' -LEGACY_PIP_MARKERS = ("# !pip install gymnasium", "# If needed, uncomment") + +BOOTSTRAP_TAGS = ["remove-cell"] def _source_text(source: list[str] | str) -> str: @@ -28,101 +30,42 @@ def _source_text(source: list[str] | str) -> str: return source -def _source_lines(text: str) -> list[str]: - text = text.replace("\r\n", "\n") - if not text.endswith("\n"): - text += "\n" - return text.splitlines(keepends=True) - - def _is_bootstrap(source: str) -> bool: return source.lstrip().startswith("BREAKRL_CHAPTER = ") -def _is_legacy_pip_cell(source: str) -> bool: - return any(marker in source for marker in LEGACY_PIP_MARKERS) - - -def _matching_brace(text: str, open_index: int) -> int: - if text[open_index] != "{": - raise ValueError("expected '{'") - depth = 0 - in_string = False - escape = False - for index in range(open_index, len(text)): - char = text[index] - if in_string: - if escape: - escape = False - elif char == "\\": - escape = True - elif char == '"': - in_string = False - continue - if char == '"': - in_string = True - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return index - raise ValueError("unbalanced cell object") - - -def _first_code_span(text: str) -> tuple[int, int]: - marker_at = text.find(FIRST_CODE_CELL) - if marker_at < 0: - raise ValueError("notebook has no code cell") - brace = marker_at + len("\n ") - close = _matching_brace(text, brace) - return marker_at + 1, close - - -def _format_cell(cell: dict) -> str: - dumped = json.dumps(cell, indent=1, ensure_ascii=False) - return "\n".join(f" {line}" if line else line for line in dumped.splitlines()) - - -def _bootstrap_cell(chapter: str, cell_id: str | None = None) -> dict: - return { - "cell_type": "code", - "execution_count": None, - "id": cell_id or uuid.uuid4().hex[:8], - "metadata": {"tags": ["remove-cell"]}, - "outputs": [], - "source": _source_lines(bootstrap_source(chapter)), - } +def _bootstrap_cell(chapter: str): + return nbformat.v4.new_code_cell( + source=bootstrap_source(chapter).splitlines(keepends=True), + metadata={"tags": list(BOOTSTRAP_TAGS)}, + ) def sync_notebook(path: Path, chapter: str) -> str: - text = path.read_text(encoding="utf-8") - notebook = json.loads(text) - first_code = next(cell for cell in notebook["cells"] if cell.get("cell_type") == "code") - current = _source_text(first_code.get("source", "")) - wanted = _source_lines(bootstrap_source(chapter)) - start, close = _first_code_span(text) - wanted_tags = ["remove-cell"] - has_tag = first_code.get("metadata", {}).get("tags") == wanted_tags - if _is_bootstrap(current): - if _source_lines(current) == wanted and has_tag: - return "unchanged" - cell_id = first_code.get("id") or uuid.uuid4().hex[:8] - block = _format_cell(_bootstrap_cell(chapter, cell_id=str(cell_id))) - text = text[:start] + block + text[close + 1 :] - action = "updated" - elif _is_legacy_pip_cell(current): - cell_id = first_code.get("id") or uuid.uuid4().hex[:8] - block = _format_cell(_bootstrap_cell(chapter, cell_id=str(cell_id))) - text = text[:start] + block + text[close + 1 :] - action = "updated" - else: - block = _format_cell(_bootstrap_cell(chapter)) - text = text[: start - 1] + "\n" + block + "," + text[start - 1 :] + notebook = nbformat.read(path, as_version=4) + wanted = bootstrap_source(chapter) + index = next( + (i for i, cell in enumerate(notebook.cells) if cell.cell_type == "code"), None + ) + if index is None: + raise ValueError(f"{path}: notebook has no code cell") + cell = notebook.cells[index] + current = _source_text(cell.source) + + if not _is_bootstrap(current): + notebook.cells.insert(index, _bootstrap_cell(chapter)) action = "inserted" - if not text.endswith("\n"): - text += "\n" - path.write_text(text, encoding="utf-8", newline="\n") + elif current == wanted and cell.metadata.get("tags") == BOOTSTRAP_TAGS: + return "unchanged" + else: + cell.source = wanted.splitlines(keepends=True) + cell.metadata["tags"] = list(BOOTSTRAP_TAGS) + action = "updated" + + # nbformat opens the file with newline translation on, which on Windows + # would rewrite every line of the notebook as CRLF. + with open(path, "w", encoding="utf-8", newline="\n") as handle: + nbformat.write(notebook, handle) return action diff --git a/scripts/test_colab_setup.py b/scripts/test_colab_setup.py index 6f84184..bf33b92 100644 --- a/scripts/test_colab_setup.py +++ b/scripts/test_colab_setup.py @@ -2,6 +2,7 @@ from __future__ import annotations import os +import subprocess import sys from pathlib import Path @@ -19,9 +20,11 @@ setup, validate_chapter, ) -from paths import NOTES, NOTES_PREFIX # noqa: E402 +from paths import NOTES # noqa: E402 -NOTES_DIR = Path(NOTES_PREFIX) +# The repository's own layout, used to build the mock clone: the bootstrap must +# resolve the chapter where the repository actually keeps it. +NOTES_RELATIVE = NOTES.relative_to(ROOT) def _fail(errors: list[str], message: str) -> None: @@ -77,16 +80,45 @@ def test_bootstrap_source() -> list[str]: def test_colab_url() -> list[str]: errors = [] - url = colab_notebook_url(f"{NOTES_PREFIX}/ppo/ppo_experiments_en.ipynb") + notebook = f"{NOTES_RELATIVE.as_posix()}/ppo/ppo_experiments_en.ipynb" + url = colab_notebook_url(notebook) expected = ( "https://colab.research.google.com/github/Powfu-zwx/BreakRL/" - f"blob/main/{NOTES_PREFIX}/ppo/ppo_experiments_en.ipynb" + f"blob/main/{notebook}" ) if url != expected: _fail(errors, f"colab_notebook_url mismatch: {url}") return errors +def test_standalone_delivery(tmp_path: Path) -> list[str]: + """The module is fetched alone by raw URL and run before any checkout exists. + + Colab downloads this file to a bare directory and executes it, so every + import it makes must resolve without the repository on ``sys.path``. A + subprocess with a clean path is the only faithful way to check that; an + in-process import would find ``scripts/`` and mask the failure. + """ + errors = [] + delivered = tmp_path / "_breakrl_colab_setup.py" + delivered.write_text( + (SCRIPTS / "colab_setup.py").read_text(encoding="utf-8"), encoding="utf-8" + ) + result = subprocess.run( + [sys.executable, "-c", f"import runpy; runpy.run_path({str(delivered)!r})"], + cwd=tmp_path, + capture_output=True, + text=True, + ) + if result.returncode != 0: + _fail( + errors, + "the delivered Colab bootstrap does not run outside a checkout: " + + result.stderr.strip().splitlines()[-1], + ) + return errors + + def test_setup_local_noop() -> list[str]: errors = [] before = Path.cwd() @@ -110,7 +142,7 @@ def runner(command, cwd=None): # noqa: ANN001 root = Path(cmd[-1]) (root / ".git").mkdir(parents=True) (root / "scripts").mkdir() - (root / NOTES_DIR / "offline-rl").mkdir(parents=True) + (root / NOTES_RELATIVE / "offline-rl").mkdir(parents=True) (root / "requirements.txt").write_text( (ROOT / "requirements.txt").read_text(encoding="utf-8"), encoding="utf-8", @@ -124,7 +156,7 @@ def runner(command, cwd=None): # noqa: ANN001 dest=dest, runner=runner, ) - if chapter_dir != dest / NOTES_DIR / "offline-rl": + if chapter_dir != dest / NOTES_RELATIVE / "offline-rl": _fail(errors, f"setup returned unexpected chapter dir: {chapter_dir}") if Path.cwd() != chapter_dir: _fail(errors, "Colab setup did not chdir into the chapter directory") @@ -162,7 +194,7 @@ def runner(command, cwd=None): # noqa: ANN001 if len(cmd) > 1 and cmd[1] == "clone": root = Path(cmd[-1]) (root / ".git").mkdir(parents=True) - (root / NOTES_DIR / "offline-rl").mkdir(parents=True) + (root / NOTES_RELATIVE / "offline-rl").mkdir(parents=True) (root / "requirements.txt").write_text("numpy>=2\n", encoding="utf-8") before = Path.cwd() @@ -301,6 +333,7 @@ def main(argv: list[str] | None = None) -> int: + test_setup_local_noop() ) with tempfile.TemporaryDirectory() as tmp: + errors += test_standalone_delivery(Path(tmp)) errors += test_setup_colab_mock(Path(tmp)) errors += test_incomplete_clone_is_replaced(Path(tmp) / "incomplete") if smoke: diff --git a/scripts/test_lang_toggle.js b/scripts/test_lang_toggle.js index f33a310..ed90a72 100644 --- a/scripts/test_lang_toggle.js +++ b/scripts/test_lang_toggle.js @@ -24,30 +24,25 @@ vm.runInNewContext(source, sandbox, { filename: sourcePath }); const toggle = sandbox.window.BreakRLLanguageToggle; assert.ok(toggle, 'language toggle test API is available'); +// Chapter notebooks pair by the `_en` suffix, so the expected pairs come from the +// repository rather than a list kept here: a new chapter is covered on arrival. +const notesDir = path.join(__dirname, '..', 'book', 'notes'); +const chapterPairs = fs + .readdirSync(notesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((chapter) => + fs + .readdirSync(path.join(notesDir, chapter.name)) + .filter((file) => file.endsWith('_experiments.ipynb')) + .map((file) => { + const zh = `notes/${chapter.name}/${file.replace(/\.ipynb$/, '.html')}`; + return [zh, zh.replace(/\.html$/, '_en.html')]; + }) + ); +assert.ok(chapterPairs.length > 0, 'found chapter notebooks to pair'); + const pairs = [ - [ - 'notes/multi-armed-bandit/multi-armed-bandit_experiments.html', - 'notes/multi-armed-bandit/multi-armed-bandit_experiments_en.html' - ], - ['notes/mdp/mdp_experiments.html', 'notes/mdp/mdp_experiments_en.html'], - [ - 'notes/temporal-difference-learning/temporal-difference-learning_experiments.html', - 'notes/temporal-difference-learning/temporal-difference-learning_experiments_en.html' - ], - ['notes/dqn/dqn_experiments.html', 'notes/dqn/dqn_experiments_en.html'], - ['notes/policy-gradient/pg_experiments.html', 'notes/policy-gradient/pg_experiments_en.html'], - ['notes/actor-critic/ac_experiments.html', 'notes/actor-critic/ac_experiments_en.html'], - ['notes/ppo/ppo_experiments.html', 'notes/ppo/ppo_experiments_en.html'], - ['notes/sac/sac_experiments.html', 'notes/sac/sac_experiments_en.html'], - ['notes/offline-rl/offline-rl_experiments.html', 'notes/offline-rl/offline-rl_experiments_en.html'], - ['notes/model-based-rl/model-based-rl_experiments.html', 'notes/model-based-rl/model-based-rl_experiments_en.html'], - [ - 'notes/decision-transformer/decision-transformer_experiments.html', - 'notes/decision-transformer/decision-transformer_experiments_en.html' - ], - ['notes/rlhf/rlhf_experiments.html', 'notes/rlhf/rlhf_experiments_en.html'], - ['notes/dpo/dpo_experiments.html', 'notes/dpo/dpo_experiments_en.html'], - ['notes/grpo/grpo_experiments.html', 'notes/grpo/grpo_experiments_en.html'], + ...chapterPairs, ['failure-atlas.html', 'failure-atlas-en.html'], ['offline-rl-text.html', 'offline-rl-text-en.html'], ['index-zh.html', 'index.html'] @@ -64,21 +59,31 @@ assert.equal(toggle.parallelPath('/BreakRL/', 'zh'), '/BreakRL/index-zh.html'); assert.equal(toggle.parallelPath('/BreakRL/index.html', 'zh'), '/BreakRL/index-zh.html'); assert.equal(toggle.parallelPath('/BreakRL/index-zh.html', 'en'), '/BreakRL/index.html'); assert.equal(toggle.parallelPath('/BreakRL/index.html', 'en'), null); -assert.equal(toggle.pageLanguage('/BreakRL/'), 'en'); -assert.equal(toggle.pageLanguage('/BreakRL/index.html'), 'en'); -assert.equal(toggle.pageLanguage('/BreakRL/index-zh.html'), 'zh'); -for (const shared of ['/BreakRL/search.html', '/BreakRL/genindex.html']) { - assert.equal(toggle.parallelPath(shared, 'zh'), null); - assert.equal(toggle.parallelPath(shared, 'en'), null); +// The three text pages are listed by hand, so a rename that skips the list +// would leave the toggle pointing at a page the book does not build. +for (const page of Object.keys(toggle.textPages)) { + const source = path.join(__dirname, '..', 'book', `${page.replace(/\.html$/, '')}.md`); + assert.ok(fs.existsSync(source), `no book source for text page ${page}`); +} + +// Sphinx's search and index pages, and the single-language demo, have no +// counterpart: the toggle must not offer a path off them. +for (const alone of ['/BreakRL/search.html', '/BreakRL/genindex.html', '/BreakRL/demo.html']) { + assert.equal(toggle.parallelPath(alone, 'zh'), null); + assert.equal(toggle.parallelPath(alone, 'en'), null); } +// Editions pair inside the chapter directories, where they are published +// together; a page named like a notebook anywhere else pairs with nothing. +assert.equal(toggle.parallelPath('/BreakRL/setup_experiments.html', 'en'), null); + assert.equal(toggle.parallelTarget('en'), '/BreakRL/notes/dqn/dqn_experiments_en.html?tab=loss#figure-3'); -assert.equal(toggle.pageLanguage('/BreakRL/notes/dqn/dqn_experiments.html'), 'zh'); -assert.equal(toggle.pageLanguage('/BreakRL/notes/dqn/dqn_experiments_en.html'), 'en'); -assert.equal(toggle.pageLanguage('/BreakRL/search.html'), null); -function preferredRedirect(pathname, savedLanguage) { +// A saved language sends the reader to the other edition of the page they are +// on. The page's own language comes from the `` the build emits, so +// each case states what the build would have written. +function preferredRedirect(pathname, savedLanguage, htmlLang) { let callback; let redirect = null; const testWindow = { @@ -101,7 +106,11 @@ function preferredRedirect(pathname, savedLanguage) { addEventListener(_event, handler) { callback = handler; }, - documentElement: {} + documentElement: { + getAttribute(name) { + return name === 'lang' ? htmlLang : null; + } + } }; vm.runInNewContext(source, { window: testWindow, document: testDocument }, { filename: sourcePath }); callback(); @@ -109,29 +118,21 @@ function preferredRedirect(pathname, savedLanguage) { } assert.equal( - preferredRedirect('/BreakRL/notes/dqn/dqn_experiments_en.html', 'zh'), + preferredRedirect('/BreakRL/notes/dqn/dqn_experiments_en.html', 'zh', 'en'), '/BreakRL/notes/dqn/dqn_experiments.html?from=deep-link#section' ); assert.equal( - preferredRedirect('/BreakRL/notes/dqn/dqn_experiments.html', 'en'), + preferredRedirect('/BreakRL/notes/dqn/dqn_experiments.html', 'en', 'zh-CN'), '/BreakRL/notes/dqn/dqn_experiments_en.html?from=deep-link#section' ); -assert.match(source, /saved \|\| pageLang \|\| LANG_EN/); -assert.match(source, /\[LANG_EN, LANG_ZH\]/); assert.equal( - preferredRedirect('/BreakRL/index.html', 'zh'), + preferredRedirect('/BreakRL/index.html', 'zh', 'en'), '/BreakRL/index-zh.html?from=deep-link#section' ); assert.equal( - preferredRedirect('/BreakRL/', 'zh'), + preferredRedirect('/BreakRL/', 'zh', 'en'), '/BreakRL/index-zh.html?from=deep-link#section' ); -assert.match(source, /index-zh\.html/); -assert.match(source, /if \(!cn \|\| !en\)/); -assert.match(source, /setAttribute\('aria-pressed'/); -assert.match(source, /role', 'group'/); -assert.match(source, /bd-sidebar-secondary/); -assert.doesNotMatch(source, /replace\(\/\\\.html\$\//); console.log(`language toggle regression checks passed (${pairs.length} explicit pairs)`);