From 9ac93e0c30a2d0053b73309d47ba08a8ff3cf46c Mon Sep 17 00:00:00 2001 From: NeoVand Date: Thu, 2 Jul 2026 00:23:20 -0500 Subject: [PATCH] =?UTF-8?q?feat(book):=20Parts=20III=20+=20IV=20become=20d?= =?UTF-8?q?ata=20=E2=80=94=20every=20teaching=20chapter=20is=20now=20block?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ch-noise, ch-generalize, and all six family-tree chapters migrate to block arrays, verbatim. With this, every teaching chapter in the guide is walkable data; only the zoo and the reference chapters remain markup. - new kinds: optcards (a chapter's optimizer-card slice — the cards themselves were already data) and frontier (the dashed closing box) - the anonymous regularization display is now formulas.regularizedLoss - the ravine heatmap and family tree move into figure snippets with their captions in blocks; the animated race is the second widget island (print fallback comes with the emitter) - the Part IV card chapters collapse to one-line shells — their prose lives in optimizerCards.ts where it always was - surfacing fix: the formula-display scrollbar styling was never globalized, so block-rendered displays had lost the themed thin scrollbar; grouped selectors now globalized alongside the base rule Verified: rendered text of all SEVENTEEN migrated chapters byte-identical to main in both themes (66,597 / 66,619 chars); frontier box, race player, and 14 race chips all render; 194 tests, svelte-check 0 errors 0 warnings. Co-Authored-By: Claude Fable 5 --- src/components/GuideBlocks.svelte | 10 + src/components/HelpModal.svelte | 723 ++++++++------------------ src/content/blocks.ts | 4 + src/content/chapters/ch-generalize.ts | 39 ++ src/content/chapters/ch-noise.ts | 42 ++ src/content/chapters/index.ts | 20 +- src/content/chapters/part4.ts | 62 +++ src/content/formulas.ts | 1 + 8 files changed, 388 insertions(+), 513 deletions(-) create mode 100644 src/content/chapters/ch-generalize.ts create mode 100644 src/content/chapters/ch-noise.ts create mode 100644 src/content/chapters/part4.ts diff --git a/src/components/GuideBlocks.svelte b/src/components/GuideBlocks.svelte index 8e78d92..10fa1a3 100644 --- a/src/components/GuideBlocks.svelte +++ b/src/components/GuideBlocks.svelte @@ -30,6 +30,7 @@ figure, conceptFig, widget, + cards, demo, onLesson, onPreset @@ -42,6 +43,8 @@ conceptFig?: Snippet<[string]>; /** Renders an interactive island (widget blocks). */ widget?: Snippet<[string]>; + /** Renders a family-tree chapter's optimizer-card slice (optcards blocks). */ + cards?: Snippet<[string]>; /** Chapter-demo override when the CTA isn't the slug's chapterPreset. */ demo?: { label: string; run: () => void }; onLesson?: (slug: string) => void; @@ -96,6 +99,13 @@ {:else if b.kind === 'widget'} {#if widget}{@render widget(b.id)}{/if} + {:else if b.kind === 'optcards'} + {#if cards}{@render cards(b.chapter)}{/if} + {:else if b.kind === 'frontier'} +
+
{b.title}
+

{@html richToHtml(b.text, dark)}

+
{:else if b.kind === 'concept'}
diff --git a/src/components/HelpModal.svelte b/src/components/HelpModal.svelte index ea2de67..22549a9 100644 --- a/src/components/HelpModal.svelte +++ b/src/components/HelpModal.svelte @@ -1026,7 +1026,82 @@ {/if} {/snippet} {#snippet chFigure(id: string, cap: string)} - {#if id === 'downhill-proof'} + {#if id === 'ravine-heat'} +
+
+ + (ravineViz = { ...ravineViz, ...p })} /> +
+ loss + {ravineFig.visMax.toFixed(2)} +
+ {ravineFig.visMin.toFixed(2)} +
+
+
{@html cap}
+
+ {:else if id === 'family-tree'} +
+ +
{@html cap}
+
+ {:else if id === 'downhill-proof'}
{:else}
- {#if id === 'derivative-secant'} + {#if id === 'noise-ball'} + + {:else if id === 'generalize-curves'} + + {:else if id === 'derivative-secant'}
+ + +
+ + + + + + + + {#each cmapStopColors(raceViz.colormap, 8, gTheme) as col, i}{/each} + + + + + + + {#each raceCont as cp} + + {/each} + + + {#each raceArrows as a}{/each} + {#each raceDemo.racers as r (r.id)} + {@const hot = raceHover === r.id} + {@const dim = raceHover !== null && !hot} + + + + {/each} + + + + {#each raceDemo.racers as r (r.id)} + {@const hot = raceHover === r.id} + {@const dim = raceHover !== null && !hot} + + + + {/each} + + + + {raceDemo.visMax.toFixed(0)} + {raceDemo.visMin.toFixed(1)} + + + + (raceViz = { ...raceViz, ...p })} /> +
+ {#each raceDemo.racers as r (r.id)} + + {/each} +
+
+ All {raceDemo.racers.length} optimizers, same start, same ravine — each running its real + update rule, the dots arriving in their true step counts. Click a name to toggle it, + hover to spotlight one. Watch the contrasts: Newton nearly teleports, GD rattles wall to + wall, Momentum glides past it, and RAdam’s self-warmup keeps it cautious to the end. Use + the play / restart controls to watch it again. +
+
+ {:else if id === 'schedule-grid'}
{#each scheduleCurves as s (s.id)}
@@ -1301,227 +1489,14 @@
Part III · Descent in the real world

Mini-batches & the S in SGD

-

- Every gradient so far has been the true one — measured on all your - data at once. That is full-batch descent: the Batch - size dial set to All. It gives the cleanest possible arrow, and it - is the most expensive thing you can do, because every single step has to read every - single data point. -

-

- Real datasets are far too large for that, so instead you estimate the - gradient from a small random batch — a handful of points, freshly - resampled each step. The arrow you get back is noisy: it jitters - around the true downhill, because a different handful would have pulled in a slightly - different direction. But it is cheap, and — this is the quiet miracle that makes modern - training possible — it still points the right way on average. Averaging your - way downhill through that noise is the S (stochastic) in - SGD, stochastic gradient descent. (Two words of vocabulary while we are - here: one batch update is an iteration or step; one full sweep through - the whole dataset is an epoch.) -

-

- Slide the Batch size down from All toward 1 and a - faint fan of arrows opens at the marker: each ray is the gradient a - different random batch would have handed you, so the width of the fan is the noise - itself. The fewer points in the batch, the wider it spreads — and it spreads in a - very specific way — the same law that steadies dice: average four rolls and the - result wobbles about half as much as a single roll. The error of an average - shrinks only with the square root - of how many samples go into it, so a batch of 4 is roughly twice as steady as a batch - of 1, and you need 16 to halve the noise again. That is the law of diminishing returns - behind every batch-size choice: a batch of 32 already looks almost as calm as the full - dataset, for a fraction of the cost. -

- -

- And the noise is not pure cost. A little jitter is genuinely useful: a - noisy step can rattle the marker out of a shallow dip or a flat saddle that a perfectly - smooth step would have settled into and never left, and the constant restlessness tends - to steer a run toward wide, gentle basins — the forgiving kind that generalize - to new data — rather than narrow, brittle cracks. This is why a touch of stochasticity - is often kept on purpose, even when the full gradient is affordable. -

- -

- The bill comes due at the end. Because the gradient never goes quiet, SGD never - fully stops: near the bottom it stops descending and starts orbiting, - buzzing around the minimum inside a small noise ball whose radius grows - with both the step size {@html tex(String.raw`\gamma`)} and the width of the fan. On the loss curve it shows up as a - fuzzy band rather than a clean line that flatlines — the run has arrived, but it - can’t hold still. This is where the schedule from the last chapter earns - its keep: a {@html tex(String.raw`\gamma`)} bled toward zero draws that ball in tight, turning the restless buzz into a - soft landing. Under noise, decay isn’t a luxury — it is how a stochastic run converges - at all. -

-
- -
- Under noisy gradients the run never quite stops — it orbits the minimum in a cloud whose - radius grows with {@html tex(String.raw`\gamma`)} (left). Bleed {@html tex(String.raw`\gamma`)} toward zero and the cloud draws in to a point (right): - the schedule, doing its quiet job. -
-
-

- Watch it: set a small Batch size so the loss settles into a fuzzy band on - Const, then switch the schedule to Cosine and see the - band pinch shut over the final steps. -

- startLessonFromChapter('ch-noise')} - demo={chapterPresets['ch-noise'] ? () => runPreset('ch-noise') : null} - /> - {#if chRefs['ch-noise']} -
- Further reading - {#each chRefs['ch-noise'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} +

Training loss isn’t the goal

- -

- Every chapter so far has worked to drive the training loss down. But that number - is only a stand-in for what we actually want. We don’t care about fitting the data we - already have — we care about predicting data we haven’t seen. Doing well - on new data is generalization, and it is the whole point. -

-

- The loss we minimize is the average error over the training set — the empirical - risk — but the real target is the average error over all future data, - the true risk. With limited or noisy data the two come apart. Push the - training loss too low and the model starts memorizing the quirks and noise of this - sample: training loss keeps falling while error on held-out data turns and climbs. That - divergence is overfitting. -

- -
- -
- Training loss (green) keeps falling; test loss (amber), measured on held-out data, bottoms - out and then rises as the model begins fitting noise. The dip is where you’d want to stop. -
-
- -

- Two fixes follow directly. The first is to measure the gap: hold out part of the - data as a test (or validation) set, and watch its loss alongside the - training loss — that is the second curve in the Loss History panel. The - second is early stopping: end training at the test-loss minimum rather - than the training-loss minimum. It is the simplest regularizer there is, and — for a - run started near zero — in the quadratic case it is provably close to an explicit - weight penalty (Bishop, 1995; Goodfellow et al., 2016, §7.8). -

-

- That penalty is regularization: instead of minimizing the loss alone, add - a term that prefers smaller, simpler parameters, -

-
{@html texD(String.raw`\min_{\boldsymbol{\theta}}\;\; \mathcal{L}(\boldsymbol{\theta}) \;+\; \tfrac{\lambda}{2}\,\lVert \boldsymbol{\theta}\rVert^2`)}
-

- where {@html tex(String.raw`\lambda`)} sets how hard to pull toward zero. (An unrelated - {@html tex(String.raw`\lambda`)}, by the way — not the curvature - {@html tex(String.raw`\lambda_{\max}`)} from the learning-rate chapter. The alphabet is - small and the field is greedy.) For plain SGD the - gradient of that penalty is exactly weight decay — - {@html tex(String.raw`\boldsymbol{\theta} \leftarrow (1-\gamma\lambda)\,\boldsymbol{\theta} - \gamma\nabla\mathcal{L}`)} - — shrinking every weight a touch each step (Krogh & Hertz, 1991). Keep this - {@html tex(String.raw`\lambda`)} in mind: you will meet it again on - AdamW in the family tree, which decouples the decay from the - adaptive scaling so it behaves like a true penalty again. -

-

- Geometry has the last word, and it loops back to the noise chapter. Not all minima - generalize equally: a wide, flat basin is forgiving — small shifts in the data - barely move the loss — while a sharp one is brittle. Flat minima tend to - generalize better (Hochreiter & Schmidhuber, 1997), the restless noise of small-batch - SGD tends to settle into them, and very large batches tend to find sharper minima with a - measurable generalization gap (Keskar et al., 2017). So the real target was never the exact - bottom of the training bowl — it is a low, wide region that also sits low on data - you will never see. Optimization gets you down; generalization decides whether down was - worth reaching. -

- - {#if chapterPresets['ch-generalize']} - runPreset('ch-generalize')} demoLabel={chapterPresets['ch-generalize'].title} /> - {/if} - {#if chRefs['ch-generalize']} -
- Further reading - {#each chRefs['ch-generalize'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} +
Part IV · The optimizer family tree

The ravine, and the race

-

- Plain gradient descent has one recurring nemesis: the ravine — a - valley far steeper across than along. The {@html tex(String.raw`\gamma`)} that’s safe on the steep walls is - hopeless along the gentle floor, so the marker rattles wall to wall. Every optimizer - in the picker is a patch for that pain (or the new pain the last patch created) — - 170 years of fix what just broke: a single trunk of fixes that, once it - reaches Adam, finally splits into the branches still being explored today. The - picker is grouped to match. -

-

- That ravine has a precise name: ill-conditioning. A smooth bowl curves - at two rates — gently along its floor ({@html tex(String.raw`\lambda_{\min}`)}) and - steeply across it ({@html tex(String.raw`\lambda_{\max}`)}) — and their ratio is the - condition number {@html tex(String.raw`\kappa = \lambda_{\max}/\lambda_{\min}`)}. - A round bowl has {@html tex(String.raw`\kappa = 1`)} and one good step reaches the bottom; - a long, thin ravine has a huge {@html tex(String.raw`\kappa`)}, and that one number sets - how slowly you converge. Even with the best fixed step, - {@html tex(String.raw`\gamma = 2/(\lambda_{\min}+\lambda_{\max})`)}, each move closes the - gap to the minimum by only a factor {@html tex(String.raw`(\kappa-1)/(\kappa+1)`)} — which - creeps toward 1 as {@html tex(String.raw`\kappa`)} grows, so a stretched valley crawls no - matter how you tune {@html tex(String.raw`\gamma`)}. Momentum sharpens that to roughly - {@html tex(String.raw`(\sqrt{\kappa}-1)/(\sqrt{\kappa}+1)`)}, a - {@html tex(String.raw`\sqrt{\kappa}`)} speed-up — the first hint of why the whole family - below exists. -

-
-
- - (ravineViz = { ...ravineViz, ...p })} /> -
- loss - {ravineFig.visMax.toFixed(2)} -
- {ravineFig.visMin.toFixed(2)} -
-
-
- The ravine: a valley far steeper across than along. One safe step size makes plain GD - (white) rattle wall to wall while it crawls along the floor; momentum (violet) builds - speed down the valley and glides to the minimum. -
-
-

- Every fix that follows is a leaf on one tree. Here is the whole lineage at a glance — - 170 years from Cauchy’s root to today’s canopy, each branch running parent → child: -

-
- -
- Every leaf is an optimizer in the picker; branches run parent → child, and the dashed - violet strand marks where momentum and the adaptive line merge into Adam. Colours match - the race below, and new methods join the canopy as the field grows. -
-
-

- Here they are racing on the same ravine from the same start — every one running its - real update rule, the dots arriving in their true step counts. Click a name to add or - remove it; hover one to pick it out of the pack: -

- -
-
- - -
- - - - - - - - {#each cmapStopColors(raceViz.colormap, 8, gTheme) as col, i}{/each} - - - - - - - {#each raceCont as cp} - - {/each} - - - {#each raceArrows as a}{/each} - {#each raceDemo.racers as r (r.id)} - {@const hot = raceHover === r.id} - {@const dim = raceHover !== null && !hot} - - - - {/each} - - - - {#each raceDemo.racers as r (r.id)} - {@const hot = raceHover === r.id} - {@const dim = raceHover !== null && !hot} - - - - {/each} - - - - {raceDemo.visMax.toFixed(0)} - {raceDemo.visMin.toFixed(1)} - - - - (raceViz = { ...raceViz, ...p })} /> -
- {#each raceDemo.racers as r (r.id)} - - {/each} -
-
- All {raceDemo.racers.length} optimizers, same start, same ravine — each running its real - update rule, the dots arriving in their true step counts. Click a name to toggle it, - hover to spotlight one. Watch the contrasts: Newton nearly teleports, GD rattles wall to - wall, Momentum glides past it, and RAdam’s self-warmup keeps it cautious to the end. Use - the play / restart controls to watch it again. -
-
- - {#each optTree.filter((o) => o.chapter === 'ch-ravine') as c (c.name)} - {@render optCard(c)} - {/each} - - startLessonFromChapter('ch-ravine')} - demo={raceExperiment ? () => runExperiment(raceExperiment) : null} - /> - {#if chRefs['ch-ravine']} -
- Further reading - {#each chRefs['ch-ravine'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} + runExperiment(raceExperiment) } : undefined} onPreset={runPreset} onLesson={startLessonFromChapter} />

Momentum & Nesterov

- {#each optTree.filter((o) => o.chapter === 'ch-momentum') as c (c.name)} - {@render optCard(c)} - {/each} - - startLessonFromChapter('ch-momentum')} - demo={narrowValleyExperiment ? () => runExperiment(narrowValleyExperiment) : null} - /> - {#if chRefs['ch-momentum']} -
- Further reading - {#each chRefs['ch-momentum'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} + runExperiment(narrowValleyExperiment) } : undefined} onPreset={runPreset} onLesson={startLessonFromChapter} />

A learning rate per parameter

- {#each optTree.filter((o) => o.chapter === 'ch-adaptive') as c (c.name)} - {@render optCard(c)} - {/each} - - startLessonFromChapter('ch-adaptive')} - demo={adagradFreezeExperiment ? () => runExperiment(adagradFreezeExperiment) : null} - /> + runExperiment(adagradFreezeExperiment) } : undefined} onPreset={runPreset} onLesson={startLessonFromChapter} />

Adam — and the fork

- {#each optTree.filter((o) => o.chapter === 'ch-adam') as c (c.name)} - {@render optCard(c)} - {/each} - - {#if rotatedRavineExperiment} - runExperiment(rotatedRavineExperiment)} demoLabel="Rotate the ravine 45°" /> - {/if} + runExperiment(rotatedRavineExperiment) } : undefined} onPreset={runPreset} onLesson={startLessonFromChapter} />

Second order: Newton & Sophia

- {#each optTree.filter((o) => o.chapter === 'ch-second-order') as c (c.name)} - {@render optCard(c)} - {/each} - - {#if chapterPresets['ch-second-order']} - runPreset('ch-second-order')} demoLabel={chapterPresets['ch-second-order'].title} /> - {/if} +

The last knob

- {#each optTree.filter((o) => o.chapter === 'ch-self-tuning') as c (c.name)} - {@render optCard(c)} - {/each} - - -
-
The frontier — and why it isn’t in the picker
-

- The optimizers winning 2025’s biggest training runs — Muon (used - to train Kimi K2 and GLM), Shampoo, and SOAP — - share a trick this playground can’t show. They treat a layer’s weights as a - matrix and precondition across it: Muon (momentum - orthogonalized by Newton–Schulz) straightens the momentum matrix, Shampoo and - SOAP whiten it. With only two independent numbers, - {@html tex(String.raw`\alpha`)} and {@html tex(String.raw`\beta`)}, there is no matrix to exploit — strip the structure away and they collapse - to methods already in the list. That matrix structure is exactly why they scale to - billions of parameters, and exactly why a two-parameter sandbox is the wrong stage - for them. To meet them you have to leave the playground — which is a fair note to - end the tree on. -

-
- - {#if chapterPresets['ch-self-tuning']} - runPreset('ch-self-tuning')} demoLabel={chapterPresets['ch-self-tuning'].title} /> - {/if} +
@@ -2565,11 +2264,11 @@ /* Wide formulas may scroll sideways; keep the scrollbar thin and themed, and never let overflow-x:auto spawn a stray vertical scrollbar. */ - .formula-display::-webkit-scrollbar, + .reading-column :global(.formula-display::-webkit-scrollbar), .opt-formula::-webkit-scrollbar { height: 5px; } - .formula-display::-webkit-scrollbar-thumb, + .reading-column :global(.formula-display::-webkit-scrollbar-thumb), .opt-formula::-webkit-scrollbar-thumb { background: var(--color-border); border-radius: 3px; } - .formula-display::-webkit-scrollbar-track, + .reading-column :global(.formula-display::-webkit-scrollbar-track), .opt-formula::-webkit-scrollbar-track { background: transparent; } /* ---------- Knob bullets ---------- */ @@ -2846,20 +2545,20 @@ .opt-lead :global(strong) { color: var(--color-text-primary); font-weight: 650; } /* The 2025 matrix-optimizer note: real, relevant, deliberately not runnable. */ - .opt-frontier { + .reading-column :global(.opt-frontier) { margin-top: 1.2rem; border: 1px dashed var(--color-border); border-radius: 10px; background: var(--color-bg-primary); padding: 0.8rem 0.95rem; } - .opt-frontier-title { + .reading-column :global(.opt-frontier-title) { font-weight: 700; font-size: 0.9rem; color: var(--color-text-primary); margin-bottom: 0.35rem; } - .opt-frontier p { + .reading-column :global(.opt-frontier p) { font-size: 0.85rem; line-height: 1.6; color: var(--color-text-secondary); diff --git a/src/content/blocks.ts b/src/content/blocks.ts index 129611f..f31d767 100644 --- a/src/content/blocks.ts +++ b/src/content/blocks.ts @@ -48,6 +48,10 @@ export type Block = | { kind: 'hd'; text: Rich } /** A figure: the id names an app-side (later: computed) visual; the caption lives here. */ | { kind: 'figure'; id: string; caption: Rich } + /** The optimizer story cards for one family-tree chapter (data in optimizerCards.ts). */ + | { kind: 'optcards'; chapter: string } + /** The dashed closing box — the frontier beyond the playground. */ + | { kind: 'frontier'; title: string; text: Rich } /** * An interactive island with no print analogue of its own (the schedule * gallery, later the race) — the shell supplies it; print will use a diff --git a/src/content/chapters/ch-generalize.ts b/src/content/chapters/ch-generalize.ts new file mode 100644 index 0000000..888463f --- /dev/null +++ b/src/content/chapters/ch-generalize.ts @@ -0,0 +1,39 @@ +import type { Block } from '../blocks'; + +/** Chapter 11 · Training loss isn’t the goal — generalization, the whole point. */ +export const chGeneralize: Block[] = [ + { + kind: 'p', + text: 'Every chapter so far has worked to drive the *training* loss down. But that number is only a stand-in for what we actually want. We don’t care about fitting the data we already have — we care about predicting data we **haven’t seen**. Doing well on new data is **generalization**, and it is the whole point.' + }, + { + kind: 'p', + text: 'The loss we minimize is the average error over the training set — the **empirical risk** — but the real target is the average error over *all* future data, the **true risk**. With limited or noisy data the two come apart. Push the training loss too low and the model starts memorizing the quirks and noise of *this* sample: training loss keeps falling while error on held-out data turns and climbs. That divergence is **overfitting**.' + }, + { + kind: 'figure', + id: 'generalize-curves', + caption: 'Training loss (green) keeps falling; test loss (amber), measured on held-out data, bottoms out and then rises as the model begins fitting noise. The dip is where you’d want to stop.' + }, + { + kind: 'p', + text: 'Two fixes follow directly. The first is to *measure* the gap: hold out part of the data as a **test** (or validation) set, and watch its loss alongside the training loss — that is the second curve in the **Loss History** panel. The second is **early stopping**: end training at the test-loss minimum rather than the training-loss minimum. It is the simplest regularizer there is, and — for a run started near zero — in the quadratic case it is provably close to an explicit weight penalty (Bishop, 1995; Goodfellow et al., 2016, §7.8).' + }, + { + kind: 'p', + text: 'That penalty is **regularization**: instead of minimizing the loss alone, add a term that prefers smaller, simpler parameters,' + }, + { kind: 'display', formula: 'regularizedLoss', center: true }, + { + kind: 'p', + text: 'where $\\lambda$ sets how hard to pull toward zero. (An unrelated $\\lambda$, by the way — not the curvature $\\lambda_{\\max}$ from the learning-rate chapter. The alphabet is small and the field is greedy.) For plain SGD the gradient of that penalty is exactly **weight decay** — $\\boldsymbol{\\theta} \\leftarrow (1-\\gamma\\lambda)\\,\\boldsymbol{\\theta} - \\gamma\\nabla\\mathcal{L}$ — shrinking every weight a touch each step (Krogh & Hertz, 1991). Keep this $\\lambda$ in mind: you will meet it again on **AdamW** in the family tree, which decouples the decay from the adaptive scaling so it behaves like a true penalty again.' + }, + { + kind: 'p', + text: 'Geometry has the last word, and it loops back to the noise chapter. Not all minima generalize equally: a *wide, flat* basin is forgiving — small shifts in the data barely move the loss — while a *sharp* one is brittle. Flat minima tend to generalize better (Hochreiter & Schmidhuber, 1997), the restless noise of small-batch SGD tends to settle into them, and very large batches tend to find sharper minima with a measurable generalization gap (Keskar et al., 2017). So the real target was never the exact bottom of the training bowl — it is a low, *wide* region that also sits low on data you will never see. Optimization gets you down; generalization decides whether down was worth reaching.' + }, + { + kind: 'hd', + text: 'Two honest asterisks on this tidy story. First, “flat” is slippery: a network can be rescaled — same function, same predictions — while its measured sharpness changes arbitrarily, so naive flatness can’t be the whole answer (Dinh et al., 2017). Second, at scale good minima aren’t isolated dips like the ones drawn here: they connect into long low-loss valleys you can walk between without climbing (Garipov et al., 2018). The intuition survives — restless SGD prefers forgiving regions — but hold it as a compass, not a theorem.' + } +]; diff --git a/src/content/chapters/ch-noise.ts b/src/content/chapters/ch-noise.ts new file mode 100644 index 0000000..c97d04e --- /dev/null +++ b/src/content/chapters/ch-noise.ts @@ -0,0 +1,42 @@ +import type { Block } from '../blocks'; + +/** Chapter 10 · Mini-batches & the S in SGD — noisy arrows, √n, the noise ball. */ +export const chNoise: Block[] = [ + { + kind: 'p', + text: 'Every gradient so far has been the **true** one — measured on all your data at once. That is **full-batch** descent: the **Batch size** dial set to *All*. It gives the cleanest possible arrow, and it is the most expensive thing you can do, because every single step has to read every single data point.' + }, + { + kind: 'p', + text: 'Real datasets are far too large for that, so instead you *estimate* the gradient from a small random **batch** — a handful of points, freshly resampled each step. The arrow you get back is **noisy**: it jitters around the true downhill, because a different handful would have pulled in a slightly different direction. But it is cheap, and — this is the quiet miracle that makes modern training possible — it still points the right way *on average*. Averaging your way downhill through that noise is the **S** (stochastic) in **SGD**, stochastic gradient descent. (Two words of vocabulary while we are here: one batch update is an **iteration** or step; one full sweep through the whole dataset is an **epoch**.)' + }, + { + kind: 'p', + text: 'Slide the **Batch size** down from *All* toward *1* and a faint **fan** of arrows opens at the marker: each ray is the gradient a different random batch would have handed you, so the *width of the fan is the noise itself.* The fewer points in the batch, the wider it spreads — and it spreads in a very specific way — the same law that steadies dice: average four rolls and the result wobbles about half as much as a single roll. The error of an average shrinks only with the *square root* of how many samples go into it, so a batch of 4 is roughly twice as steady as a batch of 1, and you need 16 to halve the noise again. That is the law of diminishing returns behind every batch-size choice: a batch of 32 already looks almost as calm as the full dataset, for a fraction of the cost.' + }, + { + kind: 'hd', + text: 'At scale this √n law becomes an economic one. The useful ratio is noise to signal: below a problem-specific *critical batch size*, doubling the batch lets you (roughly) double $\\gamma$ for the same trajectory — the linear-scaling rule behind giant training runs; above it, extra data per step buys calm the run no longer needs (Goyal et al., 2017; McCandlish et al., 2018). Bigger is not better — bigger is *quieter*, and quiet has a price and a ceiling.' + }, + { + kind: 'p', + text: 'And the noise is not pure cost. A little jitter is genuinely **useful**: a noisy step can rattle the marker out of a shallow dip or a flat saddle that a perfectly smooth step would have settled into and never left, and the constant restlessness tends to steer a run toward *wide, gentle* basins — the forgiving kind that generalize to new data — rather than narrow, brittle cracks. This is why a touch of stochasticity is often kept on purpose, even when the full gradient is affordable.' + }, + { + kind: 'hd', + text: 'The fan tells the truth in 2-D, but up there it would look strange: two random directions among a million axes are almost always nearly *perpendicular* — there are countless ways to be orthogonal and only one way to agree. So gradient noise mostly pushes *sideways*, at right angles to the true downhill, rather than backwards against it. A noisy run drifts and wanders far more than it backtracks — one reason SGD keeps making progress even when individual arrows look hopeless.' + }, + { + kind: 'p', + text: 'The bill comes due at the *end*. Because the gradient never goes quiet, SGD never fully stops: near the bottom it stops descending and starts **orbiting**, buzzing around the minimum inside a small **noise ball** whose radius grows with both the step size $\\gamma$ and the width of the fan. On the loss curve it shows up as a fuzzy *band* rather than a clean line that flatlines — the run has arrived, but it can’t hold still. This is where the **schedule** from the last chapter earns its keep: a $\\gamma$ bled toward zero draws that ball in tight, turning the restless buzz into a soft landing. Under noise, decay isn’t a luxury — it is *how a stochastic run converges at all.*' + }, + { + kind: 'figure', + id: 'noise-ball', + caption: 'Under noisy gradients the run never quite stops — it orbits the minimum in a cloud whose radius grows with $\\gamma$ (left). Bleed $\\gamma$ toward zero and the cloud draws in to a point (right): the schedule, doing its quiet job.' + }, + { + kind: 'look', + text: 'Watch it: set a small **Batch size** so the loss settles into a fuzzy band on **Const**, then switch the schedule to **Cosine** and see the band pinch shut over the final steps.' + } +]; diff --git a/src/content/chapters/index.ts b/src/content/chapters/index.ts index 9ab85de..5c824f3 100644 --- a/src/content/chapters/index.ts +++ b/src/content/chapters/index.ts @@ -14,6 +14,16 @@ import { chStep } from './ch-step'; import { chGamma } from './ch-gamma'; import { chCurvature } from './ch-curvature'; import { chSchedule } from './ch-schedule'; +import { chNoise } from './ch-noise'; +import { chGeneralize } from './ch-generalize'; +import { + chRavine, + chMomentum, + chAdaptive, + chAdam, + chSecondOrder, + chSelfTuning +} from './part4'; export const chapterBlocks: Record = { 'ch-bowl': chBowl, @@ -24,5 +34,13 @@ export const chapterBlocks: Record = { 'ch-step': chStep, 'ch-gamma': chGamma, 'ch-curvature': chCurvature, - 'ch-schedule': chSchedule + 'ch-schedule': chSchedule, + 'ch-noise': chNoise, + 'ch-generalize': chGeneralize, + 'ch-ravine': chRavine, + 'ch-momentum': chMomentum, + 'ch-adaptive': chAdaptive, + 'ch-adam': chAdam, + 'ch-second-order': chSecondOrder, + 'ch-self-tuning': chSelfTuning }; diff --git a/src/content/chapters/part4.ts b/src/content/chapters/part4.ts new file mode 100644 index 0000000..2b00f78 --- /dev/null +++ b/src/content/chapters/part4.ts @@ -0,0 +1,62 @@ +import type { Block } from '../blocks'; + +/** + * Part IV · The optimizer family tree. The prose of these chapters lives in + * the story cards (optimizerCards.ts); the chapters themselves are thin — + * the ravine opener carries the conditioning story and the race, the rest + * are their card slices, and the last closes with the frontier box. + */ + +/** Chapter 12 · The ravine, and the race — the part opener. */ +export const chRavine: Block[] = [ + { + kind: 'p', + text: 'Plain gradient descent has one recurring nemesis: the **ravine** — a valley far steeper across than along. The $\\gamma$ that’s safe on the steep walls is hopeless along the gentle floor, so the marker rattles wall to wall. Every optimizer in the picker is a patch for that pain (or the new pain the last patch created) — 170 years of *fix what just broke*: a single trunk of fixes that, once it reaches Adam, finally splits into the branches still being explored today. The picker is grouped to match.' + }, + { + kind: 'p', + text: 'That ravine has a precise name: **ill-conditioning**. A smooth bowl curves at two rates — gently along its floor ($\\lambda_{\\min}$) and steeply across it ($\\lambda_{\\max}$) — and their ratio is the **condition number** $\\kappa = \\lambda_{\\max}/\\lambda_{\\min}$. A round bowl has $\\kappa = 1$ and one good step reaches the bottom; a long, thin ravine has a huge $\\kappa$, and that one number sets how slowly you converge. Even with the best fixed step, $\\gamma = 2/(\\lambda_{\\min}+\\lambda_{\\max})$, each move closes the gap to the minimum by only a factor $(\\kappa-1)/(\\kappa+1)$ — which creeps toward 1 as $\\kappa$ grows, so a stretched valley crawls no matter how you tune $\\gamma$. Momentum sharpens that to roughly $(\\sqrt{\\kappa}-1)/(\\sqrt{\\kappa}+1)$, a $\\sqrt{\\kappa}$ speed-up — the first hint of why the whole family below exists.' + }, + { + kind: 'figure', + id: 'ravine-heat', + caption: 'The ravine: a valley far steeper across than along. One safe step size makes plain GD (white) rattle wall to wall while it crawls along the floor; momentum (violet) builds speed down the valley and glides to the minimum.' + }, + { + kind: 'p', + text: 'Every fix that follows is a leaf on one tree. Here is the whole lineage at a glance — 170 years from Cauchy’s root to today’s canopy, each branch running parent → child:' + }, + { + kind: 'figure', + id: 'family-tree', + caption: 'Every leaf is an optimizer in the picker; branches run parent → child, and the dashed violet strand marks where momentum and the adaptive line merge into Adam. Colours match the race below, and new methods join the canopy as the field grows.' + }, + { + kind: 'p', + text: 'Here they are racing on the same ravine from the same start — every one running its real update rule, the dots arriving in their true step counts. Click a name to add or remove it; hover one to pick it out of the pack:' + }, + { kind: 'widget', id: 'ravine-race' }, + { kind: 'optcards', chapter: 'ch-ravine' } +]; + +/** Chapter 13 · Momentum & Nesterov. */ +export const chMomentum: Block[] = [{ kind: 'optcards', chapter: 'ch-momentum' }]; + +/** Chapter 14 · A learning rate per parameter. */ +export const chAdaptive: Block[] = [{ kind: 'optcards', chapter: 'ch-adaptive' }]; + +/** Chapter 15 · Adam — and the fork. */ +export const chAdam: Block[] = [{ kind: 'optcards', chapter: 'ch-adam' }]; + +/** Chapter 16 · Second order: Newton & Sophia. */ +export const chSecondOrder: Block[] = [{ kind: 'optcards', chapter: 'ch-second-order' }]; + +/** Chapter 17 · The last knob — Prodigy, and the frontier past the playground. */ +export const chSelfTuning: Block[] = [ + { kind: 'optcards', chapter: 'ch-self-tuning' }, + { + kind: 'frontier', + title: 'The frontier — and why it isn’t in the picker', + text: 'The optimizers winning 2025’s biggest training runs — **Muon** (used to train Kimi K2 and GLM), **Shampoo**, and **SOAP** — share a trick this playground can’t show. They treat a layer’s weights as a *matrix* and precondition *across* it: Muon (*momentum orthogonalized by Newton–Schulz*) straightens the momentum matrix, Shampoo and SOAP whiten it. With only two independent numbers, $\\alpha$ and $\\beta$, there is no matrix to exploit — strip the structure away and they collapse to methods already in the list. That matrix structure is exactly why they scale to billions of parameters, and exactly why a two-parameter sandbox is the wrong stage for them. To meet them you have to leave the playground — which is a fair note to end the tree on.' + } +]; diff --git a/src/content/formulas.ts b/src/content/formulas.ts index b3f16b9..d8102dd 100644 --- a/src/content/formulas.ts +++ b/src/content/formulas.ts @@ -11,6 +11,7 @@ export const formulas = { gradientDefinition: String.raw`\nabla \mathcal{L} = \begin{bmatrix} \partial \mathcal{L}/\partial \alpha \\[2pt] \partial \mathcal{L}/\partial \beta \end{bmatrix}`, stepRule: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \nabla \mathcal{L}`, stability: String.raw`\gamma < \frac{2}{\lambda_{\max}}`, + regularizedLoss: String.raw`\min_{\boldsymbol{\theta}}\;\; \mathcal{L}(\boldsymbol{\theta}) \;+\; \tfrac{\lambda}{2}\,\lVert \boldsymbol{\theta}\rVert^2`, // The derivative chapter: nudge-and-divide, then let the nudge shrink. derivativeLimit: String.raw`\frac{d\mathcal{L}}{d\alpha} \;=\; \lim_{h \to 0}\; \frac{\mathcal{L}(\alpha + h) - \mathcal{L}(\alpha)}{h}`, partialDef: String.raw`\frac{\partial \mathcal{L}}{\partial \alpha} \;=\; \lim_{h \to 0}\; \frac{\mathcal{L}(\alpha + h,\; \beta) - \mathcal{L}(\alpha,\; \beta)}{h}`,