diff --git a/src/components/GuideBlocks.svelte b/src/components/GuideBlocks.svelte index 59c1897..8e78d92 100644 --- a/src/components/GuideBlocks.svelte +++ b/src/components/GuideBlocks.svelte @@ -29,15 +29,21 @@ blocks, figure, conceptFig, + widget, + demo, onLesson, onPreset }: { slug: string; blocks: Block[]; - /** Renders a full
for an id, given the caption HTML. */ + /** Renders a full
for an id, given the caption HTML. */ figure?: Snippet<[string, string]>; - /** Renders the small illustration inside a concept box. */ + /** Renders the illustration inside a concept box (or an overlay's backdrop). */ conceptFig?: Snippet<[string]>; + /** Renders an interactive island (widget blocks). */ + widget?: Snippet<[string]>; + /** Chapter-demo override when the CTA isn't the slug's chapterPreset. */ + demo?: { label: string; run: () => void }; onLesson?: (slug: string) => void; onPreset?: (slug: string) => void; } = $props(); @@ -58,7 +64,38 @@ {:else if b.kind === 'look'}

{@html richToHtml(b.text, dark)}

{:else if b.kind === 'display'} -
{@html texD(formulas[b.formula])}
+
{@html texD(formulas[b.formula])}
+ {:else if b.kind === 'recipe'} +
{@html richToHtml(b.text, dark)}
+ {:else if b.kind === 'list'} + + {:else if b.kind === 'conceptOverlay'} +
+ {#if conceptFig}{@render conceptFig(b.fig)}{/if} +
+
+

{b.title}

+ {#each b.paras as t}

{@html richToHtml(t, dark)}

{/each} +
+
+ {:else if b.kind === 'proof'} + {@const lastP = b.blocks.reduce((acc, x, i) => (x.kind === 'p' ? i : acc), -1)} +
+
{b.title}
+ {#each b.blocks as pb, i} + {#if pb.kind === 'p'} +

{@html richToHtml(pb.text, dark)}{#if i === lastP}{' '}∎{/if}

+ {:else if pb.kind === 'display'} +
{@html texD(formulas[pb.formula])}
+ {:else if pb.kind === 'figure'} + {#if figure}{@render figure(pb.id, richToHtml(pb.caption, dark))}{/if} + {/if} + {/each} +
+ {:else if b.kind === 'widget'} + {#if widget}{@render widget(b.id)}{/if} {:else if b.kind === 'concept'}
@@ -77,12 +114,12 @@ {/if} {/each} -{#if lessonId || preset} +{#if lessonId || preset || demo} onLesson(slug) : null} - demo={preset && onPreset ? () => onPreset(slug) : null} - demoLabel={preset?.title ?? 'Watch a quick demo'} + demo={demo ? demo.run : preset && onPreset ? () => onPreset(slug) : null} + demoLabel={demo?.label ?? preset?.title ?? 'Watch a quick demo'} /> {/if} {#if chRefs[slug]} diff --git a/src/components/HelpModal.svelte b/src/components/HelpModal.svelte index 3731031..ea2de67 100644 --- a/src/components/HelpModal.svelte +++ b/src/components/HelpModal.svelte @@ -976,256 +976,7 @@ figure id + owns the caption; these snippets supply the drawing (so figure styles stay in this component's scope). --> {#snippet conceptFig(id: string)} - {#if id === 'bowl-1d'} - - - - minimum - loss ↑ - - {/if} - {/snippet} - {#snippet chFigure(id: string, cap: string)} -
- {#if id === 'landscape-two-views'} - - {:else if id === 'shapes-basins-saddle'} - - {/if} -
{@html cap}
-
- {/snippet} - -
-
Part I · The landscape
-

The bottom of a bowl

- -
- - -
-

Loss is a landscape

- -
- - -
-

When the bowl isn’t a bowl

- -
- - -
-
Part II · Walking downhill
-

How steep, exactly?

- -

- Part I kept saying slope and trusted your legs to know what it meant. Before - the walking starts in earnest, let’s put a number on it — because the number is the - whole trick, and you can build it yourself with nothing but a subtraction and a - division. -

-

- Here is the move. Stand somewhere on a 1-D loss curve — say {@html tex(String.raw`\alpha = 2`)} on - Fit a Slope, whose loss happens to be {@html tex(String.raw`\mathcal{L}(\alpha)=\alpha^2`)}, so {@html tex(String.raw`\mathcal{L}(2)=4`)}. - Nudge the knob by some small amount h, and divide the - loss’s response by the nudge. Nudge by h = 0.1 and the loss climbs from - 4 to 4.41 — a rise of 0.41 over a run of 0.1: ratio 4.1. Try - h = 0.01: the ratio comes out 4.01. Try 0.001: 4.001. - The nudges are vanishing, but the ratio isn’t wandering — it is - settling, and the number it settles on is 4. -

-
- Nudge. Measure the response. Divide. Then let the nudge shrink — the number the - ratio settles on is the derivative: the slope of the loss - at a point. -
-
{@html texD(formulas.derivativeLimit)}
-

- Read it slowly, once: the fraction is exactly the nudge-and-divide you just did, and - {@html tex(String.raw`\lim_{h \to 0}`)} (“the limit as h goes to zero”) is - the settling you just watched. Nothing else is hiding in there. The settling also - tells you something about the ground itself: zoom in far enough on any smooth curve - and it straightens into a line — the derivative is that line’s slope. Two chapters - from now, that “zoom until straight” picture carries a real proof on its back; and - its fine print — the line only speaks for the ground right under you — - grows up to become the learning rate’s whole story. -

-
- -
- The limit, drawn: each grey chord leans on the curve a nudge h away — - slope {@html tex(String.raw`2\alpha + h`)} on this parabola — and as h - shrinks, the chords tilt into the one blue line whose slope is exactly - {@html tex(String.raw`2\alpha`)}: the tangent. The derivative is where the chords - were heading all along. -
-
-

- Two knobs, same recipe, one new courtesy: with {@html tex(String.raw`\alpha`)} and {@html tex(String.raw`\beta`)} both live, nudge - one and freeze the other. The ratio you get is a - partial derivative, written with a curly - {@html tex(String.raw`\partial`)} — say it “partial”, and yes, it is the symbol on - this lab’s front door: -

-
{@html texD(formulas.partialDef)}
-

- {@html tex(String.raw`\partial \mathcal{L}/\partial \alpha`)} reads: nudge {@html tex(String.raw`\alpha`)}, - hold {@html tex(String.raw`\beta`)} still, divide the response by the nudge. Do it once per knob and you are - holding two numbers. Stacking those two numbers into a single arrow is exactly where - the next chapter begins. -

-
-
- {#await import('./GuideGradient3D.svelte') then m} - - {/await} -
- {@html tex(String.raw`\beta`)} frozen — slope {@html tex(String.raw`\partial\mathcal{L}/\partial\alpha`)} - {@html tex(String.raw`\alpha`)} frozen — slope {@html tex(String.raw`\partial\mathcal{L}/\partial\beta`)} -
-
drag to orbit
-
-
- “Hold the other knob still,” made literal: each translucent plane freezes one - knob, and the surface passes through it as an ordinary curve. The blue - cut’s slope at the marker is {@html tex(String.raw`\partial\mathcal{L}/\partial\alpha`)}, - the amber cut’s is {@html tex(String.raw`\partial\mathcal{L}/\partial\beta`)} — - two ordinary derivatives, at right angles, on one surface. Drag to spin it. -
-
-

- How the app really does it: you could compute every slope by - literal nudging (the finite-difference recipe above — it’s how the curvature lens - works). But nudging carries a whisper of error, so each problem here ships a - hand-derived exact formula for its gradient instead — and the test suite trusts - nothing: every formula is re-checked against nudge-and-divide at many random - points. Differentiate by hand, verify by nudge — a professional habit - worth stealing. -

- {#if chapterPresets['ch-derivative']} - runPreset('ch-derivative')} demoLabel={chapterPresets['ch-derivative'].title} /> - {/if} - {#if chRefs['ch-derivative']} -
- Further reading - {#each chRefs['ch-derivative'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} -
- - -
-

Which way is downhill?

- -

- Standing on a hillside in fog, you can still feel which way is down — the ground - tilts under your feet. That tilt is the slope. With two knobs there - are two slopes at once: how the loss changes as you nudge {@html tex(String.raw`\alpha`)}, and how it changes as - you nudge {@html tex(String.raw`\beta`)}. Bundle those two together and you get the gradient, - written {@html tex(String.raw`\nabla\mathcal{L}`)} (say “grad L”). -

-

- The gradient is an arrow, and it always points in the direction of steepest - increase — straight uphill. So to go down, you walk the - opposite way, along {@html tex(String.raw`-\nabla\mathcal{L}`)}. That negative gradient is - the single most important arrow in this whole app. -

-

- Why steepest? Picture standing on the slope and trying every direction you - could step. Each heading has its own rate of climb, and the gradient is simply the one - whose climb is fastest. Every other direction is a watered-down version of it: its - steepness is the gradient’s shadow cast onto that heading — full strength straight along - {@html tex(String.raw`\nabla\mathcal{L}`)}, and fading to nothing at a right angle to it. Those flat, right-angle - directions are exactly the contour lines on the map: walk along a - contour and the loss never changes, so the steepest way off it has to be square across - it. The gradient is always perpendicular to the contours — which is why the - field arrows below cut straight through the white loops rather than running along them. -

- -
+ {#if id === 'downhill-field'} @@ -1247,103 +998,7 @@ -
-
-

The field of downhill arrows

-

- Every faint arrow on the loss map is {@html tex(String.raw`-\nabla\mathcal{L}`)} at that spot — the - steepest way down — and they all stream toward the basin. They are longer where - the surface is steeper. -

-

- On the marker itself, the blue arrow is this same - {@html tex(String.raw`-\nabla\mathcal{L}`)}: the steepest descent from exactly where you stand. (Its red partner arrives - in the next chapter.) -

-
-
- - - -

- Formally, the gradient is a column of partial derivatives — one - slope per parameter. Each entry answers a single, narrow question: if I wiggle - only this knob and hold the other still, how fast does the loss change? There is - nothing mystical in measuring one: nudge {@html tex(String.raw`\alpha`)} by a hair, see how far the loss moved, and - divide the change by the nudge. Do that once for {@html tex(String.raw`\alpha`)} and once for {@html tex(String.raw`\beta`)} and you have the two - numbers the gradient is built from. -

-

- Two knobs make that easy — but a real model has millions or billions, and nudging each - one in turn would be hopeless. They use backpropagation (reverse-mode - automatic differentiation): one backward sweep of the chain rule that hands back the - derivative for every parameter at once, at about the cost of a single forward - pass (Rumelhart, Hinton & Williams, 1986; Baydin et al., 2018). The meaning is exactly - the {@html tex(String.raw`\nabla\mathcal{L}`)} here — it is just computed without ever - nudging anything. -

-
{@html texD(formulas.gradientDefinition)}
-

- Stack those two answers into a little arrow and you have {@html tex(String.raw`\nabla\mathcal{L}`)}. Its - direction is the steepest way uphill; its length is - how steep. That is why the field arrows stretch long on the steep walls and shrink to - almost nothing at the basin floor — at the very bottom there is no downhill left, so - the gradient, and the step it drives, fades to zero. The marker arriving and going - still is the gradient vanishing. -

- -

- We keep calling {@html tex(String.raw`-\nabla\mathcal{L}`)} the steepest way down. That is not loose talk — and it is - worth seeing why, first in three dimensions, then in one short line of proof. -

-
-
- {#await import('./GuideGradient3D.svelte') then m} - - {/await} -
- {@html tex(String.raw`\nabla\mathcal{L}`)} — steepest ascent - −{@html tex(String.raw`\nabla\mathcal{L}`)} — toward the basin -
-
drag to orbit
-
-
- The same idea in three dimensions: on the wall of the bowl, - {@html tex(String.raw`\nabla\mathcal{L}`)} (amber) points straight up the steepest rise - and −{@html tex(String.raw`\nabla\mathcal{L}`)} (emerald) straight down toward the basin — - both perpendicular to the green level ring they sit on. Drag to spin it. -
-
- -
-
-

An arrow is two numbers

-

- Three tools before the proof. An arrow on the {@html tex(String.raw`(\alpha, \beta)`)} - plane is its two components stacked — how far along - {@html tex(String.raw`\alpha`)}, how far along {@html tex(String.raw`\beta`)}; - that is all the bracket notation means. Its length, written - {@html tex(String.raw`\lVert\mathbf{v}\rVert`)}, is Pythagoras on those legs: - for {@html tex(String.raw`\mathbf{v} = [3, 2]`)}, - {@html tex(String.raw`\lVert\mathbf{v}\rVert = \sqrt{3^2 + 2^2} = \sqrt{13}`)}. - A unit vector has length exactly 1 — pure direction, no size. - And the dot product multiplies matching components and adds: - {@html tex(String.raw`\mathbf{v}\cdot\mathbf{u} = v_1 u_1 + v_2 u_2`)} — one - number, and for a unit {@html tex(String.raw`\mathbf{u}`)} it is precisely the - length of {@html tex(String.raw`\mathbf{v}`)}’s shadow on - {@html tex(String.raw`\mathbf{u}`)}. That shadow is the whole proof below. -

-
+ {:else if id === 'downhill-vector'} @@ -1361,18 +1016,17 @@ {@html tex(String.raw`\lVert\mathbf{v}\rVert = \sqrt{13}`)} -
- -
-
Why the negative gradient is exactly the steepest descent
-

- Take a unit step in some direction {@html tex(String.raw`\mathbf{u}`)}. The loss changes - at a rate equal to the gradient’s shadow on that direction — their dot product - {@html tex(String.raw`\nabla\mathcal{L}\cdot\mathbf{u}`)}. Writing {@html tex(String.raw`\varphi`)} - (“phi”) for the angle between {@html tex(String.raw`\mathbf{u}`)} and {@html tex(String.raw`\nabla\mathcal{L}`)}, - that shadow has length {@html tex(String.raw`\lVert\nabla\mathcal{L}\rVert\cos\varphi`)}: -

-
{@html texD(formulas.directional)}
+ {:else if id === 'bowl-1d'} + + + + minimum + loss ↑ + + {/if} + {/snippet} + {#snippet chFigure(id: string, cap: string)} + {#if id === 'downhill-proof'}
-
- Left: the rate is {@html tex(String.raw`\nabla\mathcal{L}`)}’s shadow on u. Right: sweep u - around and that shadow traces a cosine — biggest along {@html tex(String.raw`\nabla\mathcal{L}`)}, zero across a contour, - most negative along {@html tex(String.raw`-\nabla\mathcal{L}`)}. -
+
{@html cap}
-

- Because {@html tex(String.raw`\cos\varphi`)} only ever runs from +1 to −1, that rate is - largest when {@html tex(String.raw`\mathbf{u}`)} lines up with {@html tex(String.raw`\nabla\mathcal{L}`)} - (the fastest rise), exactly zero at a right angle (walking a contour — the loss - holds still), and most negative along {@html tex(String.raw`-\nabla\mathcal{L}`)} (the - fastest fall). No direction can beat it. ∎ -

-
- -

- One honest caveat to carry forward: the gradient is only the truth right where you - stand. Zoom in close enough and any smooth surface flattens into a tilted plane, - and {@html tex(String.raw`\nabla\mathcal{L}`)} is exactly that tilt — but step too far and the real ground curves away from the - plane you trusted. That gap between the slope underfoot and the surface a stride away is - the whole reason a step can be too big, and taming it is what the learning rate - exists to do. -

- startLessonFromChapter('ch-downhill')} - demo={chapterPresets['ch-downhill'] ? () => runPreset('ch-downhill') : null} - /> - {#if chRefs['ch-downhill']} -
- Further reading - {#each chRefs['ch-downhill'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} -
- - -
-

One step of descent

- -

Now we can actually walk. One step of gradient descent is almost insultingly simple:

-
- Stand at your current {@html tex(String.raw`(\alpha, \beta)`)}. Look downhill — that’s {@html tex(String.raw`-\nabla\mathcal{L}`)}. Take a - step in that direction — {@html tex(String.raw`\gamma`)} times as long as the slope is steep. - Repeat. -
-

- In symbols, that is the rule the entire field is built on. We write {@html tex(String.raw`\theta`)} - (“theta”) as shorthand for the pair {@html tex(String.raw`(\alpha, \beta)`)} together: -

-
{@html texD(formulas.stepRule)}
-

- Press Step to take exactly one of these; press Train - to take many in a row and watch the marker slide into a valley. -

-

- You’ll now notice a red arrow on the marker beside the - blue one. The blue arrow is the pure downhill direction; the - red arrow is the step the optimizer actually took. Early on they - almost agree. Once you add the tricks in Part IV, they’ll split apart — and - that gap is the optimizer’s personality. -

- startLessonFromChapter('ch-step')} - demo={chapterPresets['ch-step'] ? () => runPreset('ch-step') : null} - /> - {#if chRefs['ch-step']} -
- Further reading - {#each chRefs['ch-step'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
- {/if} -
- - -
-

The learning rate {@html tex(String.raw`\gamma`)}

- -

- There is one number you’ll reach for more than any other: {@html tex(String.raw`\gamma`)} - (gamma), the learning rate — how big each step is. It’s a Goldilocks - dial. -

-
    -
  • Too small: the marker creeps; it never reaches the bottom before the steps run out.
    • -
  • Too big: it overshoots the valley floor and bounces up the far wall — loss leaps around, or rockets off to infinity. (The app catches this, stops, and explains what happened.)
    • -
  • Just right: a smooth glide into the basin. Every problem ships with a sane default — but the fastest way to feel {@html tex(String.raw`\gamma`)} is to break it on purpose.
    • -
-

- And “too big” is not vague — it has an exact edge. For a smooth bowl, gradient - descent settles only while {@html tex(String.raw`\gamma`)} stays below two divided by the sharpest bend - of the surface, a number written {@html tex(String.raw`\lambda_{\max}`)}. Careful with the word: - this is not the steepness you’ve been reading off the arrows (how tilted - the ground is) but a genuinely new quantity — how fast the tilt itself - changes as you walk. That is the curvature. -

-
{@html texD(formulas.stability)}
-

- Stay under that line and each step lands closer to the bottom than the last, so the - run converges. Cross it and the opposite compounds: every step overshoots a little - more than the one before, and the loss runs off to infinity. Where does the 2 come - from, and what exactly is bending? The next chapter builds curvature with - your own hands — the same nudge-and-divide trick, aimed at the slope this time — - and derives this edge in four honest lines. -

- -

- Sometimes the blow-up comes not from {@html tex(String.raw`\gamma`)} but from a freak gradient — a cliff in the - surface, or the deep, recurrent networks where gradients can explode. - The standard guard is gradient clipping: if the gradient’s length - exceeds a threshold {@html tex(String.raw`c`)}, rescale it back to that length before - stepping, {@html tex(String.raw`\nabla\mathcal{L} \leftarrow c\,\nabla\mathcal{L}/\lVert\nabla\mathcal{L}\rVert`)}, - keeping its direction but capping its size (Pascanu et al., 2013). The opposite failure — - gradients that vanish on a flat plateau — just stalls a run, the quiet - trap from the landscape chapter. -

-
-
+ {:else if id === 'gamma-regimes'} +
+
{#each lrRegimes as g}
{/each}
-
- Gradient descent from the same start (orange) at three step sizes, on one bowl. Below the - stability limit it settles — sluggishly, or briskly; cross the limit and every step - overshoots a little more than the last. -
-
- startLessonFromChapter('ch-gamma')} - demo={chapterPresets['ch-gamma'] ? () => runPreset('ch-gamma') : null} - /> - {#if chRefs['ch-gamma']} -
- Further reading - {#each chRefs['ch-gamma'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - +
{@html cap}
+
+ {:else} +
+ {#if id === 'derivative-secant'} + + {:else if id === 'derivative-slices-3d'} +
+ {#await import('./GuideGradient3D.svelte') then m} + + {/await} +
+ {@html tex(String.raw`\beta`)} frozen — slope {@html tex(String.raw`\partial\mathcal{L}/\partial\alpha`)} + {@html tex(String.raw`\alpha`)} frozen — slope {@html tex(String.raw`\partial\mathcal{L}/\partial\beta`)} +
+
drag to orbit
- {/if} -
- - -
-

The bend of the bowl

- -

- The last chapter ended on a formula pulled out of a hat: stay under - {@html tex(String.raw`2/\lambda_{\max}`)}. This chapter earns it. What we need is - one more number at every point of the landscape — not how tilted the ground is, but - how quickly the tilt itself changes as you walk. The slope of the slope: the - curvature. -

-

- Feel the difference first. A wine glass and a soup bowl can be equally steep where - you stand — same slope — but descend a little and the glass tightens while - the bowl relaxes. Curvature is the rate of that tightening, and you already - own the tool that measures it: nudge {@html tex(String.raw`\alpha`)} and divide — only this time, watch how the - slope answers, not the loss. The derivative of the derivative, written - {@html tex(String.raw`\partial^2 \mathcal{L}/\partial \alpha^2`)} and, for the rest - of this chapter, called {@html tex(String.raw`\lambda`)}: big {@html tex(String.raw`\lambda`)}, sharp bend; small - {@html tex(String.raw`\lambda`)}, gentle one; zero, flat as a board. -

-
-
+ {#await import('./GuideGradient3D.svelte') then m} + + {/await} +
+ {@html tex(String.raw`\nabla\mathcal{L}`)} — steepest ascent + −{@html tex(String.raw`\nabla\mathcal{L}`)} — toward the basin +
+
drag to orbit
+
+ {:else if id === 'curvature-bend'} + {@html tex(String.raw`\lambda`)} small -
- Same slope underfoot — the dashed tangent is shared — but two different futures: - the red curve tightens, the green one relaxes. The first derivative can’t tell - them apart at the marker; the second one, {@html tex(String.raw`\lambda`)}, is - exactly what does. -
-
- -
-
Where the 2 comes from — in four lines
-

- Take the cleanest bowl there is: {@html tex(String.raw`\mathcal{L} = \tfrac{1}{2}\lambda\alpha^2`)}, - curvature {@html tex(String.raw`\lambda`)} everywhere, minimum at zero. Its slope at {@html tex(String.raw`\alpha`)} is - {@html tex(String.raw`\lambda\alpha`)}, so one step of gradient descent is -

-
{@html texD(formulas.contraction)}
-

- Every step multiplies the distance to the bottom by the same factor - {@html tex(String.raw`(1-\gamma\lambda)`)} — and that one multiplier is the whole - story. While {@html tex(String.raw`\gamma\lambda < 1`)} the factor sits between 0 - and 1: a smooth glide in. At {@html tex(String.raw`\gamma\lambda = 1`)} the factor - is 0 — you land at the bottom in one hop ({@html tex(String.raw`\gamma = 1/\lambda`)} is this bowl’s own - perfect learning rate). Between 1 and 2 the factor is negative but small: - overshoot to the far wall, yet closer each bounce. At exactly 2, you bounce - between two mirror points forever. And past 2 every bounce lands - higher than the last — divergence. There is the edge, and there is - the 2. ∎ -

-
-
-
- -

- Now open the second knob. At any point of a real landscape the surface bends by a - different amount in different directions — along a valley’s floor, barely; - across it, sharply. The honest bookkeeping is a small table of bendings called the - Hessian: -

-
{@html texD(formulas.hessianMatrix)}
-

- Don’t let the box intimidate you. The two diagonal entries are exactly the - {@html tex(String.raw`\partial^2`)} curvatures you just built, one per knob; the - corner entry (the same number twice) records the twist — how nudging one - knob changes the other knob’s slope. Four numbers, and together they pin - down the little bowl that best fits the surface right where you stand. Zoom in on - any smooth landscape and that fitted bowl is the landscape — the same way - the fitted line was, one derivative ago. -

-

- Meet the whole family. Stack one output’s slopes into a column and - you have the gradient. Give the machine many outputs — a - network predicting a hundred things at once — and each output brings its own row of - slopes; the full table is the Jacobian, the gradient’s big sibling. - And the Hessian you just met? Differentiate each entry of the gradient and stack the - results: the Hessian is exactly the Jacobian of the gradient. One move — - nudge, divide, tabulate — wearing three sizes. -

-

- A stretched or twisted bowl still has a gentlest direction and a - sharpest one — turn it in your hands until you face them. Their two - bendings are called {@html tex(String.raw`\lambda_{\min}`)} and - {@html tex(String.raw`\lambda_{\max}`)}, and the last chapter’s speed limit can now - be read honestly: the sharpest bend polices {@html tex(String.raw`\gamma`)} — that is - {@html tex(String.raw`\gamma < 2/\lambda_{\max}`)} — while your progress along the - gentlest direction is paid at the rate - {@html tex(String.raw`(1 - \gamma\lambda_{\min})`)} per step. One {@html tex(String.raw`\gamma`)}, two masters. -

-

- How badly can the two masters disagree? Take their ratio: -

-
{@html texD(formulas.kappa)}
-

- the condition number. {@html tex(String.raw`\kappa = 1`)} is a perfectly round bowl: any safe {@html tex(String.raw`\gamma`)} - lands you in a few hops. {@html tex(String.raw`\kappa = 10`)} means the sharp direction forces a {@html tex(String.raw`\gamma`)} so timid that - the gentle direction keeps about 80% of its remaining distance every step. - Ravines, trenches, the long crawl — they are all this one number wearing different - landscapes, and Part IV’s entire optimizer family tree is organised around - outwitting it. -

-

- The app will show you the Hessian live. In the Loss & Gradient panel’s header, - switch on the curvature lens: the ellipse drawn at the marker - is the fitted bowl seen from above — long axis the gentle bend, short axis - the sharp one — with {@html tex(String.raw`\kappa`)} read out beside it. On a saddle, the direction that curves - down turns red and dashed: the escape route. -

- - {#if chapterPresets['ch-curvature']} - runPreset('ch-curvature')} demoLabel={chapterPresets['ch-curvature'].title} /> - {/if} - {#if chRefs['ch-curvature']} -
- Further reading - {#each chRefs['ch-curvature'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - + {:else if id === 'landscape-two-views'} +
+ + the map — from straight above + + + + + + + + + {#each landscapeFig.rings as r} + + {/each} + + the surface — from the side + + {:else if id === 'shapes-basins-saddle'} + + {/if} +
{@html cap}
+
{/if} - - - -
-

Scheduling the learning rate

-

- The learning rate just handed us a single, unavoidable compromise: a large - {@html tex(String.raw`\gamma`)} covers ground fast but overshoots the floor; a small {@html tex(String.raw`\gamma`)} lands precisely - but crawls to get there. You don’t actually have to choose. Stop treating {@html tex(String.raw`\gamma`)} as one frozen - number and schedule it — large early to cover ground, small late to settle - cleanly — and you get both halves of the bargain. The Schedule control - beneath the learning rate does exactly that: it multiplies your base {@html tex(String.raw`\gamma`)} by a factor that - changes on every step of the run. -

-

- The four schedules trace four different shapes for that factor over a run — flat, then - three ways of bleeding {@html tex(String.raw`\gamma`)} away as the steps tick by: -

-
+ {/snippet} + {#snippet chWidget(id: string)} + {#if id === 'schedule-grid'} +
{#each scheduleCurves as s (s.id)}
{/each}
-

- Constant holds {@html tex(String.raw`\gamma`)} start to finish — the honest baseline, and always the - compromise above. Step decay keeps {@html tex(String.raw`\gamma`)} flat, then cuts it by a fixed factor - at set milestones (here ×0.3 a third of the way in, and again at two-thirds). It leaves the - loss curve’s most recognizable fingerprint: a long plateau, then a sudden cliff - the instant {@html tex(String.raw`\gamma`)} drops and the smaller step resolves detail the larger one skated over. For - most of deep learning’s history, that staircase trained nearly every network. -

-

- Cosine does the same work without the jolts — {@html tex(String.raw`\gamma`)} eases down the first half - of a cosine from full strength to a small floor (about 5%): gentle at first, fastest - through the middle, feather-light by the end. Lingering near full strength early is - the point — the run banks its fast progress before precision matters. With no brutal - transition it then simply settles, which is why cosine annealing is the modern default. Warmup + cosine bolts a short - on-ramp onto the front: {@html tex(String.raw`\gamma`)} starts near zero and climbs over the first tenth before the - cosine takes over. That protects the opening, where a run begins at a random, - often dreadful point and one full-size step could fling the marker off the map — so it is - now standard for training large models from scratch. -

-

- One practical wrinkle: each shape stretches to fit the run, and at 1× the decay only - finishes on the run’s very last step — so you never get to watch the settled - tail. The Decay speed slider — it appears whenever a non-constant - schedule is active on a finite run (in ∞ mode there is no horizon, so schedules switch - off) — compresses the whole schedule into a fraction of the run, so at 4× it - finishes annealing a quarter of the way in and the rest of the run shows you the - landing. Turn it up and read the result off the dotted {@html tex(String.raw`\gamma(t)`)} line in the loss chart. -

-

- Scheduling has a second, deeper payoff that only lands once gradients turn noisy - — the subject of the next part. A {@html tex(String.raw`\gamma`)} bled toward zero is the one thing that pulls a restless - run in to a clean stop. And one optimizer you’ll meet there, Lion, takes a - fixed-size step and so cannot settle at all on a constant {@html tex(String.raw`\gamma`)}: it just orbits the - minimum forever. It is the purest illustration of why schedules exist — switch it to cosine - and the orbit closes to a point. -

- {#if scheduleExperiment} - runExperiment(scheduleExperiment)} demoLabel="Watch Lion orbit, then land" /> - {/if} - {#if chRefs['ch-schedule']} -
- Further reading - {#each chRefs['ch-schedule'] as r} - - {#if r.kind === 'paper'}{:else}{/if} - {r.label} - - {/each} -
{/if} + {/snippet} + +
+
Part I · The landscape
+

The bottom of a bowl

+ +
+ + +
+

Loss is a landscape

+ +
+ + +
+

When the bowl isn’t a bowl

+ +
+ + +
+
Part II · Walking downhill
+

How steep, exactly?

+ +
+ + +
+

Which way is downhill?

+ +
+ + +
+

One step of descent

+ +
+ + +
+

The learning rate {@html tex(String.raw`\gamma`)}

+ +
+ + +
+

The bend of the bowl

+ +
+ + +
+

Scheduling the learning rate

+ runExperiment(scheduleExperiment) } : undefined} onPreset={runPreset} onLesson={startLessonFromChapter} />
@@ -2875,16 +2336,16 @@ font-style: italic; } - ul { margin: 0.5rem 0 1rem 0; padding-left: 1.4rem; } - li { margin: 0.5rem 0; font-size: 1.0rem; line-height: 1.6; color: var(--color-text-secondary); } - li strong { color: var(--color-text-primary); } + .reading-column :global(ul) { margin: 0.5rem 0 1rem 0; padding-left: 1.4rem; } + .reading-column :global(li) { margin: 0.5rem 0; font-size: 1.0rem; line-height: 1.6; color: var(--color-text-secondary); } + .reading-column :global(li strong) { color: var(--color-text-primary); } .reading-column :global(em.g) { font-family: Georgia, serif; font-style: italic; color: #10b981; font-weight: 500; } - .ink-blue { color: #3b82f6; font-weight: 600; } - .ink-red { color: #ef4444; font-weight: 600; } + .reading-column :global(.ink-blue) { color: #3b82f6; font-weight: 600; } + .reading-column :global(.ink-red) { color: #ef4444; font-weight: 600; } /* ---------- Recipe blockquote ---------- */ - .recipe { + .reading-column :global(.recipe) { margin: 0 0 1rem 0; padding: 0.85rem 1.1rem; border-left: 2px solid var(--color-border); @@ -2894,7 +2355,7 @@ line-height: 1.65; color: var(--color-text-secondary); } - .recipe strong { color: var(--color-text-primary); } + .reading-column :global(.recipe strong) { color: var(--color-text-primary); } /* ---------- Go deeper ---------- */ /* ---------- Schedule preview cards ---------- */ @@ -3070,24 +2531,24 @@ } /* ---------- Proof callout (the steepest-descent argument) ---------- */ - .proof { + .reading-column :global(.proof) { margin: 1.6rem 0; border: 1px solid var(--color-border); border-radius: 10px; background: var(--color-bg-tertiary); padding: 1rem 1.2rem; } - .proof-title { + .reading-column :global(.proof-title) { font-size: 0.7rem; font-weight: 800; letter-spacing: 0.08em; text-transform: uppercase; color: var(--color-text-secondary); margin-bottom: 0.7rem; } - .proof-p { font-size: 0.88rem; line-height: 1.65; margin: 0 0 0.7rem; } + .reading-column :global(.proof-p) { font-size: 0.88rem; line-height: 1.65; margin: 0 0 0.7rem; } .reading-column :global(.proof .formula-display) { margin: 0.6rem 0; } .proof-fig { margin: 0.7rem 0 0.5rem; } .proof-fig svg { display: block; width: 100%; height: auto; } .proof-lbl { font-size: 10.5px; font-weight: 600; font-family: inherit; text-anchor: middle; } .proof-figcap { font-size: 0.76rem; line-height: 1.5; text-align: left; color: var(--color-text-tertiary); margin-top: 0.5rem; } - .proof-qed { color: #10b981; font-weight: 700; margin-left: 0.15rem; } + .reading-column :global(.proof-qed) { color: #10b981; font-weight: 700; margin-left: 0.15rem; } /* ---------- Formulas ---------- */ .reading-column :global(.formula-display) { @@ -3112,13 +2573,13 @@ .opt-formula::-webkit-scrollbar-track { background: transparent; } /* ---------- Knob bullets ---------- */ - .knob-bullets { padding-left: 0; list-style: none; } - .knob-bullets li { + .reading-column :global(.knob-bullets) { padding-left: 0; list-style: none; } + .reading-column :global(.knob-bullets li) { position: relative; padding-left: 1.1rem; font-size: 0.95rem; } - .knob-bullets li::before { + .reading-column :global(.knob-bullets li::before) { content: ''; position: absolute; left: 0; top: 0.65em; @@ -3570,7 +3031,7 @@ .hero-svg { height: 150px; } .hero-title { font-size: 1.3rem; } h3 { font-size: 1.25rem; } - .reading-column :global(p), li, .recipe { font-size: 0.95rem; } + .reading-column :global(p), .reading-column :global(li), .reading-column :global(.recipe) { font-size: 0.95rem; } .reading-column :global(.concept) { grid-template-columns: 1fr; } .reading-column :global(.concept-svg) { height: 100px; } diff --git a/src/content/blocks.ts b/src/content/blocks.ts index 366340d..129611f 100644 --- a/src/content/blocks.ts +++ b/src/content/blocks.ts @@ -25,10 +25,32 @@ export type Block = /** A "look at the app right now" pointer (p.look) — an app-only register. */ | { kind: 'look'; text: Rich } /** A numbered display equation from the formula registry. */ - | { kind: 'display'; formula: FormulaKey } + | { kind: 'display'; formula: FormulaKey; center?: boolean } + /** The imperative recipe blockquote — "Nudge. Measure. Divide." */ + | { kind: 'recipe'; text: Rich } + /** A bulleted list (the ▸-marker knob-bullets style). */ + | { kind: 'list'; items: Rich[] } /** A titled concept box, optionally with a small illustration beside it. */ | { kind: 'concept'; title: string; text: Rich; fig?: string } + /** + * The full-bleed concept variant: a background visual (named by fig, + * supplied by the chapter shell) behind a fade, with the titled text + * overlaid on the right. + */ + | { kind: 'conceptOverlay'; title: string; paras: Rich[]; fig: string } + /** + * A boxed derivation. Inner blocks render in the proof register (p → + * .proof-p, displays centred, figures as .proof-fig); the renderer seals + * the last paragraph with the ∎. + */ + | { kind: 'proof'; title: string; blocks: Block[] } /** An "In a billion dimensions" honesty note — print's margin channel. */ | { kind: 'hd'; text: Rich } /** A figure: the id names an app-side (later: computed) visual; the caption lives here. */ - | { kind: 'figure'; id: string; caption: Rich }; + | { kind: 'figure'; id: string; caption: Rich } + /** + * An interactive island with no print analogue of its own (the schedule + * gallery, later the race) — the shell supplies it; print will use a + * fallback figure. + */ + | { kind: 'widget'; id: string }; diff --git a/src/content/chapters/ch-curvature.ts b/src/content/chapters/ch-curvature.ts new file mode 100644 index 0000000..6df998e --- /dev/null +++ b/src/content/chapters/ch-curvature.ts @@ -0,0 +1,72 @@ +import type { Block } from '../blocks'; + +/** Chapter 8 · The bend of the bowl — curvature, the Hessian, κ, and the 2. */ +export const chCurvature: Block[] = [ + { + kind: 'p', + text: 'The last chapter ended on a formula pulled out of a hat: stay under $2/\\lambda_{\\max}$. This chapter earns it. What we need is one more number at every point of the landscape — not how tilted the ground is, but how quickly the tilt itself changes as you walk. The slope of the slope: the **curvature**.' + }, + { + kind: 'p', + text: 'Feel the difference first. A wine glass and a soup bowl can be equally steep where you stand — same slope — but descend a little and the glass *tightens* while the bowl *relaxes*. Curvature is the rate of that tightening, and you already own the tool that measures it: nudge $\\alpha$ and divide — only this time, watch how the *slope* answers, not the loss. The derivative of the derivative, written $\\partial^2 \\mathcal{L}/\\partial \\alpha^2$ and, for the rest of this chapter, called $\\lambda$: big $\\lambda$, sharp bend; small $\\lambda$, gentle one; zero, flat as a board.' + }, + { + kind: 'figure', + id: 'curvature-bend', + caption: 'Same slope underfoot — the dashed tangent is shared — but two different futures: the red curve tightens, the green one relaxes. The first derivative can’t tell them apart at the marker; the second one, $\\lambda$, is exactly what does.' + }, + { + kind: 'proof', + title: 'Where the 2 comes from — in four lines', + blocks: [ + { + kind: 'p', + text: 'Take the cleanest bowl there is: $\\mathcal{L} = \\tfrac{1}{2}\\lambda\\alpha^2$, curvature $\\lambda$ everywhere, minimum at zero. Its slope at $\\alpha$ is $\\lambda\\alpha$, so one step of gradient descent is' + }, + { kind: 'display', formula: 'contraction', center: true }, + { + kind: 'p', + text: 'Every step *multiplies the distance to the bottom* by the same factor $(1-\\gamma\\lambda)$ — and that one multiplier is the whole story. While $\\gamma\\lambda < 1$ the factor sits between 0 and 1: a smooth glide in. At $\\gamma\\lambda = 1$ the factor is 0 — you land at the bottom in *one hop* ($\\gamma = 1/\\lambda$ is this bowl’s own perfect learning rate). Between 1 and 2 the factor is negative but small: overshoot to the far wall, yet closer each bounce. At exactly 2, you bounce between two mirror points forever. And past 2 every bounce lands *higher* than the last — divergence. There is the edge, and there is the 2.' + } + ] + }, + { + kind: 'figure', + id: 'curvature-regimes', + caption: 'The multiplier, run for real: amber dots are actual gradient-descent iterates on $\\mathcal{L} = \\tfrac{1}{2}\\lambda\\alpha^2$, starting from the ringed point. Glide, one-hop, shrinking bounce, growing bounce — four values of $\\gamma\\lambda$, one factor $(1-\\gamma\\lambda)$.' + }, + { + kind: 'p', + text: 'Now open the second knob. At any point of a real landscape the surface bends by a *different amount in different directions* — along a valley’s floor, barely; across it, sharply. The honest bookkeeping is a small table of bendings called the **Hessian**:' + }, + { kind: 'display', formula: 'hessianMatrix', center: true }, + { + kind: 'p', + text: 'Don’t let the box intimidate you. The two diagonal entries are exactly the $\\partial^2$ curvatures you just built, one per knob; the corner entry (the same number twice) records the *twist* — how nudging one knob changes the *other* knob’s slope. Four numbers, and together they pin down the little bowl that best fits the surface right where you stand. Zoom in on any smooth landscape and that fitted bowl *is* the landscape — the same way the fitted line was, one derivative ago.' + }, + { + kind: 'aside', + text: '**Meet the whole family.** Stack one output’s slopes into a column and you have the **gradient**. Give the machine *many* outputs — a network predicting a hundred things at once — and each output brings its own row of slopes; the full table is the **Jacobian**, the gradient’s big sibling. And the Hessian you just met? Differentiate each entry of the gradient and stack the results: *the Hessian is exactly the Jacobian of the gradient.* One move — nudge, divide, tabulate — wearing three sizes.' + }, + { + kind: 'p', + text: 'A stretched or twisted bowl still has a **gentlest** direction and a **sharpest** one — turn it in your hands until you face them. Their two bendings are called $\\lambda_{\\min}$ and $\\lambda_{\\max}$, and the last chapter’s speed limit can now be read honestly: the *sharpest* bend polices $\\gamma$ — that is $\\gamma < 2/\\lambda_{\\max}$ — while your progress along the *gentlest* direction is paid at the rate $(1 - \\gamma\\lambda_{\\min})$ per step. One $\\gamma$, two masters.' + }, + { + kind: 'p', + text: 'How badly can the two masters disagree? Take their ratio:' + }, + { kind: 'display', formula: 'kappa', center: true }, + { + kind: 'p', + text: 'the **condition number**. $\\kappa = 1$ is a perfectly round bowl: any safe $\\gamma$ lands you in a few hops. $\\kappa = 10$ means the sharp direction forces a $\\gamma$ so timid that the gentle direction keeps about 80% of its remaining distance *every step*. Ravines, trenches, the long crawl — they are all this one number wearing different landscapes, and Part IV’s entire optimizer family tree is organised around outwitting it.' + }, + { + kind: 'look', + text: 'The app will show you the Hessian live. In the Loss & Gradient panel’s header, switch on the **curvature lens**: the ellipse drawn at the marker *is* the fitted bowl seen from above — long axis the gentle bend, short axis the sharp one — with $\\kappa$ read out beside it. On a saddle, the direction that curves *down* turns red and dashed: the escape route.' + }, + { + kind: 'hd', + text: 'Up there the ravine doesn’t just stretch — it multiplies. A real network’s loss has millions of curvature directions, and measured spectra show a vast, nearly flat bulk hugging zero plus a handful of steep outliers: less a valley than a canyon system with a few sheer walls and endless soft floor. Condition numbers in the wild reach 10⁵ and beyond, so the crawl this chapter proved isn’t a corner case — it is the default condition of deep learning. That is why every method in Part IV ships in every deep-learning library.' + } +]; diff --git a/src/content/chapters/ch-derivative.ts b/src/content/chapters/ch-derivative.ts new file mode 100644 index 0000000..bd6685f --- /dev/null +++ b/src/content/chapters/ch-derivative.ts @@ -0,0 +1,45 @@ +import type { Block } from '../blocks'; + +/** Chapter 4 · How steep, exactly? — the derivative built from nudge-and-divide. */ +export const chDerivative: Block[] = [ + { + kind: 'p', + text: 'Part I kept saying *slope* and trusted your legs to know what it meant. Before the walking starts in earnest, let’s put a number on it — because the number is the whole trick, and you can build it yourself with nothing but a subtraction and a division.' + }, + { + kind: 'p', + text: 'Here is the move. Stand somewhere on a 1-D loss curve — say $\\alpha = 2$ on **Fit a Slope**, whose loss happens to be $\\mathcal{L}(\\alpha)=\\alpha^2$, so $\\mathcal{L}(2)=4$. **Nudge** the knob by some small amount *h*, and divide the loss’s response by the nudge. Nudge by *h* = 0.1 and the loss climbs from 4 to 4.41 — a rise of 0.41 over a run of 0.1: ratio **4.1**. Try *h* = 0.01: the ratio comes out 4.01. Try 0.001: **4.001**. The nudges are vanishing, but the ratio isn’t wandering — it is *settling*, and the number it settles on is 4.' + }, + { + kind: 'recipe', + text: 'Nudge. Measure the response. Divide. Then let the nudge shrink — the number the ratio settles on is the **derivative**: the slope of the loss *at a point*.' + }, + { kind: 'display', formula: 'derivativeLimit', center: true }, + { + kind: 'p', + text: 'Read it slowly, once: the fraction is exactly the nudge-and-divide you just did, and $\\lim_{h \\to 0}$ (“the limit as *h* goes to zero”) is the settling you just watched. Nothing else is hiding in there. The settling also tells you something about the ground itself: zoom in far enough on any smooth curve and it straightens into a line — the derivative is that line’s slope. Two chapters from now, that “zoom until straight” picture carries a real proof on its back; and its fine print — *the line only speaks for the ground right under you* — grows up to become the learning rate’s whole story.' + }, + { + kind: 'figure', + id: 'derivative-secant', + caption: 'The limit, drawn: each grey chord leans on the curve a nudge *h* away — slope $2\\alpha + h$ on this parabola — and as *h* shrinks, the chords tilt into the one blue line whose slope is exactly $2\\alpha$: the tangent. The derivative is where the chords were heading all along.' + }, + { + kind: 'p', + text: 'Two knobs, same recipe, one new courtesy: with $\\alpha$ and $\\beta$ both live, nudge **one and freeze the other**. The ratio you get is a **partial derivative**, written with a curly $\\partial$ — say it “partial”, and yes, it is the symbol on this lab’s front door:' + }, + { kind: 'display', formula: 'partialDef', center: true }, + { + kind: 'p', + text: '$\\partial \\mathcal{L}/\\partial \\alpha$ reads: *nudge $\\alpha$, hold $\\beta$ still, divide the response by the nudge.* Do it once per knob and you are holding two numbers. Stacking those two numbers into a single arrow is exactly where the next chapter begins.' + }, + { + kind: 'figure', + id: 'derivative-slices-3d', + caption: '“Hold the other knob still,” made literal: each translucent plane freezes one knob, and the surface passes through it as an *ordinary curve*. The blue cut’s slope at the marker is $\\partial\\mathcal{L}/\\partial\\alpha$, the amber cut’s is $\\partial\\mathcal{L}/\\partial\\beta$ — two ordinary derivatives, at right angles, on one surface. Drag to spin it.' + }, + { + kind: 'aside', + text: '**How the app really does it:** you could compute every slope by literal nudging (the finite-difference recipe above — it’s how the curvature lens works). But nudging carries a whisper of error, so each problem here ships a hand-derived exact formula for its gradient instead — and the test suite trusts nothing: every formula is re-checked against nudge-and-divide at many random points. *Differentiate by hand, verify by nudge* — a professional habit worth stealing.' + } +]; diff --git a/src/content/chapters/ch-downhill.ts b/src/content/chapters/ch-downhill.ts new file mode 100644 index 0000000..fb2a3cf --- /dev/null +++ b/src/content/chapters/ch-downhill.ts @@ -0,0 +1,82 @@ +import type { Block } from '../blocks'; + +/** Chapter 5 · Which way is downhill? — the gradient, the field, and the proof. */ +export const chDownhill: Block[] = [ + { + kind: 'p', + text: 'Standing on a hillside in fog, you can still feel which way is down — the ground tilts under your feet. That tilt is the **slope**. With two knobs there are two slopes at once: how the loss changes as you nudge $\\alpha$, and how it changes as you nudge $\\beta$. Bundle those two together and you get the **gradient**, written **$\\nabla\\mathcal{L}$** (say “grad L”).' + }, + { + kind: 'p', + text: 'The gradient is an arrow, and it always points in the direction of *steepest increase* — straight uphill. So to go **down**, you walk the **opposite** way, along **$-\\nabla\\mathcal{L}$**. That negative gradient is the single most important arrow in this whole app.' + }, + { + kind: 'p', + text: 'Why *steepest*? Picture standing on the slope and trying every direction you could step. Each heading has its own rate of climb, and the gradient is simply the one whose climb is fastest. Every other direction is a watered-down version of it: its steepness is the gradient’s shadow cast onto that heading — full strength straight along $\\nabla\\mathcal{L}$, and fading to *nothing* at a right angle to it. Those flat, right-angle directions are exactly the **contour lines** on the map: walk along a contour and the loss never changes, so the steepest way off it has to be square across it. *The gradient is always perpendicular to the contours* — which is why the field arrows below cut straight through the white loops rather than running along them.' + }, + { + kind: 'conceptOverlay', + title: 'The field of downhill arrows', + fig: 'downhill-field', + paras: [ + 'Every faint arrow on the loss map is **$-\\nabla\\mathcal{L}$** at that spot — the steepest way down — and they all stream toward the basin. They are longer where the surface is steeper.', + 'On the marker itself, the {blue:blue arrow} is this same $-\\nabla\\mathcal{L}$: the steepest descent from exactly where you stand. (Its red partner arrives in the next chapter.)' + ] + }, + { + kind: 'hd', + text: 'A quiet upgrade for later. With two knobs a contour is a loop — one thin line of “no change”. With $d$ knobs it is a whole $(d-1)$-dimensional sheet: at any point there is a single steepest way up, and a vast flat wall of sideways directions that change nothing at all. The gradient’s job gets lonelier as $d$ grows — one needle of change in a haystack of directions that don’t.' + }, + { + kind: 'p', + text: 'Formally, the gradient is a column of **partial derivatives** — one slope per parameter. Each entry answers a single, narrow question: *if I wiggle only this knob and hold the other still, how fast does the loss change?* There is nothing mystical in measuring one: nudge $\\alpha$ by a hair, see how far the loss moved, and divide the change by the nudge. Do that once for $\\alpha$ and once for $\\beta$ and you have the two numbers the gradient is built from.' + }, + { + kind: 'aside', + text: 'Two knobs make that easy — but a real model has millions or billions, and nudging each one in turn would be hopeless. They use **backpropagation** (reverse-mode automatic differentiation): one backward sweep of the chain rule that hands back the derivative for *every* parameter at once, at about the cost of a single forward pass (Rumelhart, Hinton & Williams, 1986; Baydin et al., 2018). The meaning is exactly the $\\nabla\\mathcal{L}$ here — it is just computed without ever nudging anything.' + }, + { kind: 'display', formula: 'gradientDefinition' }, + { + kind: 'p', + text: 'Stack those two answers into a little arrow and you have $\\nabla\\mathcal{L}$. Its **direction** is the steepest way uphill; its **length** is how steep. That is why the field arrows stretch long on the steep walls and shrink to almost nothing at the basin floor — at the very bottom there is no downhill left, so the gradient, and the step it drives, fades to zero. The marker arriving and going still *is* the gradient vanishing.' + }, + { + kind: 'p', + text: 'We keep calling $-\\nabla\\mathcal{L}$ the *steepest* way down. That is not loose talk — and it is worth seeing why, first in three dimensions, then in one short line of proof.' + }, + { + kind: 'figure', + id: 'downhill-3d', + caption: 'The same idea in three dimensions: on the wall of the bowl, $\\nabla\\mathcal{L}$ (amber) points straight up the steepest rise and −$\\nabla\\mathcal{L}$ (emerald) straight down toward the basin — both perpendicular to the green level ring they sit on. Drag to spin it.' + }, + { + kind: 'concept', + title: 'An arrow is two numbers', + fig: 'downhill-vector', + text: 'Three tools before the proof. An arrow on the $(\\alpha, \\beta)$ plane *is* its two components stacked — how far along $\\alpha$, how far along $\\beta$; that is all the bracket notation means. Its **length**, written $\\lVert\\mathbf{v}\\rVert$, is Pythagoras on those legs: for $\\mathbf{v} = [3, 2]$, $\\lVert\\mathbf{v}\\rVert = \\sqrt{3^2 + 2^2} = \\sqrt{13}$. A **unit vector** has length exactly 1 — pure direction, no size. And the **dot product** multiplies matching components and adds: $\\mathbf{v}\\cdot\\mathbf{u} = v_1 u_1 + v_2 u_2$ — one number, and for a unit $\\mathbf{u}$ it is precisely the length of $\\mathbf{v}$’s *shadow* on $\\mathbf{u}$. That shadow is the whole proof below.' + }, + { + kind: 'proof', + title: 'Why the negative gradient is exactly the steepest descent', + blocks: [ + { + kind: 'p', + text: 'Take a unit step in some direction $\\mathbf{u}$. The loss changes at a rate equal to the gradient’s *shadow* on that direction — their dot product $\\nabla\\mathcal{L}\\cdot\\mathbf{u}$. Writing $\\varphi$ (“phi”) for the angle between $\\mathbf{u}$ and $\\nabla\\mathcal{L}$, that shadow has length $\\lVert\\nabla\\mathcal{L}\\rVert\\cos\\varphi$:' + }, + { kind: 'display', formula: 'directional', center: true }, + { + kind: 'figure', + id: 'downhill-proof', + caption: 'Left: the rate is $\\nabla\\mathcal{L}$’s shadow on **u**. Right: sweep **u** around and that shadow traces a cosine — biggest along $\\nabla\\mathcal{L}$, zero across a contour, most negative along $-\\nabla\\mathcal{L}$.' + }, + { + kind: 'p', + text: 'Because $\\cos\\varphi$ only ever runs from +1 to −1, that rate is largest when $\\mathbf{u}$ lines up with $\\nabla\\mathcal{L}$ (the fastest *rise*), exactly zero at a right angle (walking a contour — the loss holds still), and most negative along $-\\nabla\\mathcal{L}$ (the fastest *fall*). No direction can beat it.' + } + ] + }, + { + kind: 'p', + text: 'One honest caveat to carry forward: the gradient is only the truth *right where you stand.* Zoom in close enough and any smooth surface flattens into a tilted plane, and $\\nabla\\mathcal{L}$ is exactly that tilt — but step too far and the real ground curves away from the plane you trusted. That gap between the slope underfoot and the surface a stride away is the whole reason a step can be *too big*, and taming it is what the learning rate exists to do.' + } +]; diff --git a/src/content/chapters/ch-gamma.ts b/src/content/chapters/ch-gamma.ts new file mode 100644 index 0000000..4d14ae3 --- /dev/null +++ b/src/content/chapters/ch-gamma.ts @@ -0,0 +1,39 @@ +import type { Block } from '../blocks'; + +/** Chapter 7 · The learning rate γ — the Goldilocks dial and its exact edge. */ +export const chGamma: Block[] = [ + { + kind: 'p', + text: 'There is one number you’ll reach for more than any other: {g:$\\gamma$} (gamma), the **learning rate** — how big each step is. It’s a Goldilocks dial.' + }, + { + kind: 'list', + items: [ + '**Too small:** the marker creeps; it never reaches the bottom before the steps run out.', + '**Too big:** it overshoots the valley floor and bounces up the far wall — loss leaps around, or rockets off to infinity. (The app catches this, stops, and explains what happened.)', + '**Just right:** a smooth glide into the basin. Every problem ships with a sane default — but the fastest way to *feel* $\\gamma$ is to break it on purpose.' + ] + }, + { + kind: 'p', + text: 'And “too big” is not vague — it has an exact edge. For a smooth bowl, gradient descent settles only while $\\gamma$ stays below **two divided by the sharpest bend of the surface**, a number written $\\lambda_{\\max}$. Careful with the word: this is *not* the steepness you’ve been reading off the arrows (how tilted the ground is) but a genuinely new quantity — how fast the tilt *itself* changes as you walk. That is the **curvature**.' + }, + { kind: 'display', formula: 'stability' }, + { + kind: 'p', + text: 'Stay under that line and each step lands closer to the bottom than the last, so the run converges. Cross it and the opposite compounds: every step overshoots a little more than the one before, and the loss runs off to infinity. Where does the 2 come from, and what exactly is bending? The *next chapter* builds curvature with your own hands — the same nudge-and-divide trick, aimed at the slope this time — and derives this edge in four honest lines.' + }, + { + kind: 'hd', + text: 'For the clean bowls here, $\\gamma > 2/\\lambda_{\\max}$ means certain divergence — a theorem you can verify with a slider. The modern surprise: full-batch training of real networks was found to hover *right at* that edge — the curvature itself rises until $2/\\lambda_{\\max}$ meets whatever $\\gamma$ you chose, and the loss then falls raggedly along the knife’s edge (Cohen et al., 2021, “edge of stability”). The law you can check on this bowl becomes, at scale, a strange equilibrium the theory is still catching up to.' + }, + { + kind: 'aside', + text: 'Sometimes the blow-up comes not from $\\gamma$ but from a freak gradient — a cliff in the surface, or the deep, recurrent networks where gradients can **explode**. The standard guard is **gradient clipping**: if the gradient’s length exceeds a threshold $c$, rescale it back to that length before stepping, $\\nabla\\mathcal{L} \\leftarrow c\\,\\nabla\\mathcal{L}/\\lVert\\nabla\\mathcal{L}\\rVert$, keeping its direction but capping its size (Pascanu et al., 2013). The opposite failure — gradients that **vanish** on a flat plateau — just stalls a run, the quiet trap from the landscape chapter.' + }, + { + kind: 'figure', + id: 'gamma-regimes', + caption: 'Gradient descent from the same start (orange) at three step sizes, on one bowl. Below the stability limit it settles — sluggishly, or briskly; cross the limit and every step overshoots a little more than the last.' + } +]; diff --git a/src/content/chapters/ch-schedule.ts b/src/content/chapters/ch-schedule.ts new file mode 100644 index 0000000..569e1a8 --- /dev/null +++ b/src/content/chapters/ch-schedule.ts @@ -0,0 +1,30 @@ +import type { Block } from '../blocks'; + +/** Chapter 9 · Scheduling the learning rate — four shapes for one dial. */ +export const chSchedule: Block[] = [ + { + kind: 'p', + text: 'The learning rate just handed us a single, unavoidable compromise: a **large** $\\gamma$ covers ground fast but overshoots the floor; a **small** $\\gamma$ lands precisely but crawls to get there. You don’t actually have to choose. Stop treating $\\gamma$ as one frozen number and **schedule** it — large early to cover ground, small late to settle cleanly — and you get both halves of the bargain. The **Schedule** control beneath the learning rate does exactly that: it multiplies your base $\\gamma$ by a factor that changes on every step of the run.' + }, + { + kind: 'p', + text: 'The four schedules trace four different shapes for that factor over a run — flat, then three ways of bleeding $\\gamma$ away as the steps tick by:' + }, + { kind: 'widget', id: 'schedule-grid' }, + { + kind: 'p', + text: '**Constant** holds $\\gamma$ start to finish — the honest baseline, and always the compromise above. **Step decay** keeps $\\gamma$ flat, then cuts it by a fixed factor at set milestones (here ×0.3 a third of the way in, and again at two-thirds). It leaves the loss curve’s most recognizable fingerprint: a long plateau, then a sudden *cliff* the instant $\\gamma$ drops and the smaller step resolves detail the larger one skated over. For most of deep learning’s history, that staircase trained nearly every network.' + }, + { + kind: 'p', + text: '**Cosine** does the same work without the jolts — $\\gamma$ eases down the first half of a cosine from full strength to a small floor (about 5%): gentle at first, fastest through the middle, feather-light by the end. Lingering near full strength early is the point — the run banks its fast progress before precision matters. With no brutal transition it then simply settles, which is why cosine annealing is the modern default. **Warmup + cosine** bolts a short on-ramp onto the front: $\\gamma$ starts near zero and climbs over the first tenth before the cosine takes over. That protects the opening, where a run *begins* at a random, often dreadful point and one full-size step could fling the marker off the map — so it is now standard for training large models from scratch.' + }, + { + kind: 'p', + text: 'One practical wrinkle: each shape stretches to fit the run, and at 1× the decay only finishes on the run’s very last step — so you never get to watch the *settled* tail. The **Decay speed** slider — it appears whenever a non-constant schedule is active on a finite run (in ∞ mode there is no horizon, so schedules switch off) — compresses the whole schedule into a fraction of the run, so at *4×* it finishes annealing a quarter of the way in and the rest of the run shows you the landing. Turn it up and read the result off the dotted $\\gamma(t)$ line in the loss chart.' + }, + { + kind: 'p', + text: 'Scheduling has a second, deeper payoff that only lands once gradients turn *noisy* — the subject of the next part. A $\\gamma$ bled toward zero is the one thing that pulls a restless run in to a clean stop. And one optimizer you’ll meet there, **Lion**, takes a fixed-size step and so cannot settle *at all* on a constant $\\gamma$: it just orbits the minimum forever. It is the purest illustration of why schedules exist — switch it to cosine and the orbit closes to a point.' + } +]; diff --git a/src/content/chapters/ch-step.ts b/src/content/chapters/ch-step.ts new file mode 100644 index 0000000..cf1b5e5 --- /dev/null +++ b/src/content/chapters/ch-step.ts @@ -0,0 +1,26 @@ +import type { Block } from '../blocks'; + +/** Chapter 6 · One step of descent — the update rule, and the two arrows. */ +export const chStep: Block[] = [ + { + kind: 'p', + text: 'Now we can actually walk. One step of **gradient descent** is almost insultingly simple:' + }, + { + kind: 'recipe', + text: 'Stand at your current $(\\alpha, \\beta)$. Look downhill — that’s **$-\\nabla\\mathcal{L}$**. Take a step in that direction — {g:$\\gamma$} times as long as the slope is steep. Repeat.' + }, + { + kind: 'p', + text: 'In symbols, that is the rule the entire field is built on. We write **$\\theta$** (“theta”) as shorthand for the pair $(\\alpha, \\beta)$ together:' + }, + { kind: 'display', formula: 'stepRule', center: true }, + { + kind: 'p', + text: 'Press **Step** to take exactly one of these; press **Train** to take many in a row and watch the marker slide into a valley.' + }, + { + kind: 'p', + text: 'You’ll now notice a {red:red arrow} on the marker beside the blue one. The blue arrow is the pure downhill direction; the **red arrow is the step the optimizer actually took**. Early on they almost agree. Once you add the tricks in Part IV, they’ll split apart — and *that gap is the optimizer’s personality.*' + } +]; diff --git a/src/content/chapters/chapters.test.ts b/src/content/chapters/chapters.test.ts index 911ad66..31b5652 100644 --- a/src/content/chapters/chapters.test.ts +++ b/src/content/chapters/chapters.test.ts @@ -4,8 +4,15 @@ import { chapterBySlug } from '../registry'; import { formulas } from '../formulas'; import { parseRich } from '../rich'; -const richFields = (b: Record): string[] => - ['text', 'caption'].flatMap(k => (typeof b[k] === 'string' ? [b[k] as string] : [])); +const richFields = (b: Record): string[] => [ + ...['text', 'caption'].flatMap(k => (typeof b[k] === 'string' ? [b[k] as string] : [])), + ...(Array.isArray(b.items) ? (b.items as string[]) : []), + ...(Array.isArray(b.paras) ? (b.paras as string[]) : []), + // proof blocks nest one level + ...(Array.isArray(b.blocks) + ? (b.blocks as Record[]).flatMap(inner => richFields(inner)) + : []) +]; describe('chapter blocks', () => { it('keys every migrated chapter by a real registry slug', () => { diff --git a/src/content/chapters/index.ts b/src/content/chapters/index.ts index 35818d0..9ab85de 100644 --- a/src/content/chapters/index.ts +++ b/src/content/chapters/index.ts @@ -8,9 +8,21 @@ import type { Block } from '../blocks'; import { chBowl } from './ch-bowl'; import { chLandscape } from './ch-landscape'; import { chShapes } from './ch-shapes'; +import { chDerivative } from './ch-derivative'; +import { chDownhill } from './ch-downhill'; +import { chStep } from './ch-step'; +import { chGamma } from './ch-gamma'; +import { chCurvature } from './ch-curvature'; +import { chSchedule } from './ch-schedule'; export const chapterBlocks: Record = { 'ch-bowl': chBowl, 'ch-landscape': chLandscape, - 'ch-shapes': chShapes + 'ch-shapes': chShapes, + 'ch-derivative': chDerivative, + 'ch-downhill': chDownhill, + 'ch-step': chStep, + 'ch-gamma': chGamma, + 'ch-curvature': chCurvature, + 'ch-schedule': chSchedule }; diff --git a/src/content/rich.test.ts b/src/content/rich.test.ts index c7b06ca..30fc885 100644 --- a/src/content/rich.test.ts +++ b/src/content/rich.test.ts @@ -83,6 +83,12 @@ describe('richToHtml', () => { expect(html).toContain(''); }); + it('renders ink spans for the app arrows', () => { + expect(richToHtml('the {blue:blue arrow} and its {red:red partner}', true)).toBe( + 'the blue arrow and its red partner' + ); + }); + it('picks the matching theme span only', () => { const src = 'the {dark:bright}{light:deep-coloured} dimple'; expect(richToHtml(src, true)).toBe('the bright dimple'); diff --git a/src/content/rich.ts b/src/content/rich.ts index fc9d17d..35cf9be 100644 --- a/src/content/rich.ts +++ b/src/content/rich.ts @@ -6,6 +6,8 @@ * **...** strong * *...* emphasis * {g:...} a knob symbol at first mention (em.g in-app; italic in print) + * {blue:...} ink for the app's blue −∇ℒ arrow (.ink-blue; plain bold in print) + * {red:...} ink for the app's red Δθ arrow (.ink-red; plain bold in print) * {dark:...} rendered only in dark mode * {light:...} rendered only in day mode — and always in print * @@ -24,6 +26,7 @@ export type RichToken = | { t: 'strong'; children: RichToken[] } | { t: 'em'; children: RichToken[] } | { t: 'g'; children: RichToken[] } + | { t: 'ink'; color: 'blue' | 'red'; children: RichToken[] } | { t: 'theme'; mode: 'dark' | 'light'; children: RichToken[] }; /** Find `close` at or after `from`, skipping over $...$ math spans. */ @@ -66,6 +69,8 @@ function findClosingBrace(src: string, from: number): number { const SPAN_OPENERS: { prefix: string; make: (children: RichToken[]) => RichToken }[] = [ { prefix: '{g:', make: children => ({ t: 'g', children }) }, + { prefix: '{blue:', make: children => ({ t: 'ink', color: 'blue', children }) }, + { prefix: '{red:', make: children => ({ t: 'ink', color: 'red', children }) }, { prefix: '{dark:', make: children => ({ t: 'theme', mode: 'dark', children }) }, { prefix: '{light:', make: children => ({ t: 'theme', mode: 'light', children }) } ]; @@ -153,6 +158,9 @@ function tokensToHtml(tokens: RichToken[], dark: boolean): string { case 'g': html += `${tokensToHtml(tk.children, dark)}`; break; + case 'ink': + html += `${tokensToHtml(tk.children, dark)}`; + break; case 'theme': if ((tk.mode === 'dark') === dark) html += tokensToHtml(tk.children, dark); break;