From ef32c042670e909d65c97faeacba0a2c2afd3c44 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 1 Jul 2026 23:49:05 -0500 Subject: [PATCH 1/4] =?UTF-8?q?fix(guide):=20every=20optimizer=20formula?= =?UTF-8?q?=20walks=20the=20whole=20way=20from=20=E2=88=87=E2=84=92=20to?= =?UTF-8?q?=20=CE=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several cards showed only the final step and expected the reader to carry m, s, h from earlier cards — the loss never appeared, so the formula couldn't be eyeballed on its own. Every card now shows the complete pipeline (state feeds from ∇ℒ, then the parameter update), repeating the EMA pattern deliberately so each card is self-contained: - Adam: both memory feeds + the step (the bias-correction hats are narrated in the prose right beside it) - AdamW: the same two feeds, so the one change (+λθ) pops - RAdam: feeds + the rectified step; ρ∞ bookkeeping stays in prose, which now names the rectification factor r_t - Sophia: the loss enters twice — m from ∇ℒ, h from diag(H) - Prodigy: d-weighted feeds, the growing estimate, the step - AdaDelta: the missing s-feed opens the chain The core updateRuleLatex strings (the Formulas panel) are synced to the same vocabulary — Prodigy's still had the old v/s notation, Sophia and AdamW's were missing their feeds, and the tight (1{-}β) minus is now the spaced (1-β) everywhere. The panel's fit-to-width had a silent-clipping mode: at its 11.5px floor, over-long rows were cut mid-formula (the Gaussian-Mixture model row already hit this). Floor lowered to 10px — the complete rules fit on laptop widths — and the viewport scrolls instead of clipping past the floor. Verified live: all fifteen cards contain ∇ℒ (Sophia also 𝐇) with zero overflow at desktop width; formulas soft-wrap cleanly on mobile; the panel shows Prodigy's full rule at 10.2px uncut; 194 tests, svelte-check clean. Co-Authored-By: Claude Fable 5 --- src/components/GuidePanel.svelte | 9 +++++++-- src/content/optimizerCards.ts | 14 +++++++------- src/optim/optimizers/adadelta.ts | 2 +- src/optim/optimizers/adam.ts | 2 +- src/optim/optimizers/adamw.ts | 2 +- src/optim/optimizers/lion.ts | 2 +- src/optim/optimizers/nadam.ts | 2 +- src/optim/optimizers/prodigy.ts | 2 +- src/optim/optimizers/radam.ts | 2 +- src/optim/optimizers/sophia.ts | 2 +- 10 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/components/GuidePanel.svelte b/src/components/GuidePanel.svelte index 7cdd4f5..ccca296 100644 --- a/src/components/GuidePanel.svelte +++ b/src/components/GuidePanel.svelte @@ -217,7 +217,10 @@ let ro: ResizeObserver | undefined; const BASE_PX = 16; // measurement baseline - const MIN_PX = 11.5; // never smaller than this + // The floor allows the complete update rules (Prodigy is the longest) to + // fit on typical laptop widths; below the floor, the viewport scrolls + // rather than clipping mid-formula. + const MIN_PX = 10; // never smaller than this const MAX_PX = 23; // never larger than this function fitFormulas() { @@ -346,7 +349,9 @@ min-height: 0; display: flex; align-items: center; - overflow: hidden; + /* If even the floored font can't fit a row, scroll — never clip a formula. */ + overflow-x: auto; + overflow-y: hidden; } .formula-fit { diff --git a/src/content/optimizerCards.ts b/src/content/optimizerCards.ts index 0f480b9..038b957 100644 --- a/src/content/optimizerCards.ts +++ b/src/content/optimizerCards.ts @@ -114,7 +114,7 @@ export const optTree: OptChapter[] = [ by: 'Matthew Zeiler — same year, same fix, one step further', idea: 'RMSProp’s twin, born the same year against the same AdaGrad flaw — but Zeiler spotted a deeper oddity: a raw gradient step has the wrong units. AdaDelta divides by $\\mathrm{RMS}[\\nabla\\mathcal{L}]$ like RMSProp, then multiplies by the RMS of its OWN recent steps. That second memory hands the step $\\theta$’s own units — the very thing a raw gradient step lacks, and exactly what Newton’s $\\mathbf H^{-1}\\nabla\\mathcal{L}$ buys with curvature — so no unit-carrying $\\gamma$ is needed and the learning rate falls out of the math entirely: there is nothing left to set but the decay $\\rho$.', - formula: String.raw`\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L}, \qquad \mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2`, + formula: String.raw`\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2`, fix: 'no learning rate to tune — it sizes its own steps', brk: 'one knob fewer, but no $\\gamma$ to crank when you DO want it faster' }, @@ -127,7 +127,7 @@ export const optTree: OptChapter[] = [ by: 'Kingma & Ba — "adaptive moments"', idea: 'The merger the whole trunk builds to: take Momentum’s moving average of gradients (decay $\\beta_1$) AND RMSProp’s moving average of squared gradients (decay $\\beta_2$), and use them together. One honest detail: both averages start at zero and read too low at first, so each is divided by $1-\\beta^t$ to correct that early bias — giving the bias-corrected $\\hat{\\mathbf{m}}$ and $\\hat{\\mathbf{s}}$ that the update below pits against each other. The result became the workhorse of modern deep learning — its paper is now one of the most-cited in all of science — and the launch point for every branch that follows.', - formula: String.raw`\hat{\mathbf{m}} = \frac{\mathbf{m}}{1-\beta_1^t}, \quad \hat{\mathbf{s}} = \frac{\mathbf{s}}{1-\beta_2^t}, \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, fix: 'robust out of the box almost everywhere', brk: 'not perfect — three later papers each sand down one rough edge' }, @@ -152,7 +152,7 @@ export const optTree: OptChapter[] = [ by: 'Loshchilov & Hutter — the actual default today', idea: 'The refinement that matters most: nearly every large model — GPT, BERT, the lot — trains with AdamW, not plain Adam. Weight decay gently pulls every parameter toward zero to curb overfitting; Adam folded that pull into the gradient, where its adaptive $\\sqrt{\\hat{\\mathbf{s}}}$ scaling then distorted it. AdamW decouples them — the $\\lambda\\boldsymbol\\theta$ decay lands straight on $\\boldsymbol\\theta$, outside the scaling. One honest caveat here: these toy losses carry no overfitting to regularize, so $\\lambda$ shows up as a literal, visible pull of the marker toward the origin. Crank it and watch the fit drift inward; set $\\lambda$ to 0 and you are back to exact Adam.', - formula: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, + formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, fix: 'decoupled decay — why it’s the real-world default', brk: 'with no overfitting to fight here, $\\lambda$ is a pull toward 0 more than a regularizer' }, @@ -164,8 +164,8 @@ export const optTree: OptChapter[] = [ lead: 'The third thread was the quietest of all. For years practitioners had patched a rough spot in Adam’s opening steps with a hand-tuned warmup, half-superstition — runs just blew up without it, and nobody could say exactly why. What if that warmup could be derived instead of guessed?', by: 'Liu et al. — Adam’s warmup, automated', idea: - 'The third refinement closes a quieter Adam wart. In the first handful of steps Adam has barely any squared-gradient history, so its $\\sqrt{\\hat{\\mathbf{s}}}$ scaling is pure noise — the practitioner’s fix was a hand-tuned warmup that crept the rate up by hand. RAdam computes how trustworthy that variance actually is (a number $\\rho_t$) and, until it can be trusted, just skips the scaling and takes a plain momentum step. A rectification factor then eases the adaptive part in. Warmup, but derived rather than guessed — nothing to tune.', - formula: String.raw`\rho_t = \rho_\infty - \frac{2t\,\beta_2^{t}}{1-\beta_2^{t}}, \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}}+\varepsilon}\;\;(\rho_t > 4)`, + 'The third refinement closes a quieter Adam wart. In the first handful of steps Adam has barely any squared-gradient history, so its $\\sqrt{\\hat{\\mathbf{s}}}$ scaling is pure noise — the practitioner’s fix was a hand-tuned warmup that crept the rate up by hand. RAdam computes how trustworthy that variance actually is (a number $\\rho_t$) and, until it can be trusted, just skips the scaling and takes a plain momentum step. A rectification factor $r_t$ — near zero at first, easing toward one — then lets the adaptive part in. Warmup, but derived rather than guessed — nothing to tune.', + formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, fix: 'an automatic warmup — no schedule to hand-tune', brk: 'only smooths the opening steps; past warmup it just is Adam' }, @@ -206,7 +206,7 @@ export const optTree: OptChapter[] = [ by: 'Liu et al. — Newton, cut down to fit an LLM', idea: 'Newton’s curvature is unbeatable and unaffordable; Sophia keeps the affordable part. Drop the full Hessian for just its DIAGONAL — one curvature number $\\mathbf h$ per parameter, no matrix to invert — and precondition the momentum by it. Then the safety move: CLIP every coordinate’s step to $\\pm\\rho$ (Sophia’s own $\\rho$, a clip radius — no relation to RMSProp’s decay; the alphabet is small and the field is greedy). Where the diagonal estimate is tiny or noisy (and $\\mathbf m/\\mathbf h$ would blow up) the clip bounds the move; where it’s solid, the step stays curvature-scaled. Its paper reports GPT-2-scale pretraining in roughly half the steps Adam needs — a headline later independent benchmarks have contested — but either way, it is second-order thinking made cheap enough to try.', - formula: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, + formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, fix: 'diagonal curvature + a clip — second-order on a budget', brk: 'only the diagonal: blind to the off-axis stretch Newton corrects' }, @@ -219,7 +219,7 @@ export const optTree: OptChapter[] = [ by: 'Mishchenko & Defazio — the learning rate, removed', idea: 'Every method so far still made you pick $\\gamma$. This branch deletes that last knob. The insight: the ideal step size is set by how far the start is from the solution — a distance $d$. You don’t know $d$, so Prodigy estimates it live, ramping a tiny seed upward from how the gradients line up with how far you’ve already travelled ($\\langle g,\\, x_0 - x\\rangle$ — in the formula, $r$ and $\\mathbf{w}$ are two running tallies of exactly that alignment), and scales an Adam step by it. Set nothing and watch the marker creep, then accelerate as $d$ finds its level — the learning rate, discovered rather than tuned. Prodigy sharpens the same lab’s earlier D-Adaptation, and the parameter-free idea is taken seriously: a sibling schedule-free method from these authors won the self-tuning track of MLCommons’ 2024 AlgoPerf benchmark.', - formula: String.raw`d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`, + formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2, \;\; d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`, fix: 'no learning rate to choose — it finds its own', brk: 'the estimate only climbs, so a bad early ramp can overshoot' } diff --git a/src/optim/optimizers/adadelta.ts b/src/optim/optimizers/adadelta.ts index 3c67bca..1184db6 100644 --- a/src/optim/optimizers/adadelta.ts +++ b/src/optim/optimizers/adadelta.ts @@ -42,7 +42,7 @@ export const adadelta: CoreOptimizer = { id: 'adadelta', name: 'AdaDelta', description: 'RMSProp with no learning rate — the units fix themselves', - updateRuleLatex: String.raw`\mathbf{s} \leftarrow \rho \mathbf{s} + (1{-}\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u} + \varepsilon}}{\sqrt{\mathbf{s} + \varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho \mathbf{u} + (1{-}\rho)\Delta\boldsymbol{\theta}^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}`, + updateRuleLatex: String.raw`\mathbf{s} \leftarrow \rho \mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u} + \varepsilon}}{\sqrt{\mathbf{s} + \varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho \mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}`, hyperparams: [ADADELTA_RHO], fixedLearningRate: 1.0, init: (d) => ({ sG: zeros(d), sX: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/adam.ts b/src/optim/optimizers/adam.ts index 320bf93..e78381c 100644 --- a/src/optim/optimizers/adam.ts +++ b/src/optim/optimizers/adam.ts @@ -23,7 +23,7 @@ export const adam: CoreOptimizer = { id: 'adam', name: 'Adam', description: 'Momentum + per-parameter scaling, bias-corrected', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1{-}\beta_1) \nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1{-}\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, hyperparams: [BETA1_SPEC, BETA2_SPEC], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/adamw.ts b/src/optim/optimizers/adamw.ts index b20bdb1..8af2c07 100644 --- a/src/optim/optimizers/adamw.ts +++ b/src/optim/optimizers/adamw.ts @@ -37,7 +37,7 @@ export const adamw: CoreOptimizer = { id: 'adamw', name: 'AdamW', description: 'Adam with decoupled weight decay — the real default', - updateRuleLatex: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, + updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, hyperparams: [BETA1_SPEC, BETA2_SPEC, ADAMW_WD], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/lion.ts b/src/optim/optimizers/lion.ts index 119e230..4f8ac14 100644 --- a/src/optim/optimizers/lion.ts +++ b/src/optim/optimizers/lion.ts @@ -47,7 +47,7 @@ export const lion: CoreOptimizer = { id: 'lion', name: 'Lion', description: 'Sign of momentum: fixed-size steps, very light', - updateRuleLatex: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1{-}\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1{-}\beta_2)\nabla\mathcal{L}`, + updateRuleLatex: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}`, hyperparams: [LION_BETA1, LION_BETA2], fixedLearningRate: 0.05, init: (d) => ({ m: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/nadam.ts b/src/optim/optimizers/nadam.ts index 9f46c73..fd4db17 100644 --- a/src/optim/optimizers/nadam.ts +++ b/src/optim/optimizers/nadam.ts @@ -24,7 +24,7 @@ export const nadam: CoreOptimizer = { id: 'nadam', name: 'Nadam', description: 'Adam with Nesterov look-ahead on the momentum', - updateRuleLatex: String.raw`\bar{\mathbf{m}} = \beta_1 \hat{\mathbf{m}} + \frac{(1{-}\beta_1)\nabla \mathcal{L}}{1-\beta_1^t}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\bar{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + updateRuleLatex: String.raw`\bar{\mathbf{m}} = \beta_1 \hat{\mathbf{m}} + \frac{(1-\beta_1)\nabla \mathcal{L}}{1-\beta_1^t}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\bar{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, hyperparams: [BETA1_SPEC, BETA2_SPEC], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/prodigy.ts b/src/optim/optimizers/prodigy.ts index e74527b..a5a2935 100644 --- a/src/optim/optimizers/prodigy.ts +++ b/src/optim/optimizers/prodigy.ts @@ -64,7 +64,7 @@ export const prodigy: CoreOptimizer = { id: 'prodigy', name: 'Prodigy', description: 'Parameter-free: estimates its own learning rate', - updateRuleLatex: String.raw`d_{t+1} = \max\!\left(d_t,\, \frac{r_{t+1}}{\lVert \mathbf{s}_{t+1}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d_t\,\frac{\mathbf{m}}{\sqrt{\mathbf{v}} + d_t\varepsilon}`, + updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2, \;\; d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`, hyperparams: [PRODIGY_BETA1, PRODIGY_BETA2], fixedLearningRate: 1.0, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0, dEst: NaN, rNum: 0, sDen: zeros(d), x0: null }), diff --git a/src/optim/optimizers/radam.ts b/src/optim/optimizers/radam.ts index 007a7fe..2b72d69 100644 --- a/src/optim/optimizers/radam.ts +++ b/src/optim/optimizers/radam.ts @@ -26,7 +26,7 @@ export const radam: CoreOptimizer = { id: 'radam', name: 'RAdam', description: 'Adam with a built-in, automatic warmup', - updateRuleLatex: String.raw`\rho_t = \rho_\infty - \frac{2t\,\beta_2^{t}}{1-\beta_2^{t}}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\;\;(\rho_t > 4),\;\; \text{else}\; -\gamma\,\hat{\mathbf{m}}`, + updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, hyperparams: [BETA1_SPEC, BETA2_SPEC], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/sophia.ts b/src/optim/optimizers/sophia.ts index ccc6c60..1a06f69 100644 --- a/src/optim/optimizers/sophia.ts +++ b/src/optim/optimizers/sophia.ts @@ -54,7 +54,7 @@ export const sophia: CoreOptimizer = { id: 'sophia', name: 'Sophia', description: 'Diagonal curvature, with clipped steps', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1\mathbf{m} + (1{-}\beta_1)\nabla \mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, + updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, hyperparams: [SOPHIA_BETA1, SOPHIA_RHO], fixedLearningRate: 0.1, usesHessian: true, From 3f6fb43f01dff625d1910c6925e65dd4882a7d50 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Thu, 2 Jul 2026 00:07:02 -0500 Subject: [PATCH 2/4] style(guide): long update rules break at clause boundaries, never mid-equation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KaTeX's inline soft-wrap could split a clause anywhere — on mobile, Prodigy's s-feed broke after '(1−'. Every formula with three or more clauses (Adam, AdamW, RAdam, Sophia, Prodigy, AdaDelta, Lion) is now an aligned block, one clause per row — it reads like the pseudocode it is, and a row either fits or the container scrolls; an equation never tears. Cards and the Formulas panel share the stacked form (the panel's AdaDelta keeps its honest θ ← θ + γΔθ tail). Bonus: the stacked Update row is much narrower, so the panel's fit recovers from its font floor — Prodigy now renders at 12.1px (was clipped at 11.5px, then 10.2px) with zero clipping on either axis. Verified: desktop cards read as clean rows; mobile shows zero overflow on all seven stacked cards with no mid-clause breaks; panel checked with Prodigy selected; 194 tests, svelte-check clean. Co-Authored-By: Claude Fable 5 --- src/content/optimizerCards.ts | 14 +++++++------- src/optim/optimizers/adadelta.ts | 2 +- src/optim/optimizers/adam.ts | 2 +- src/optim/optimizers/adamw.ts | 2 +- src/optim/optimizers/lion.ts | 2 +- src/optim/optimizers/prodigy.ts | 2 +- src/optim/optimizers/radam.ts | 2 +- src/optim/optimizers/sophia.ts | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/content/optimizerCards.ts b/src/content/optimizerCards.ts index 038b957..f216aa8 100644 --- a/src/content/optimizerCards.ts +++ b/src/content/optimizerCards.ts @@ -114,7 +114,7 @@ export const optTree: OptChapter[] = [ by: 'Matthew Zeiler — same year, same fix, one step further', idea: 'RMSProp’s twin, born the same year against the same AdaGrad flaw — but Zeiler spotted a deeper oddity: a raw gradient step has the wrong units. AdaDelta divides by $\\mathrm{RMS}[\\nabla\\mathcal{L}]$ like RMSProp, then multiplies by the RMS of its OWN recent steps. That second memory hands the step $\\theta$’s own units — the very thing a raw gradient step lacks, and exactly what Newton’s $\\mathbf H^{-1}\\nabla\\mathcal{L}$ buys with curvature — so no unit-carrying $\\gamma$ is needed and the learning rate falls out of the math entirely: there is nothing left to set but the decay $\\rho$.', - formula: String.raw`\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2`, + formula: String.raw`\begin{aligned}&\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2 \\[2pt] &\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L} \\[2pt] &\mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2\end{aligned}`, fix: 'no learning rate to tune — it sizes its own steps', brk: 'one knob fewer, but no $\\gamma$ to crank when you DO want it faster' }, @@ -127,7 +127,7 @@ export const optTree: OptChapter[] = [ by: 'Kingma & Ba — "adaptive moments"', idea: 'The merger the whole trunk builds to: take Momentum’s moving average of gradients (decay $\\beta_1$) AND RMSProp’s moving average of squared gradients (decay $\\beta_2$), and use them together. One honest detail: both averages start at zero and read too low at first, so each is divided by $1-\\beta^t$ to correct that early bias — giving the bias-corrected $\\hat{\\mathbf{m}}$ and $\\hat{\\mathbf{s}}$ that the update below pits against each other. The result became the workhorse of modern deep learning — its paper is now one of the most-cited in all of science — and the launch point for every branch that follows.', - formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`, fix: 'robust out of the box almost everywhere', brk: 'not perfect — three later papers each sand down one rough edge' }, @@ -152,7 +152,7 @@ export const optTree: OptChapter[] = [ by: 'Loshchilov & Hutter — the actual default today', idea: 'The refinement that matters most: nearly every large model — GPT, BERT, the lot — trains with AdamW, not plain Adam. Weight decay gently pulls every parameter toward zero to curb overfitting; Adam folded that pull into the gradient, where its adaptive $\\sqrt{\\hat{\\mathbf{s}}}$ scaling then distorted it. AdamW decouples them — the $\\lambda\\boldsymbol\\theta$ decay lands straight on $\\boldsymbol\\theta$, outside the scaling. One honest caveat here: these toy losses carry no overfitting to regularize, so $\\lambda$ shows up as a literal, visible pull of the marker toward the origin. Crank it and watch the fit drift inward; set $\\lambda$ to 0 and you are back to exact Adam.', - formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, + formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)\end{aligned}`, fix: 'decoupled decay — why it’s the real-world default', brk: 'with no overfitting to fight here, $\\lambda$ is a pull toward 0 more than a regularizer' }, @@ -165,7 +165,7 @@ export const optTree: OptChapter[] = [ by: 'Liu et al. — Adam’s warmup, automated', idea: 'The third refinement closes a quieter Adam wart. In the first handful of steps Adam has barely any squared-gradient history, so its $\\sqrt{\\hat{\\mathbf{s}}}$ scaling is pure noise — the practitioner’s fix was a hand-tuned warmup that crept the rate up by hand. RAdam computes how trustworthy that variance actually is (a number $\\rho_t$) and, until it can be trusted, just skips the scaling and takes a plain momentum step. A rectification factor $r_t$ — near zero at first, easing toward one — then lets the adaptive part in. Warmup, but derived rather than guessed — nothing to tune.', - formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`, fix: 'an automatic warmup — no schedule to hand-tune', brk: 'only smooths the opening steps; past warmup it just is Adam' }, @@ -178,7 +178,7 @@ export const optTree: OptChapter[] = [ by: 'Chen et al. (Google) — found by program search, not designed', idea: 'Adam scaled the step by gradient history. Lion throws that out and takes a different shape — and it wasn’t invented by a person: a program searched the space of optimizers and this fell out. The name is a fitting backronym — EvoLved Sign Momentum. Keep one momentum buffer, blend it with the fresh gradient, and step by the $\\operatorname{sign}$ of the result — so every step is the same size $\\gamma$ on each axis, no matter how steep or flat. That makes it light (one buffer, no squared-gradient term) and competitive with Adam on big vision and language models. The catch is the very thing that makes it clean: a step that never shrinks can’t settle by itself.', - formula: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}`, + formula: String.raw`\begin{aligned}&\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L} \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}) \\[2pt] &\mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}\end{aligned}`, fix: 'fixed-size steps from one tiny buffer — light and fast', brk: 'the step never shrinks, so it orbits the minimum until $\\gamma$ is decayed by a schedule', hd: 'Watch Lion’s red step arrow: with two knobs, $\\operatorname{sign}(\\mathbf c)$ can only point in eight directions — the axes and the four diagonals. That is the whole geometry of a sign step: it moves $\\gamma$ along every axis at once, so in $d$ dimensions its true length is $\\gamma\\sqrt{d}$ no matter how faint the gradient, and it can point far from steepest descent. At a billion parameters that $\\sqrt{d}$ is enormous — which is why Lion runs on a much smaller $\\gamma$ than Adam.' @@ -206,7 +206,7 @@ export const optTree: OptChapter[] = [ by: 'Liu et al. — Newton, cut down to fit an LLM', idea: 'Newton’s curvature is unbeatable and unaffordable; Sophia keeps the affordable part. Drop the full Hessian for just its DIAGONAL — one curvature number $\\mathbf h$ per parameter, no matrix to invert — and precondition the momentum by it. Then the safety move: CLIP every coordinate’s step to $\\pm\\rho$ (Sophia’s own $\\rho$, a clip radius — no relation to RMSProp’s decay; the alphabet is small and the field is greedy). Where the diagonal estimate is tiny or noisy (and $\\mathbf m/\\mathbf h$ would blow up) the clip bounds the move; where it’s solid, the step stays curvature-scaled. Its paper reports GPT-2-scale pretraining in roughly half the steps Adam needs — a headline later independent benchmarks have contested — but either way, it is second-order thinking made cheap enough to try.', - formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, + formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)\end{aligned}`, fix: 'diagonal curvature + a clip — second-order on a budget', brk: 'only the diagonal: blind to the off-axis stretch Newton corrects' }, @@ -219,7 +219,7 @@ export const optTree: OptChapter[] = [ by: 'Mishchenko & Defazio — the learning rate, removed', idea: 'Every method so far still made you pick $\\gamma$. This branch deletes that last knob. The insight: the ideal step size is set by how far the start is from the solution — a distance $d$. You don’t know $d$, so Prodigy estimates it live, ramping a tiny seed upward from how the gradients line up with how far you’ve already travelled ($\\langle g,\\, x_0 - x\\rangle$ — in the formula, $r$ and $\\mathbf{w}$ are two running tallies of exactly that alignment), and scales an Adam step by it. Set nothing and watch the marker creep, then accelerate as $d$ finds its level — the learning rate, discovered rather than tuned. Prodigy sharpens the same lab’s earlier D-Adaptation, and the parameter-free idea is taken seriously: a sibling schedule-free method from these authors won the self-tuning track of MLCommons’ 2024 AlgoPerf benchmark.', - formula: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2, \;\; d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`, + formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2 \\[2pt] &d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}\end{aligned}`, fix: 'no learning rate to choose — it finds its own', brk: 'the estimate only climbs, so a bad early ramp can overshoot' } diff --git a/src/optim/optimizers/adadelta.ts b/src/optim/optimizers/adadelta.ts index 1184db6..746eefb 100644 --- a/src/optim/optimizers/adadelta.ts +++ b/src/optim/optimizers/adadelta.ts @@ -42,7 +42,7 @@ export const adadelta: CoreOptimizer = { id: 'adadelta', name: 'AdaDelta', description: 'RMSProp with no learning rate — the units fix themselves', - updateRuleLatex: String.raw`\mathbf{s} \leftarrow \rho \mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u} + \varepsilon}}{\sqrt{\mathbf{s} + \varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho \mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2 \\[2pt] &\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L} \\[2pt] &\mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}\end{aligned}`, hyperparams: [ADADELTA_RHO], fixedLearningRate: 1.0, init: (d) => ({ sG: zeros(d), sX: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/adam.ts b/src/optim/optimizers/adam.ts index e78381c..9ce275b 100644 --- a/src/optim/optimizers/adam.ts +++ b/src/optim/optimizers/adam.ts @@ -23,7 +23,7 @@ export const adam: CoreOptimizer = { id: 'adam', name: 'Adam', description: 'Momentum + per-parameter scaling, bias-corrected', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`, hyperparams: [BETA1_SPEC, BETA2_SPEC], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/adamw.ts b/src/optim/optimizers/adamw.ts index 8af2c07..a1d875f 100644 --- a/src/optim/optimizers/adamw.ts +++ b/src/optim/optimizers/adamw.ts @@ -37,7 +37,7 @@ export const adamw: CoreOptimizer = { id: 'adamw', name: 'AdamW', description: 'Adam with decoupled weight decay — the real default', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)\end{aligned}`, hyperparams: [BETA1_SPEC, BETA2_SPEC, ADAMW_WD], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/lion.ts b/src/optim/optimizers/lion.ts index 4f8ac14..6936d43 100644 --- a/src/optim/optimizers/lion.ts +++ b/src/optim/optimizers/lion.ts @@ -47,7 +47,7 @@ export const lion: CoreOptimizer = { id: 'lion', name: 'Lion', description: 'Sign of momentum: fixed-size steps, very light', - updateRuleLatex: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L} \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}) \\[2pt] &\mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}\end{aligned}`, hyperparams: [LION_BETA1, LION_BETA2], fixedLearningRate: 0.05, init: (d) => ({ m: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/prodigy.ts b/src/optim/optimizers/prodigy.ts index a5a2935..81e1d6d 100644 --- a/src/optim/optimizers/prodigy.ts +++ b/src/optim/optimizers/prodigy.ts @@ -64,7 +64,7 @@ export const prodigy: CoreOptimizer = { id: 'prodigy', name: 'Prodigy', description: 'Parameter-free: estimates its own learning rate', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2, \;\; d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2 \\[2pt] &d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}\end{aligned}`, hyperparams: [PRODIGY_BETA1, PRODIGY_BETA2], fixedLearningRate: 1.0, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0, dEst: NaN, rNum: 0, sDen: zeros(d), x0: null }), diff --git a/src/optim/optimizers/radam.ts b/src/optim/optimizers/radam.ts index 2b72d69..3b6404c 100644 --- a/src/optim/optimizers/radam.ts +++ b/src/optim/optimizers/radam.ts @@ -26,7 +26,7 @@ export const radam: CoreOptimizer = { id: 'radam', name: 'RAdam', description: 'Adam with a built-in, automatic warmup', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`, hyperparams: [BETA1_SPEC, BETA2_SPEC], fixedLearningRate: 0.1, init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }), diff --git a/src/optim/optimizers/sophia.ts b/src/optim/optimizers/sophia.ts index 1a06f69..c44b534 100644 --- a/src/optim/optimizers/sophia.ts +++ b/src/optim/optimizers/sophia.ts @@ -54,7 +54,7 @@ export const sophia: CoreOptimizer = { id: 'sophia', name: 'Sophia', description: 'Diagonal curvature, with clipped steps', - updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L}, \;\; \mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`, + updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)\end{aligned}`, hyperparams: [SOPHIA_BETA1, SOPHIA_RHO], fixedLearningRate: 0.1, usesHessian: true, From f9eeb32e02ad81c152c0eecb838f6965ceaa1a4b Mon Sep 17 00:00:00 2001 From: NeoVand Date: Thu, 2 Jul 2026 00:10:38 -0500 Subject: [PATCH 3/4] =?UTF-8?q?revert(panel):=20no=20scrollbars=20in=20the?= =?UTF-8?q?=20Formulas=20panel=20=E2=80=94=20restore=20hidden=20overflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overflow-x:auto fallback and the lowered font floor from two commits ago put a scrollbar on the Formulas panel for long problem formulas. Wrong trade for this app: the panel is a quiet reference, not a scroll region. Both changes reverted — MIN_PX back to 11.5, overflow back to hidden — restoring the panel's exact original behavior. The clipping this originally chased is gone anyway: the stacked one-clause-per-row update rules are far narrower than the old single lines, so the fit sits comfortably above its floor (Prodigy: 12.1px, update row and block fully inside the viewport, no scrollbar — verified live). Co-Authored-By: Claude Fable 5 --- src/components/GuidePanel.svelte | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/components/GuidePanel.svelte b/src/components/GuidePanel.svelte index ccca296..7cdd4f5 100644 --- a/src/components/GuidePanel.svelte +++ b/src/components/GuidePanel.svelte @@ -217,10 +217,7 @@ let ro: ResizeObserver | undefined; const BASE_PX = 16; // measurement baseline - // The floor allows the complete update rules (Prodigy is the longest) to - // fit on typical laptop widths; below the floor, the viewport scrolls - // rather than clipping mid-formula. - const MIN_PX = 10; // never smaller than this + const MIN_PX = 11.5; // never smaller than this const MAX_PX = 23; // never larger than this function fitFormulas() { @@ -349,9 +346,7 @@ min-height: 0; display: flex; align-items: center; - /* If even the floored font can't fit a row, scroll — never clip a formula. */ - overflow-x: auto; - overflow-y: hidden; + overflow: hidden; } .formula-fit { From d6a2b9426683510b31f4f3e995b8874737dc5518 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Thu, 2 Jul 2026 00:13:20 -0500 Subject: [PATCH 4/4] fix(panel): the Update label leads the first row of a stacked rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centred against a four-row aligned block, the label floated beside the third row — reading as if the memory feeds above it belonged to the Gradient formula. Rows whose rule is stacked now top-align, with the label padded onto the first row's centreline; single-line rows are untouched. Co-Authored-By: Claude Fable 5 --- src/components/GuidePanel.svelte | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/GuidePanel.svelte b/src/components/GuidePanel.svelte index 7cdd4f5..bb5dd71 100644 --- a/src/components/GuidePanel.svelte +++ b/src/components/GuidePanel.svelte @@ -147,6 +147,10 @@ : gradientFormulas[problemType]; // The update rule tracks the selected optimizer $: updateFormula = optimizers[$optimizerStore.id].updateRuleLatex; + // Stacked (one-clause-per-row) rules top-align their label with the first + // row — centred, the label floats beside a middle row and reads as if the + // rows above belonged to the previous formula. + $: updateIsStacked = updateFormula.includes('\\begin{aligned}'); // Render LaTeX when component mounts or problem changes function renderLatex() { @@ -298,7 +302,7 @@ -
+
Update:
@@ -363,6 +367,9 @@ gap: 0.55em; white-space: nowrap; } + /* Multi-row (aligned) formulas: label sits in front of the FIRST row. */ + .equation-row.stacked { align-items: flex-start; } + .equation-row.stacked .equation-label { padding-top: 0.45em; } .equation-label { font-weight: 600;