Fix: every optimizer formula shows the full path from loss to update

src/components/GuidePanel.svelte

@@ -147,6 +147,10 @@
     : gradientFormulas[problemType];
   // The update rule tracks the selected optimizer
   $: updateFormula = optimizers[$optimizerStore.id].updateRuleLatex;
+  // Stacked (one-clause-per-row) rules top-align their label with the first
+  // row — centred, the label floats beside a middle row and reads as if the
+  // rows above belonged to the previous formula.
+  $: updateIsStacked = updateFormula.includes('\\begin{aligned}');
 
   // Render LaTeX when component mounts or problem changes
   function renderLatex() {
@@ -298,7 +302,7 @@
         <span class="latex-inline" bind:this={gradientFormulaElement}></span>
       </div>
 
-      <div class="equation-row">
+      <div class="equation-row" class:stacked={updateIsStacked}>
         <span class="equation-label">Update:</span>
         <span class="latex-inline" bind:this={updateFormulaElement}></span>
       </div>
@@ -363,6 +367,9 @@
     gap: 0.55em;
     white-space: nowrap;
   }
+  /* Multi-row (aligned) formulas: label sits in front of the FIRST row. */
+  .equation-row.stacked { align-items: flex-start; }
+  .equation-row.stacked .equation-label { padding-top: 0.45em; }
 
   .equation-label {
     font-weight: 600;

src/content/optimizerCards.ts

@@ -114,7 +114,7 @@ export const optTree: OptChapter[] = [
     by: 'Matthew Zeiler — same year, same fix, one step further',
     idea:
       'RMSProp’s twin, born the same year against the same AdaGrad flaw — but Zeiler spotted a deeper oddity: a raw gradient step has the wrong units. AdaDelta divides by $\\mathrm{RMS}[\\nabla\\mathcal{L}]$ like RMSProp, then multiplies by the RMS of its OWN recent steps. That second memory hands the step $\\theta$’s own units — the very thing a raw gradient step lacks, and exactly what Newton’s $\\mathbf H^{-1}\\nabla\\mathcal{L}$ buys with curvature — so no unit-carrying $\\gamma$ is needed and the learning rate falls out of the math entirely: there is nothing left to set but the decay $\\rho$.',
-    formula: String.raw`\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L}, \qquad \mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2`,
+    formula: String.raw`\begin{aligned}&\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2 \\[2pt] &\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L} \\[2pt] &\mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2\end{aligned}`,
     fix: 'no learning rate to tune — it sizes its own steps',
     brk: 'one knob fewer, but no $\\gamma$ to crank when you DO want it faster'
   },
@@ -127,7 +127,7 @@ export const optTree: OptChapter[] = [
     by: 'Kingma & Ba — "adaptive moments"',
     idea:
       'The merger the whole trunk builds to: take Momentum’s moving average of gradients (decay $\\beta_1$) AND RMSProp’s moving average of squared gradients (decay $\\beta_2$), and use them together. One honest detail: both averages start at zero and read too low at first, so each is divided by $1-\\beta^t$ to correct that early bias — giving the bias-corrected $\\hat{\\mathbf{m}}$ and $\\hat{\\mathbf{s}}$ that the update below pits against each other. The result became the workhorse of modern deep learning — its paper is now one of the most-cited in all of science — and the launch point for every branch that follows.',
-    formula: String.raw`\hat{\mathbf{m}} = \frac{\mathbf{m}}{1-\beta_1^t}, \quad \hat{\mathbf{s}} = \frac{\mathbf{s}}{1-\beta_2^t}, \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`,
+    formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`,
     fix: 'robust out of the box almost everywhere',
     brk: 'not perfect — three later papers each sand down one rough edge'
   },
@@ -152,7 +152,7 @@ export const optTree: OptChapter[] = [
     by: 'Loshchilov & Hutter — the actual default today',
     idea:
       'The refinement that matters most: nearly every large model — GPT, BERT, the lot — trains with AdamW, not plain Adam. Weight decay gently pulls every parameter toward zero to curb overfitting; Adam folded that pull into the gradient, where its adaptive $\\sqrt{\\hat{\\mathbf{s}}}$ scaling then distorted it. AdamW decouples them — the $\\lambda\\boldsymbol\\theta$ decay lands straight on $\\boldsymbol\\theta$, outside the scaling. One honest caveat here: these toy losses carry no overfitting to regularize, so $\\lambda$ shows up as a literal, visible pull of the marker toward the origin. Crank it and watch the fit drift inward; set $\\lambda$ to 0 and you are back to exact Adam.',
-    formula: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`,
+    formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)\end{aligned}`,
     fix: 'decoupled decay — why it’s the real-world default',
     brk: 'with no overfitting to fight here, $\\lambda$ is a pull toward 0 more than a regularizer'
   },
@@ -164,8 +164,8 @@ export const optTree: OptChapter[] = [
     lead: 'The third thread was the quietest of all. For years practitioners had patched a rough spot in Adam’s opening steps with a hand-tuned <em>warmup</em>, half-superstition — runs just blew up without it, and nobody could say exactly why. What if that warmup could be <em>derived</em> instead of guessed?',
     by: 'Liu et al. — Adam’s warmup, automated',
     idea:
-      'The third refinement closes a quieter Adam wart. In the first handful of steps Adam has barely any squared-gradient history, so its $\\sqrt{\\hat{\\mathbf{s}}}$ scaling is pure noise — the practitioner’s fix was a hand-tuned warmup that crept the rate up by hand. RAdam computes how trustworthy that variance actually is (a number $\\rho_t$) and, until it can be trusted, just skips the scaling and takes a plain momentum step. A rectification factor then eases the adaptive part in. Warmup, but derived rather than guessed — nothing to tune.',
-    formula: String.raw`\rho_t = \rho_\infty - \frac{2t\,\beta_2^{t}}{1-\beta_2^{t}}, \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}}+\varepsilon}\;\;(\rho_t > 4)`,
+      'The third refinement closes a quieter Adam wart. In the first handful of steps Adam has barely any squared-gradient history, so its $\\sqrt{\\hat{\\mathbf{s}}}$ scaling is pure noise — the practitioner’s fix was a hand-tuned warmup that crept the rate up by hand. RAdam computes how trustworthy that variance actually is (a number $\\rho_t$) and, until it can be trusted, just skips the scaling and takes a plain momentum step. A rectification factor $r_t$ — near zero at first, easing toward one — then lets the adaptive part in. Warmup, but derived rather than guessed — nothing to tune.',
+    formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`,
     fix: 'an automatic warmup — no schedule to hand-tune',
     brk: 'only smooths the opening steps; past warmup it just is Adam'
   },
@@ -178,7 +178,7 @@ export const optTree: OptChapter[] = [
     by: 'Chen et al. (Google) — found by program search, not designed',
     idea:
       'Adam scaled the step by gradient history. Lion throws that out and takes a different shape — and it wasn’t invented by a person: a program searched the space of optimizers and this fell out. The name is a fitting backronym — EvoLved Sign Momentum. Keep one momentum buffer, blend it with the fresh gradient, and step by the $\\operatorname{sign}$ of the result — so every step is the same size $\\gamma$ on each axis, no matter how steep or flat. That makes it light (one buffer, no squared-gradient term) and competitive with Adam on big vision and language models. The catch is the very thing that makes it clean: a step that never shrinks can’t settle by itself.',
-    formula: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}`,
+    formula: String.raw`\begin{aligned}&\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L} \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}) \\[2pt] &\mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}\end{aligned}`,
     fix: 'fixed-size steps from one tiny buffer — light and fast',
     brk: 'the step never shrinks, so it orbits the minimum until $\\gamma$ is decayed by a schedule',
     hd: 'Watch Lion’s red step arrow: with two knobs, $\\operatorname{sign}(\\mathbf c)$ can only point in <em>eight</em> directions — the axes and the four diagonals. That is the whole geometry of a sign step: it moves $\\gamma$ along every axis at once, so in $d$ dimensions its true length is $\\gamma\\sqrt{d}$ no matter how faint the gradient, and it can point far from steepest descent. At a billion parameters that $\\sqrt{d}$ is enormous — which is why Lion runs on a much smaller $\\gamma$ than Adam.'
@@ -206,7 +206,7 @@ export const optTree: OptChapter[] = [
     by: 'Liu et al. — Newton, cut down to fit an LLM',
     idea:
       'Newton’s curvature is unbeatable and unaffordable; Sophia keeps the affordable part. Drop the full Hessian for just its DIAGONAL — one curvature number $\\mathbf h$ per parameter, no matrix to invert — and precondition the momentum by it. Then the safety move: CLIP every coordinate’s step to $\\pm\\rho$ (Sophia’s own $\\rho$, a clip radius — no relation to RMSProp’s decay; the alphabet is small and the field is greedy). Where the diagonal estimate is tiny or noisy (and $\\mathbf m/\\mathbf h$ would blow up) the clip bounds the move; where it’s solid, the step stays curvature-scaled. Its paper reports GPT-2-scale pretraining in roughly half the steps Adam needs — a headline later independent benchmarks have contested — but either way, it is second-order thinking made cheap enough to try.',
-    formula: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)`,
+    formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{h} \leftarrow \beta_2 \mathbf{h} + (1-\beta_2)\,\operatorname{diag}(\mathbf{H}) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{clip}\!\left(\frac{\mathbf{m}}{\max(\mathbf{h},\varepsilon)},\,\rho\right)\end{aligned}`,
     fix: 'diagonal curvature + a clip — second-order on a budget',
     brk: 'only the diagonal: blind to the off-axis stretch Newton corrects'
   },
@@ -219,7 +219,7 @@ export const optTree: OptChapter[] = [
     by: 'Mishchenko & Defazio — the learning rate, removed',
     idea:
       'Every method so far still made you pick $\\gamma$. This branch deletes that last knob. The insight: the ideal step size is set by how far the start is from the solution — a distance $d$. You don’t know $d$, so Prodigy estimates it live, ramping a tiny seed upward from how the gradients line up with how far you’ve already travelled ($\\langle g,\\, x_0 - x\\rangle$ — in the formula, $r$ and $\\mathbf{w}$ are two running tallies of exactly that alignment), and scales an Adam step by it. Set nothing and watch the marker creep, then accelerate as $d$ finds its level — the learning rate, discovered rather than tuned. Prodigy sharpens the same lab’s earlier D-Adaptation, and the parameter-free idea is taken seriously: a sibling schedule-free method from these authors won the self-tuning track of MLCommons’ 2024 AlgoPerf benchmark.',
-    formula: String.raw`d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right), \qquad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}`,
+    formula: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2 \\[2pt] &d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}\end{aligned}`,
     fix: 'no learning rate to choose — it finds its own',
     brk: 'the estimate only climbs, so a bad early ramp can overshoot'
   }

src/optim/optimizers/adadelta.ts

@@ -42,7 +42,7 @@ export const adadelta: CoreOptimizer<AdadeltaState> = {
   id: 'adadelta',
   name: 'AdaDelta',
   description: 'RMSProp with no learning rate — the units fix themselves',
-  updateRuleLatex: String.raw`\mathbf{s} \leftarrow \rho \mathbf{s} + (1{-}\rho)(\nabla \mathcal{L})^2, \;\; \Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u} + \varepsilon}}{\sqrt{\mathbf{s} + \varepsilon}}\,\nabla \mathcal{L}, \;\; \mathbf{u} \leftarrow \rho \mathbf{u} + (1{-}\rho)\Delta\boldsymbol{\theta}^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{s} \leftarrow \rho\,\mathbf{s} + (1-\rho)(\nabla \mathcal{L})^2 \\[2pt] &\Delta\boldsymbol{\theta} = -\frac{\sqrt{\mathbf{u}+\varepsilon}}{\sqrt{\mathbf{s}+\varepsilon}}\,\nabla \mathcal{L} \\[2pt] &\mathbf{u} \leftarrow \rho\,\mathbf{u} + (1-\rho)\,\Delta\boldsymbol{\theta}^2, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} + \gamma\,\Delta\boldsymbol{\theta}\end{aligned}`,
   hyperparams: [ADADELTA_RHO],
   fixedLearningRate: 1.0,
   init: (d) => ({ sG: zeros(d), sX: zeros(d), t: 0 }),

src/optim/optimizers/adam.ts

@@ -23,7 +23,7 @@ export const adam: CoreOptimizer<AdamState> = {
   id: 'adam',
   name: 'Adam',
   description: 'Momentum + per-parameter scaling, bias-corrected',
-  updateRuleLatex: String.raw`\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1{-}\beta_1) \nabla \mathcal{L}, \;\; \mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1{-}\beta_2)(\nabla \mathcal{L})^2, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`,
   hyperparams: [BETA1_SPEC, BETA2_SPEC],
   fixedLearningRate: 0.1,
   init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }),

src/optim/optimizers/adamw.ts

@@ -37,7 +37,7 @@ export const adamw: CoreOptimizer<AdamwState> = {
   id: 'adamw',
   name: 'AdamW',
   description: 'Adam with decoupled weight decay — the real default',
-  updateRuleLatex: String.raw`\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\left(\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon} + \lambda\,\boldsymbol{\theta}\right)\end{aligned}`,
   hyperparams: [BETA1_SPEC, BETA2_SPEC, ADAMW_WD],
   fixedLearningRate: 0.1,
   init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }),

src/optim/optimizers/lion.ts

@@ -47,7 +47,7 @@ export const lion: CoreOptimizer<LionState> = {
   id: 'lion',
   name: 'Lion',
   description: 'Sign of momentum: fixed-size steps, very light',
-  updateRuleLatex: String.raw`\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1{-}\beta_1)\nabla\mathcal{L}, \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\,\operatorname{sign}(\mathbf{c}), \;\; \mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1{-}\beta_2)\nabla\mathcal{L}`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{c} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\nabla\mathcal{L} \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \operatorname{sign}(\mathbf{c}) \\[2pt] &\mathbf{m} \leftarrow \beta_2 \mathbf{m} + (1-\beta_2)\nabla\mathcal{L}\end{aligned}`,
   hyperparams: [LION_BETA1, LION_BETA2],
   fixedLearningRate: 0.05,
   init: (d) => ({ m: zeros(d), t: 0 }),

src/optim/optimizers/nadam.ts

@@ -24,7 +24,7 @@ export const nadam: CoreOptimizer<NadamState> = {
   id: 'nadam',
   name: 'Nadam',
   description: 'Adam with Nesterov look-ahead on the momentum',
-  updateRuleLatex: String.raw`\bar{\mathbf{m}} = \beta_1 \hat{\mathbf{m}} + \frac{(1{-}\beta_1)\nabla \mathcal{L}}{1-\beta_1^t}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\bar{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`,
+  updateRuleLatex: String.raw`\bar{\mathbf{m}} = \beta_1 \hat{\mathbf{m}} + \frac{(1-\beta_1)\nabla \mathcal{L}}{1-\beta_1^t}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, \frac{\bar{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}`,
   hyperparams: [BETA1_SPEC, BETA2_SPEC],
   fixedLearningRate: 0.1,
   init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }),

src/optim/optimizers/prodigy.ts

@@ -64,7 +64,7 @@ export const prodigy: CoreOptimizer<ProdigyState> = {
   id: 'prodigy',
   name: 'Prodigy',
   description: 'Parameter-free: estimates its own learning rate',
-  updateRuleLatex: String.raw`d_{t+1} = \max\!\left(d_t,\, \frac{r_{t+1}}{\lVert \mathbf{s}_{t+1}\rVert_1}\right), \;\; \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d_t\,\frac{\mathbf{m}}{\sqrt{\mathbf{v}} + d_t\varepsilon}`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\, d\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)\, d^2 (\nabla \mathcal{L})^2 \\[2pt] &d \leftarrow \max\!\left(d,\, \frac{r}{\lVert \mathbf{w}\rVert_1}\right) \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, d\,\frac{\mathbf{m}}{\sqrt{\mathbf{s}} + d\,\varepsilon}\end{aligned}`,
   hyperparams: [PRODIGY_BETA1, PRODIGY_BETA2],
   fixedLearningRate: 1.0,
   init: (d) => ({ m: zeros(d), v: zeros(d), t: 0, dEst: NaN, rNum: 0, sDen: zeros(d), x0: null }),

src/optim/optimizers/radam.ts

@@ -26,7 +26,7 @@ export const radam: CoreOptimizer<RadamState> = {
   id: 'radam',
   name: 'RAdam',
   description: 'Adam with a built-in, automatic warmup',
-  updateRuleLatex: String.raw`\rho_t = \rho_\infty - \frac{2t\,\beta_2^{t}}{1-\beta_2^{t}}, \quad \boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\;\;(\rho_t > 4),\;\; \text{else}\; -\gamma\,\hat{\mathbf{m}}`,
+  updateRuleLatex: String.raw`\begin{aligned}&\mathbf{m} \leftarrow \beta_1 \mathbf{m} + (1-\beta_1)\,\nabla \mathcal{L} \\[2pt] &\mathbf{s} \leftarrow \beta_2 \mathbf{s} + (1-\beta_2)(\nabla \mathcal{L})^2 \\[2pt] &\boldsymbol{\theta} \leftarrow \boldsymbol{\theta} - \gamma\, r_t\,\frac{\hat{\mathbf{m}}}{\sqrt{\hat{\mathbf{s}}} + \varepsilon}\end{aligned}`,
   hyperparams: [BETA1_SPEC, BETA2_SPEC],
   fixedLearningRate: 0.1,
   init: (d) => ({ m: zeros(d), v: zeros(d), t: 0 }),