Cap LaTeX math conversion depth so nested frac/scripts cannot overflow the stack

CHANGELOG.md

@@ -8,6 +8,10 @@ loosely while pre-1.0 (breaking changes can land on minor bumps).
 ## [Unreleased]
 
 ### Fixed
+- **LaTeX math recursion depth.** `latex_math_to_plain` now stops converting
+  after 64 nested `\frac` / `^` / `_` / `\sqrt` / `\text` groups and emits
+  the remaining raw fragment, matching the JSON parser's nesting cap so
+  deeply nested model or user math cannot overflow the TUI/API thread stack.
 - **Secret key/token writes do not follow a planted dest symlink.**
   `write_key_file` and admin-token generate opened
   `~/.arbiter/{openrouter_api_key,search_api_key,admin_token}` with

include/latex_math.h

@@ -10,6 +10,9 @@ namespace arbiter {
 // Unicode approximation for the TUI. Handles common LLM output: fractions,
 // superscripts/subscripts, \times/\approx/\text{}, and Greek letters.
 // Unrecognized commands are stripped to their name or arguments when safe.
+// Nesting deeper than 64 (\frac / ^ / _ / \sqrt / \text) stops converting
+// and emits the remaining raw fragment so a hostile or accidental nest
+// cannot overflow the renderer thread stack.
 [[nodiscard]] std::string latex_math_to_plain(std::string_view latex);
 
 } // namespace arbiter

src/latex_math.cpp

@@ -243,10 +243,17 @@ bool is_decoration_cmd(std::string_view name) {
            name == "Bigr";
 }
 
-std::string convert(std::string_view latex);
+// Hard cap on recursive convert() depth.  Nested \frac / ^ / _ / \sqrt /
+// \text in model or user markdown would otherwise blow the TUI/API thread
+// stack — each nesting level recurses through convert().  64 is deeper
+// than any legitimate equation and far below the ~10k-frame limit of an
+// 8 MiB pthread stack (same rationale as json.cpp::kMaxDepth).
+static constexpr int kMaxConvertDepth = 64;
 
-std::string convert_script(std::string_view raw, bool super) {
-    std::string inner = convert(raw);
+std::string convert(std::string_view latex, int depth);
+
+std::string convert_script(std::string_view raw, bool super, int depth) {
+    std::string inner = convert(raw, depth);
     if (std::string mapped = try_script(inner, super); !mapped.empty()) {
         return mapped;
     }
@@ -260,7 +267,11 @@ std::string convert_script(std::string_view raw, bool super) {
     return out;
 }
 
-std::string convert(std::string_view latex) {
+std::string convert(std::string_view latex, int depth) {
+    if (depth > kMaxConvertDepth) {
+        // Fail closed: emit the raw fragment rather than recurse.
+        return std::string(latex);
+    }
     std::string out;
     out.reserve(latex.size());
     size_t i = 0;
@@ -275,14 +286,14 @@ std::string convert(std::string_view latex) {
 
         if (c == '^') {
             auto [arg, end] = read_script_arg(latex, i + 1);
-            out += convert_script(arg, true);
+            out += convert_script(arg, true, depth + 1);
             i = end;
             continue;
         }
 
         if (c == '_') {
             auto [arg, end] = read_script_arg(latex, i + 1);
-            out += convert_script(arg, false);
+            out += convert_script(arg, false, depth + 1);
             i = end;
             continue;
         }
@@ -368,8 +379,8 @@ std::string convert(std::string_view latex) {
             if (name == "frac" || name == "dfrac" || name == "tfrac") {
                 auto [num, after_num] = read_brace_group(latex, skip_ws(latex, j));
                 auto [den, after_den] = read_brace_group(latex, skip_ws(latex, after_num));
-                const std::string n = convert(num);
-                const std::string d = convert(den);
+                const std::string n = convert(num, depth + 1);
+                const std::string d = convert(den, depth + 1);
                 const bool simple_n = n.size() == 1;
                 const bool simple_d = d.size() == 1;
                 if (!simple_n) out += '(';
@@ -390,7 +401,8 @@ std::string convert(std::string_view latex) {
                 if (at < latex.size() && latex[at] == '[') {
                     size_t close = latex.find(']', at + 1);
                     if (close != std::string_view::npos) {
-                        std::string idx = convert(latex.substr(at + 1, close - at - 1));
+                        std::string idx = convert(latex.substr(at + 1, close - at - 1),
+                                                  depth + 1);
                         if (std::string mapped = try_script(idx, true); !mapped.empty()) {
                             out += mapped;
                         } else {
@@ -401,7 +413,7 @@ std::string convert(std::string_view latex) {
                 }
                 auto [body, end] = read_brace_group(latex, skip_ws(latex, at));
                 out += "\u221a";  // √
-                const std::string inner = convert(body);
+                const std::string inner = convert(body, depth + 1);
                 if (inner.size() == 1) {
                     out += inner;
                 } else {
@@ -428,7 +440,7 @@ std::string convert(std::string_view latex) {
                 }
                 out.append(body.data(), lead);
                 if (trail > lead) {
-                    out += convert(body.substr(lead, trail - lead));
+                    out += convert(body.substr(lead, trail - lead), depth + 1);
                 }
                 out.append(body.data() + trail, body.size() - trail);
                 i = (end == skip_ws(latex, j)) ? j : end;
@@ -440,7 +452,7 @@ std::string convert(std::string_view latex) {
                 name == "dot" || name == "ddot" || name == "widehat" ||
                 name == "widetilde" || name == "mathbf" || name == "boldsymbol") {
                 auto [body, end] = read_brace_group(latex, skip_ws(latex, j));
-                out += convert(body);
+                out += convert(body, depth + 1);
                 if (name == "hat" || name == "widehat") out += "\u0302";
                 else if (name == "tilde" || name == "widetilde") out += "\u0303";
                 else if (name == "bar" || name == "overline") out += "\u0305";
@@ -482,7 +494,7 @@ std::string convert(std::string_view latex) {
 } // namespace
 
 std::string latex_math_to_plain(std::string_view latex) {
-    return convert(latex);
+    return convert(latex, 0);
 }
 
 } // namespace arbiter

tests/test_markdown_diff.cpp

@@ -648,6 +648,48 @@ TEST_CASE("latex_math_to_plain converts screenshot-style equations") {
           "N=(8.1\u00d710\u00b2\u2075)/(0.004)");
 }
 
+TEST_CASE("latex_math_to_plain converts modest nested fractions") {
+    CHECK(latex_math_to_plain(R"(\frac{\frac{a}{b}}{c})") == "(a/b)/c");
+    CHECK(latex_math_to_plain(R"(a^{b^{2}})") == "a^(b\u00b2)");
+}
+
+TEST_CASE("latex_math_to_plain caps deep nesting instead of overflowing") {
+    // 400 nested \frac groups is well past the 64-level cap and would
+    // have recursed far enough to overflow an 8 MiB thread stack.
+    std::string nested;
+    nested.reserve(400 * 16);
+    for (int i = 0; i < 400; ++i) nested += "\\frac{";
+    nested += "x";
+    for (int i = 0; i < 400; ++i) nested += "}{y}";
+
+    const std::string out = latex_math_to_plain(nested);
+    CHECK_FALSE(out.empty());
+    // Outer converted layers still emit the slash form.
+    CHECK(out.find('/') != std::string::npos);
+    // Past the cap the remaining nest is left raw rather than converted.
+    CHECK(out.find("\\frac") != std::string::npos);
+
+    std::string scripts;
+    for (int i = 0; i < 400; ++i) scripts += "a^{";
+    scripts += "n";
+    scripts += std::string(400, '}');
+    const std::string script_out = latex_math_to_plain(scripts);
+    CHECK_FALSE(script_out.empty());
+}
+
+TEST_CASE("MarkdownRenderer survives deeply nested display math") {
+    std::string body;
+    for (int i = 0; i < 400; ++i) body += "\\frac{";
+    body += "x";
+    for (int i = 0; i < 400; ++i) body += "}{y}";
+
+    MarkdownRenderer md;
+    std::string out = md.feed("\\[\n" + body + "\n\\]\n");
+    out += md.flush();
+    CHECK(out.find("\\[") == std::string::npos);
+    CHECK_FALSE(out.empty());
+}
+
 TEST_CASE("MarkdownRenderer renders display and inline math as Unicode") {
     MarkdownRenderer md;
     const char* input =