From d860fe6c4a37039ce3ef7132caf65f861ba2d627 Mon Sep 17 00:00:00 2001
From: jun <jun@lidge.dev>
Date: Thu, 3 Sep 2026 18:21:48 +0900
Subject: [PATCH] test(semver): measure the ReDoS guard as best-of-N, not one
 sample
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The guard timed a single parse against a 50ms budget. On a loaded macOS runner
that sample came back at 53.77ms and failed the v2.41.0 promotion, for a parse
whose subject regression is 522ms — three orders of magnitude away. A first call
also carries regex compilation and JIT warm-up that the parse itself does not.

A gate that fires on runner weather teaches everyone to re-run it, which is
exactly how a real ReDoS regression would get waved through.

Best-of-N is the right statistic here, and that is measured rather than assumed.
Running the semver.org prerelease pattern this module replaced, three runs each:
17.4-17.6ms at 20 repetitions, then 491-545ms at 30, 39 and 45. The blowup is on
every run with under 10% spread, because superlinear backtracking is a property
of the pattern; the 4ms of jitter that broke the single-sample form is not. If
the exponential path returns, no run is fast.

The measurements are in the test file, so the next person to touch the budget
can see what it is actually separating.

Verification: bun test tests/strict-semver.test.ts 7 pass 0 fail;
bun run typecheck exit 0.
---
 tests/strict-semver.test.ts | 56 +++++++++++++++++++++++++++----------
 1 file changed, 42 insertions(+), 14 deletions(-)

diff --git a/tests/strict-semver.test.ts b/tests/strict-semver.test.ts
index 934dabc048..e47d511f2b 100644
--- a/tests/strict-semver.test.ts
+++ b/tests/strict-semver.test.ts
@@ -10,35 +10,63 @@ import { parseStrictSemver } from "../src/lib/strict-semver";
  *
  * The length ceiling did not help. It only chose where on the curve the input landed.
  */
+/**
+ * Every timing assertion here measures the BEST of several runs, not a single one.
+ *
+ * A first call carries one-time cost the parse itself does not: regex compilation, JIT
+ * warm-up, and whatever the shared CI runner was doing during that millisecond. On a
+ * loaded macOS runner that noise reached 53.77ms against a 50ms budget and failed a
+ * suite whose subject is three orders of magnitude away from the regression it guards
+ * (522ms). A gate that fires on runner weather rather than on the defect teaches
+ * everyone to re-run it, which is how a real ReDoS regression would get waved through.
+ *
+ * The minimum is the right statistic for this question. Superlinear backtracking is a
+ * property of the pattern, so it reproduces on EVERY iteration; scheduler noise does
+ * not. If the exponential path returns, no run is fast.
+ *
+ * That claim was measured rather than assumed. Running the semver.org prerelease
+ * pattern this module replaced against the same inputs, three runs each:
+ *
+ *     reps=20  len=68    17.6ms  17.4ms  17.4ms
+ *     reps=30  len=98   545.4ms 521.2ms 500.0ms
+ *     reps=39  len=125  492.3ms 493.5ms 491.3ms
+ *     reps=45  len=128  495.3ms 507.9ms 527.8ms
+ *
+ * The blowup is on every run, not the first, so a best-of-N below 50ms still fails
+ * loudly if it comes back. The spread across runs is under 10%, which is what a
+ * deterministic cost looks like next to the 4ms of scheduler jitter that broke the
+ * single-sample form.
+ */
+function fastestParseMs(input: string, runs = 5): number {
+  let best = Infinity;
+  for (let i = 0; i < runs; i++) {
+    const started = performance.now();
+    parseStrictSemver(input);
+    const elapsed = performance.now() - started;
+    if (elapsed < best) best = elapsed;
+  }
+  return best;
+}
+
 describe("parseStrictSemver ReDoS resistance", () => {
   test("the flagged attack shape stays linear at the length ceiling", () => {
     // "0.0.0-0." followed by repetitions of "--." is the input CodeQL named.
     const attack = ("0.0.0-0." + "--.".repeat(45)).slice(0, 128);
     expect(attack.length).toBe(128);
 
-    const started = performance.now();
     expect(parseStrictSemver(attack)).toBeNull();
-    const elapsed = performance.now() - started;
 
     // The vulnerable pattern took ~522ms for this input. Anything in that region means the
     // superlinear path is back; a linear parse lands three orders of magnitude below it.
-    expect(elapsed).toBeLessThan(50);
+    expect(fastestParseMs(attack)).toBeLessThan(50);
   });
 
   test("cost does not grow with the number of repetitions", () => {
-    const measure = (reps: number): number => {
-      const input = ("0.0.0-0." + "--.".repeat(reps)).slice(0, 128);
-      const started = performance.now();
-      parseStrictSemver(input);
-      return performance.now() - started;
-    };
+    const inputFor = (reps: number): string => ("0.0.0-0." + "--.".repeat(reps)).slice(0, 128);
 
     // Under the old pattern, going from 20 to 39 repetitions moved 16ms to 524ms.
-    measure(20);
-    const short = measure(20);
-    const long = measure(39);
-    expect(short).toBeLessThan(50);
-    expect(long).toBeLessThan(50);
+    expect(fastestParseMs(inputFor(20))).toBeLessThan(50);
+    expect(fastestParseMs(inputFor(39))).toBeLessThan(50);
   });
 
   test("the length guard still rejects before any matching work", () => {