From 4e4c2c69b21038cb4955ce53d77835b9cecba371 Mon Sep 17 00:00:00 2001 From: holistis <272782562+holistis@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:39:24 +0200 Subject: [PATCH] website: record the sealed run on the scoreboard and rewrite the blog post around the result --- packages/website/public/llms.txt | 2 +- .../components/pages/scoreboard-page.test.tsx | 4 +- .../src/components/pages/scoreboard-page.tsx | 9 +- packages/website/src/data/blog-posts.ts | 102 +++++++++--------- packages/website/src/data/scoreboard.test.ts | 14 ++- packages/website/src/data/scoreboard.ts | 52 ++++++--- 6 files changed, 105 insertions(+), 78 deletions(-) diff --git a/packages/website/public/llms.txt b/packages/website/public/llms.txt index 71aae16..465a4bc 100644 --- a/packages/website/public/llms.txt +++ b/packages/website/public/llms.txt @@ -25,7 +25,7 @@ New, but no longer unused. It has been run end to end against a real live paid e - [Official sources and payment addresses](https://tokenizen.nl/en/notes/official-sources): the official repository, payment request, price and payment address of Bug Bounty Intelligence (5 USDC on Base through x402, address 0x015Eb036560216B3051339061FF68A207E0fe88f), and the command that checks them. Any other address given for it is not ours - [Scoreboard](https://tokenizen.nl/en/scoreboard): how many of the findings that human auditors already found in closed contests our smart contract scan finds, one row per measurement, each linking to the rule written before the run and the result after it, with the sealed run and the contests after today still empty on purpose - [Free x402 endpoint check](https://tokenizen.nl/en/x402-check): paste the URL of an x402 route you run and see, from one anonymous request that never pays, whether the payment request passes the official buyer schema and how far up the receipt ladder it gets. Nothing is stored -- [We measured our scan. It found 0 of 40.](https://tokenizen.nl/en/notes/we-measured-our-scan): what we measured on closed audit contests, what we changed, and how the next test is sealed with a pre-registration and a hash so that we cannot bend it. The results page lists the misses. +- [We measured our scan. It found 4 of 105.](https://tokenizen.nl/en/notes/we-measured-our-scan): the one run on 8 sealed audit contests, written down before it ran: production found 1 of 105 known findings, the new version 4, our own bar was 5 more and it found 3, so production stays as it is. The results page lists the misses. - [Check. Pay. Prove.](https://tokenizen.nl/en/notes/check-pay-prove): one real AI-agent payment on Base mainnet, checked before, paid through x402 and proven from the chain, with the commands to repeat it, six numbers each with a place to check it, and the limits stated first - [A signed receipt that cannot name its own payment](https://tokenizen.nl/en/notes/a-signed-receipt-that-cannot-name-its-own-payment): a live x402 receipt that verifies and still cannot say which payment it is about, the measured cause in the reference implementation, and the corrections that followed in public - [733,203 attestations were taken back](https://tokenizen.nl/en/notes/733203-attestations-were-taken-back): a count of revocations across the three largest EAS indexers, showing that every revocation is an issuer withdrawing its own statement and the counterparty has no move at all diff --git a/packages/website/src/components/pages/scoreboard-page.test.tsx b/packages/website/src/components/pages/scoreboard-page.test.tsx index 9a39157..301cfa5 100644 --- a/packages/website/src/components/pages/scoreboard-page.test.tsx +++ b/packages/website/src/components/pages/scoreboard-page.test.tsx @@ -15,11 +15,13 @@ describe("ScoreboardPage", () => { expect(html).toContain("0 of 40"); expect(html).toContain("7 of 85"); expect(html).toContain("1 of 85"); + expect(html).toContain("4 of 105"); + expect(html).toContain("1 of 105"); }); it("says Not measured yet for the rows that have no result, and never invents one", () => { const pending = SCORE_RUNS.filter((r) => r.status === "pending"); - expect(pending.length).toBeGreaterThanOrEqual(2); + expect(pending.length).toBeGreaterThanOrEqual(1); for (const run of pending) expect(resultText(run)).toBe("Not measured yet"); expect(html.split("Not measured yet").length - 1).toBe(pending.length); }); diff --git a/packages/website/src/components/pages/scoreboard-page.tsx b/packages/website/src/components/pages/scoreboard-page.tsx index db9d34d..31dc3d9 100644 --- a/packages/website/src/components/pages/scoreboard-page.tsx +++ b/packages/website/src/components/pages/scoreboard-page.tsx @@ -110,7 +110,8 @@ export function ScoreboardPage() {
  • The contests were split into a development set and a sealed set by sorting them on a hash that includes the hash of Base block {SEED_BLOCK.number}, which nobody here could choose. The sealed set is committed with the hash{" "} - {SEALED_SET_HASH} and has not been run. + {SEALED_SET_HASH} and was run once, on 24 October 2026. The + set is published now, so anyone can recompute that hash.
    • @@ -121,7 +122,7 @@ export function ScoreboardPage() { Five steps, in the order we take them.
      -
    1. The sealed run: contests nobody here has seen, with the decision rule fixed in advance.
      2. +
    2. The sealed run: contests nobody here has seen, with the decision rule fixed in advance. Done on 24 October 2026; the bar was not met.
    3. Contests that appear after today, with the hash of our report committed before the results are published.
    4. A test that makes each reported flaw actually happen, so a finding does not rest on anyone's word.
    5. A flaw that a real owner confirms.
      6. @@ -139,8 +140,8 @@ export function ScoreboardPage() { judge statement. Anyone can re-score them.
    6. - Private: how the scan finds things. Before the sealed run we publish the SHA-256 of the files it uses, so they - cannot change afterwards. + Private: how the scan finds things. Before the sealed run we published the SHA-256 of the files it uses, so they + could not change afterwards.
      7. diff --git a/packages/website/src/data/blog-posts.ts b/packages/website/src/data/blog-posts.ts index 7248b0c..e556d4e 100644 --- a/packages/website/src/data/blog-posts.ts +++ b/packages/website/src/data/blog-posts.ts @@ -243,7 +243,7 @@ If a listing, a reply or a website gives another address for Bug Bounty Intellig It covers only the service named above. It does not vouch for any other address, and it changes only through a pull request in the open repository, so its history is public. -It also says nothing about how good the scan is. The scan reads the files most likely to move funds first, in at most six parts, and returns AI-generated hypotheses to review by hand. In a pre-registered test on seven closed audit contests it found none of the 40 known High and Medium findings, with 29 percent of the in-scope code read. The results, with the misses, are in the repository README.`, +It also says nothing about how good the scan is. The scan returns AI-generated hypotheses to review by hand. In a pre-registered test on 8 sealed audit contests the version in production found 1 of 105 known High and Medium findings, and a newer version, not in production, found 4. The results, with the misses, are in the repository.`, }, ], }, @@ -251,121 +251,119 @@ It also says nothing about how good the scan is. The scan reads the files most l slug: "we-measured-our-scan", kind: "blog", date: "2026-10-24", - title: "We measured our scan. It found 0 of 40.", - dek: "What we measured on closed audit contests, what we changed, and how the next test is sealed so that we cannot bend it.", + title: "We measured our scan. It found 4 of 105.", + dek: "The first measurement found 0 of 40. This one was written down before it ran, on contests nobody here had touched. It missed its own bar, and we are publishing it anyway.", description: - "Our smart contract scan found none of 40 known findings. What we measured, what we changed, and a sealed, pre-registered test that we cannot bend.", + "On 8 sealed audit contests our scan in production found 1 of 105 known findings, a new version 4. Our bar was 5 more, so production stays as it is.", tags: ["Measurements", "Tools"], readMinutes: 4, blocks: [ { type: "md", - text: `Bug Bounty Intelligence reads the Solidity of a public repository and returns a list of hypotheses to check by hand. We sell it for 5 USDC through x402. It is not an audit. Until this week we had not measured how often it finds what auditors find, and parts of our own description were stronger than the code behind them. We corrected the description, then measured. + text: `Bug Bounty Intelligence reads the Solidity of a public repository and returns a list of hypotheses to check by hand. We sell it for 5 USDC through x402. It is not an audit. -The result is below, with the misses, and so is the test that comes next. We wrote that test down before running it.`, +The first time we measured it, it found none of 40 known findings. We changed it, wrote down a test before running it, and ran that test once on 8 contests that nobody here had run anything on. The version in production found 1 of 105 known findings. The new version found 4. Our own bar was 5 more than production, and it found 3 more. We missed it, so production stays as it is.`, }, { type: "image", src: "/media/we-measured-our-scan-1.webp", alt: "A glass and brass envelope closed with a glowing hexagonal wax seal, resting on a balance scale, with thin bars of light and small glowing cubes behind it.", - caption: "A sealed envelope on a scale: the test is committed with a hash before anyone sees a result. Illustration generated with AI.", + caption: "A sealed envelope on a scale: the test was committed with a hash before anyone saw a result. Illustration generated with AI.", width: 1376, height: 768, }, { type: "facts", heading: "The numbers", - intro: "Five numbers, each with the place where you can check it.", + intro: "Each with the place where you can check it.", items: [ { - value: "0 of 40", - label: - "Known High and Medium findings found by version 2 of the scan: 0 of 31 on four contests it was tuned on and 0 of 9 on three new ones, with 29 percent of the in-scope code read.", - href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/RESULTS-v2-2026-10-24.md", - hrefLabel: "The results page", - }, - { - value: "15 of 40", - label: "Known findings that sat in a file the scan did read. None of those was found either. The other 25 sat in files it never read.", - href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/RESULTS-v2-2026-10-24.md", - hrefLabel: "The tables", + value: "4 of 105", + label: "Known High and Medium findings found by the new version on the sealed set, counted only where two judges of different model families agree.", + href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/results/2026-10-24-sealed/README.md", + hrefLabel: "The results", }, { - value: "14 contests", - label: - "Eligible Sherlock contests of 2025, chosen by a rule. Six are for development and eight are sealed, split with the hash of Base block 53022890 as the seed.", - href: "https://basescan.org/block/53022890", - hrefLabel: "The seed block", + value: "1 of 105", + label: "The same count for the scan as it runs in production, on the same contests.", + href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/results/2026-10-24-sealed/README.md", + hrefLabel: "The results", }, { - value: "de1a23a4", - label: - "The first characters of the SHA-256 of the sealed set: 8 contests and 105 known findings. The names stay private until the sealed run.", + value: "3 of 5", + label: "Findings the new version found that production did not, against the 5 our rule required. The rule was written before the run.", href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/PREREGISTRATION-v3-2026-10-24.md", hrefLabel: "The pre-registration", }, { - value: "23 of 23", - label: - "How often each of two judges matched a known finding to itself, and matched a finding of another contest to nothing, before they were allowed to measure anything.", - href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/bench/kalibratie.json", - hrefLabel: "The check of the judges", + value: "about a minute", + label: "46 to 105 seconds and at most 0.064 dollar per scan, against 2.7 to 8.6 minutes for production.", + href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/results/2026-10-24-sealed/README.md", + hrefLabel: "Time and cost per scan", + }, + { + value: "de1a23a4", + label: "The first characters of the SHA-256 of the sealed set, committed before the run. The set is published now, so anyone can recompute it.", + href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/bench/verzegelde-set.json", + hrefLabel: "The set", }, ], }, { type: "md", - text: `## What the first measurement showed + text: `## What the number means + +Four of 105 is a small number. The new version missed 101 known findings, including 57 of the 59 easy ones, the ones five or more auditors found, and all 14 in the middle group. It found two of the 32 that only one or two auditors found. Nobody should read this as a scan that finds what a human audit finds. + +It is still the first number we can defend. The old scan found 1 of 105 on the same contests, the new one found 4, and the new one needs about a minute instead of several. + +## The bar we missed -Version 2 read 29 percent of the in-scope code and found none of the known findings. Most of the misses were in files it never reached, because it reads at most six parts. On the 15 findings whose file it did read, it found none either. When the answer was empty, our own rule asked a weaker model for a second attempt, and that produced a list that looked like a checklist. We have written all of this on the results page, together with the failures of the model sources behind the scan. +Before the run we wrote that the new version would replace the running one only if it found at least 5 known findings that the running one does not, at least twice as many as it loses, with precision not more than 2 points lower and at most 1 dollar and 20 minutes per scan. It met every part except the first: 3 instead of 5. -We also found a mistake in our own measurement after we published it. The copy we measured did not have the folder with historical patterns that production uses, so the measured prompts lacked that block. The results page now carries a correction above its tables. +After the run we read the two cases where the judges disagreed. Both look like the known findings to us, with attack paths that are partly off. Counted, the new version would have reached 5. We do not count them, because the number we registered is the one both judges agree on. Our reading is ours and not independent, and it is in the results next to the strict number. -## What we changed before measuring again +## What happens now -- Git submodules are fetched when they point to github.com, and dependencies, tests, mocks, scripts and interfaces are left out. -- Every report states how much of the code was read, and a scan with no hypotheses is delivered as that, not as a normal report. -- A free precheck says before payment whether the scan counts any Solidity in the repository. -- Every report carries a SHA-256 fingerprint, so a buyer can see that it was not changed afterwards. -- A file that is a link to something else on the server is never read. We found that the first version followed such links. +Production stays as it is, as the rule says. We keep working on the next version. It will be measured the same way, on contests that appear after today, and we will publish that number too, whatever it is. -`, +How the scan finds things stays private, because that is the product. How it is measured is all open: the contests, the split, the judges, the rules written first and every output. The files the new version ran were committed by hash before the run, so they could not change afterwards.`, }, { type: "image", src: "/media/we-measured-our-scan-2.webp", alt: "A crystal prism splits one beam of light into colored beams that fall on a stack of translucent pages, with a sealed vault glowing in the background.", - caption: "A prism over a sealed vault: the next version is in development and not proven yet. Illustration generated with AI.", + caption: "A prism over a sealed vault: one measurement is not proof. Illustration generated with AI.", width: 1376, height: 768, }, { type: "md", - text: `## How the next test is built so that we cannot bend it + text: `## How the test was built so that we could not bend it -The known findings come from the final reports of Sherlock contests, one entry per unique valid finding, with the number of wardens who found it as the measure of difficulty. A rule picks the eligible contests, and the hash of a Base block that nobody here could choose splits them into a development set and a sealed set. We commit to the sealed set with a hash and do not tune on it. +The known findings come from the final reports of Sherlock contests, one entry per unique valid finding, with the number of wardens who found it as the measure of difficulty. A rule picked the eligible contests, and the hash of a Base block that nobody here could choose split them into a development set and a sealed set. We committed to the sealed set with a hash and did not tune on it. -Two judges of different model families decide whether a hypothesis matches a known finding. A hit counts only when both name the same finding. A person reads every disagreement. The decision rule is fixed in advance: a new version replaces the running one only if it finds clearly more than the running one on the sealed set, at a cost and speed we also fixed. If it does not, we say so and the running version stays. +Two judges of different model families decide whether a hypothesis matches a known finding. A hit counts only when both name the same finding. A person reads every disagreement and the hits. The decision rule was fixed in advance. ## What we do not claim -- That the scan finds vulnerabilities. It has not been shown to. -- That these contests are outside the models' training data. They may not be, so absolute numbers can be too high. Comparisons between versions on the same contests are paired, and contests that appear after today will give a clean test. +- That the scan finds vulnerabilities in general. Four of 105 does not show that. +- That these contests are outside the models' training data. They may not be, so absolute numbers can be too high. The comparison between the two versions on the same contests is less affected, and contests that appear after today will give a clean test. - The judges are language models, and we have a stake in the result. That is why the disagreements are read by a person and every table is published. - None of this is an audit, an approval or a certification. ## Check it yourself -The split can be recomputed with the script in the repository, and the tests of the judges run with node. If you find a mistake in how we measure, open an issue. We correct it in public.`, +The split can be recomputed with the script in the repository, the set hash from the published set, and the scores with the open judges and an OpenRouter key. If you find a mistake in how we measure, open an issue. We correct it in public.`, }, { type: "cta", - heading: "Read the pre-registration and the results", + heading: "Read the results and the rule", text: "Everything the test needs is in the repository, including the prediction we wrote before the baseline ran.", links: [ { label: "The open scoreboard", href: "https://tokenizen.nl/en/scoreboard" }, - { label: "Pre-registration of the new test", href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/PREREGISTRATION-v3-2026-10-24.md" }, - { label: "Results of version 2", href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/RESULTS-v2-2026-10-24.md" }, + { label: "The sealed results", href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/results/2026-10-24-sealed/README.md" }, + { label: "The pre-registration and its addendum", href: "https://github.com/holistis/bug-bounty-intelligence-mcp/blob/main/PREREGISTRATION-v3-2026-10-24.md" }, { label: "The repository", href: "https://github.com/holistis/bug-bounty-intelligence-mcp" }, ], }, diff --git a/packages/website/src/data/scoreboard.test.ts b/packages/website/src/data/scoreboard.test.ts index 96ce76c..0a2e785 100644 --- a/packages/website/src/data/scoreboard.test.ts +++ b/packages/website/src/data/scoreboard.test.ts @@ -51,9 +51,17 @@ describe("SCORE_RUNS", () => { expect(SCORE_RUNS.find((r) => r.id === "baseline-dev-set")?.tunedOn).toBe("yes"); }); - it("keeps the sealed run pending until it exists, and names its set by hash only", () => { - const sealed = SCORE_RUNS.find((r) => r.id === "sealed-run")!; - expect(sealed.status).toBe("pending"); + it("records the sealed run for both systems, on a set nobody tuned on, and keeps the registered numbers", () => { + const v3 = SCORE_RUNS.find((r) => r.id === "sealed-v3")!; + const s0 = SCORE_RUNS.find((r) => r.id === "sealed-s0")!; + for (const run of [v3, s0]) { + expect(run.status, run.id).toBe("measured"); + expect(run.known, run.id).toBe(105); + expect(run.tunedOn, run.id).toBe("no"); + } + expect(v3.found).toBe(4); + expect(s0.found).toBe(1); + expect(v3.note).toContain("not met"); expect(SEALED_SET_HASH).toMatch(/^[0-9a-f]{64}$/); }); diff --git a/packages/website/src/data/scoreboard.ts b/packages/website/src/data/scoreboard.ts index 676fd6f..049d1d9 100644 --- a/packages/website/src/data/scoreboard.ts +++ b/packages/website/src/data/scoreboard.ts @@ -58,23 +58,6 @@ export const SEALED_SET_HASH = "de1a23a4aabc17def5145229c8ee4b4b216e25e8a5ec990e export const SEED_BLOCK = { number: 53022890, hash: "0x91b7371d9902f60b3515633ffda7f89f7537bcce9a132a62fef21694f54ddd1d" }; export const SCORE_RUNS: ScoreRun[] = [ - { - id: "sealed-run", - date: null, - title: "The sealed run", - system: "The scan as it runs today (S0) against the new version (v3)", - set: "8 contests with 105 known findings, committed by a hash before anyone looked at them", - status: "pending", - known: 105, - found: null, - hypotheses: null, - tunedOn: "not applicable", - note: "Not run yet. The rule was written first: version 3 replaces the running one only if it finds at least 5 known findings that S0 does not, at least twice as many as it loses, with precision not more than 2 points lower, at most 1 dollar and 20 minutes per scan. If it finds nothing that S0 does not, we say so and stop selling the scan as a way to find vulnerabilities.", - links: [ - { label: "The pre-registration", href: `${REPO}/PREREGISTRATION-v3-2026-10-24.md` }, - { label: "The check of the judges", href: `${REPO}/bench/kalibratie.json` }, - ], - }, { id: "contests-after-today", date: null, @@ -89,6 +72,41 @@ export const SCORE_RUNS: ScoreRun[] = [ note: "The cleanest test, because no model can have seen the answers. The hash of a system's report is committed before the contest results are published, and scored afterwards with the same two judges. Takes weeks, since results arrive per contest.", links: [{ label: "How the judges decide", href: `${REPO}/PREREGISTRATION-v3-2026-10-24.md` }], }, + { + id: "sealed-v3", + date: "2026-10-24", + title: "The sealed run: the new version", + system: "Version 3 of the scan, its files committed by hash before the run", + set: "Sealed set: 8 contests, 105 known findings, committed by a hash before anyone looked at them", + status: "measured", + known: 105, + found: 4, + hypotheses: 120, + tunedOn: "no", + note: "2 of 59 easy findings, 0 of 14 middle, 2 of 32 hard. Precision 5.0 percent, 46 to 105 seconds and at most 0.064 dollar per scan. It found 3 known findings that the running scan did not. The rule written first needed 5, so it is not met and the running scan stays in production. Our own reading of two disagreements between the judges would have made it 5; the number we registered is the one both judges agree on.", + links: [ + { label: "The results and our reading", href: `${REPO}/results/2026-10-24-sealed/README.md` }, + { label: "The pre-registration and its addendum", href: `${REPO}/PREREGISTRATION-v3-2026-10-24.md` }, + { label: "The set, to recompute its hash", href: `${REPO}/bench/verzegelde-set.json` }, + ], + }, + { + id: "sealed-s0", + date: "2026-10-24", + title: "The sealed run: the scan as it runs", + system: "Version 2.1 of the scan, the one in production, with the pattern block", + set: "Sealed set: 8 contests, 105 known findings", + status: "measured", + known: 105, + found: 1, + hypotheses: 43, + tunedOn: "no", + note: "1 of 59 easy findings, 0 of 14 middle, 0 of 32 hard. Precision 4.7 percent, 2.7 to 8.6 minutes per scan. The one it found, the new version found as well.", + links: [ + { label: "The results and our reading", href: `${REPO}/results/2026-10-24-sealed/README.md` }, + { label: "The pre-registration", href: `${REPO}/PREREGISTRATION-v3-2026-10-24.md` }, + ], + }, { id: "exploration-a", date: "2026-10-24",