From 275ff99851e01f4f76c4b07f0e059ce8b7bc3d75 Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:55:37 +0800 Subject: [PATCH] docs: visualize detector regression --- README.md | 14 ++++ app/layout.tsx | 2 +- package.json | 4 +- public/detector-regression.svg | 82 +++++++++++++++++++ scripts/render-readme-evidence.ts | 130 ++++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 public/detector-regression.svg create mode 100644 scripts/render-readme-evidence.ts diff --git a/README.md b/README.md index 973f560..0eb11f5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # DataTrust Gate +![DataTrust Gate release-audit visual](public/og.png) + **An evidence-led release auditor for bounded AI dataset candidates.** I built DataTrust Gate to inspect a local CSV or flat JSON file and return a @@ -65,6 +67,14 @@ See [docs/PRIVACY.md](docs/PRIVACY.md) for the lifecycle, tested egress paths, a ## Fixed detector regression suite +![DataTrust Gate fixed detector regression coverage](public/detector-regression.svg) + +The figure is generated from the maintained synthetic fixture suite. A filled +bar means the detector matched a separately declared expected finding; it does +not represent measured precision, recall, robustness, or coverage of real +datasets. The clean control and unexpected/missed counts are shown separately +so a matched finding cannot hide an extra or absent result. + The repository includes fixed synthetic detector-regression fixtures with a clean control, isolated single-fault cases, and a compound-fault case. Expected findings are declared separately from scanner inputs. The suite checks whether maintained examples still trigger the intended rules; it is not a measured accuracy benchmark. Reproduce it: @@ -74,6 +84,10 @@ npm run regression npm run regression -- --json ``` +Regenerate the SVG with `npm run evidence:render`. `npm run check` includes +`npm run evidence:check` and fails when the committed figure no longer matches +the fixed suite. + The implementation and expected findings are in [lib/regression-suite.ts](lib/regression-suite.ts); exact assertions are in [tests/audit.test.ts](tests/audit.test.ts). Current `dtg-fixed-regression-v1` (`1.0.0`) result: diff --git a/app/layout.tsx b/app/layout.tsx index 7ff5cfe..6fcd46f 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -29,7 +29,7 @@ export async function generateMetadata(): Promise { title, description, type: "website", - images: [{ url: imageUrl, width: 1664, height: 928, alt: "DataTrust Gate — Evidence before release" }], + images: [{ url: imageUrl, width: 1659, height: 948, alt: "DataTrust Gate — Evidence before release" }], }, twitter: { card: "summary_large_image", diff --git a/package.json b/package.json index 8109901..bedea53 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "test:unit": "tsx --test tests/audit.test.ts tests/parser.test.ts", "test:integration": "node --test tests/rendered-html.test.mjs", "test": "npm run test:unit && npm run build && npm run test:integration", - "check": "npm run lint && npm run typecheck && npm run test", + "evidence:render": "tsx scripts/render-readme-evidence.ts", + "evidence:check": "tsx scripts/render-readme-evidence.ts --check", + "check": "npm run lint && npm run typecheck && npm run evidence:check && npm run test", "regression": "tsx scripts/print-regression.ts", "lint": "eslint . --ignore-pattern dist --ignore-pattern .next", "typecheck": "tsc --noEmit" diff --git a/public/detector-regression.svg b/public/detector-regression.svg new file mode 100644 index 0000000..eef242c --- /dev/null +++ b/public/detector-regression.svg @@ -0,0 +1,82 @@ + + DataTrust Gate fixed detector regression coverage + Fifteen expected findings across eleven detector categories are matched in ten maintained synthetic fixture cases. There are zero unexpected findings, zero missed expected findings, and zero of ten clean-control rows flagged. + + + + + + + + MAINTAINED SYNTHETIC FIXTURES + Fixed detector regression coverage + Filled bar = matched expected finding · scale ends at two findings · this is not an accuracy estimate + + + Email pattern + + + 2/2 + + Malaysian mobile + + + 1/1 + + NRIC-like pattern + + + 1/1 + + IPv4 pattern + + + 2/2 + + Exact duplicate + + + 1/1 + + Near duplicate + + + 1/1 + + Split leakage + + + 1/1 + + Conflicting label + + + 1/1 + + Class imbalance + + + 1/1 + + Missing provenance + + + 2/2 + + Missing licence + + + 2/2 + + + + Expected matched15/15 + Unexpected0 + Missed0 + Clean rows flagged0/10 + + input 8540c2419466 · gold 69eb48c4e568 + dtg-fixed-regression-v1 v1.0.0 + diff --git a/scripts/render-readme-evidence.ts b/scripts/render-readme-evidence.ts new file mode 100644 index 0000000..e4c74f4 --- /dev/null +++ b/scripts/render-readme-evidence.ts @@ -0,0 +1,130 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { runFixedRegressionSuite } from "../lib/regression-suite"; + +const outputUrl = new URL( + "../public/detector-regression.svg", + import.meta.url, +); + +const categoryLabel: Record = { + "pii-email": "Email pattern", + "pii-phone": "Malaysian mobile", + "pii-nric": "NRIC-like pattern", + "pii-ip": "IPv4 pattern", + "exact-duplicate": "Exact duplicate", + "near-duplicate": "Near duplicate", + "split-leakage": "Split leakage", + "conflicting-label": "Conflicting label", + "class-imbalance": "Class imbalance", + "missing-provenance": "Missing provenance", + "missing-license": "Missing licence", +}; + +function escapeXml(value: string) { + return value.replace( + /[&<>"']/g, + (character) => + ({ + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", + })[character]!, + ); +} + +async function renderSvg() { + const report = await runFixedRegressionSuite(); + if ( + report.expectedFindings !== 15 || + report.matchedExpectedFindings !== 15 || + report.unexpectedFindings !== 0 || + report.missedExpectedFindings !== 0 || + report.cleanRows !== 10 || + report.cleanRowsFlagged !== 0 + ) { + throw new Error( + "The fixed detector regression result changed; review the README claim before rendering.", + ); + } + + const groups = [report.byCategory.slice(0, 6), report.byCategory.slice(6)]; + const panels = groups.map((entries, panelIndex) => { + const originX = panelIndex === 0 ? 80 : 790; + const rows = entries.map((entry, rowIndex) => { + if (entry.unexpected !== 0 || entry.missed !== 0) { + throw new Error(`${entry.category} no longer matches its fixed fixture.`); + } + const y = 204 + rowIndex * 72; + const maxWidth = 250; + const expectedWidth = (entry.expected / 2) * maxWidth; + const matchedWidth = (entry.matched / 2) * maxWidth; + const label = categoryLabel[entry.category] ?? entry.category; + return ` + ${escapeXml(label)} + + + ${entry.matched}/${entry.expected} + `; + }); + return `${rows.join("")}`; + }); + + const inputHash = report.inputArtifactHash.slice(0, 12); + const goldHash = report.goldArtifactHash.slice(0, 12); + + return ` + DataTrust Gate fixed detector regression coverage + Fifteen expected findings across eleven detector categories are matched in ten maintained synthetic fixture cases. There are zero unexpected findings, zero missed expected findings, and zero of ten clean-control rows flagged. + + + + + + + + MAINTAINED SYNTHETIC FIXTURES + Fixed detector regression coverage + Filled bar = matched expected finding · scale ends at two findings · this is not an accuracy estimate + + ${panels.join("")} + + + Expected matched${report.matchedExpectedFindings}/${report.expectedFindings} + Unexpected${report.unexpectedFindings} + Missed${report.missedExpectedFindings} + Clean rows flagged${report.cleanRowsFlagged}/${report.cleanRows} + + input ${inputHash} · gold ${goldHash} + ${escapeXml(report.id)} v${escapeXml(report.suiteVersion)} + +`; +} + +const expected = await renderSvg(); +const check = process.argv.includes("--check"); +const normalizeLineEndings = (value: string) => value.replace(/\r\n/g, "\n"); + +if (check) { + let current = ""; + try { + current = readFileSync(outputUrl, "utf8"); + } catch { + // A missing file is reported by the stale-evidence message below. + } + if (normalizeLineEndings(current) !== expected) { + process.stderr.write( + `README evidence is stale. Run npm run evidence:render (${fileURLToPath(outputUrl)}).\n`, + ); + process.exitCode = 1; + } else { + process.stdout.write("README evidence matches the fixed regression suite.\n"); + } +} else { + writeFileSync(outputUrl, expected, "utf8"); + process.stdout.write(`Wrote ${fileURLToPath(outputUrl)}\n`); +}