Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@ the `--check` mode.

## Website

The public site at <https://dataflowbench.brokk.ai> is an Astro
The public site at <https://dataflowbench.slopcop.com> is an Astro
Starlight package in `docs/`; every number it shows is rendered from the
generated `results/results.json`, never hand-authored. Develop locally with:

Expand Down
51 changes: 49 additions & 2 deletions docs/astro.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { defineConfig } from 'astro/config';
import starlight from '@astrojs/starlight';
import { snapshots } from './src/data/snapshots';
import { remarkHeadingIds } from './src/plugins/remark-heading-ids.mjs';
import { rehypeTableScroll } from './src/plugins/rehype-table-scroll.mjs';

// ---------------------------------------------------------------------------
// Snapshot navigation.
Expand Down Expand Up @@ -58,12 +59,21 @@ const previousLabel = previousSnapshot
: '';

// Production deployment target for GitHub Pages.
const site = 'https://dataflowbench.slopcop.com';
const base = '/';
/** The published root, for the absolute URLs social cards need. */
const siteRoot = new URL(base.replace(/\/?$/, '/'), site);
const socialCard = new URL('og-image.png', siteRoot).href;

export default defineConfig({
site: 'https://dataflowbench.brokk.ai',
site,
base,
markdown: {
// `## Heading {#legacy-id}` — explicit anchors for headings whose ids are
// citable and must not follow the auto-slugger. See the plugin's comment.
remarkPlugins: [remarkHeadingIds],
// Each table in a scroll box, so a wide one pans instead of overflowing.
rehypePlugins: [rehypeTableScroll],
},
redirects: {
// Explicit current-snapshot pointer alongside versioned snapshot URLs.
Expand All @@ -75,7 +85,44 @@ export default defineConfig({
favicon: '/favicon.svg',
description:
'Analyzer-neutral benchmark for data-flow analysis, published exclusively from immutable freeze evidence.',
customCss: ['./src/styles/custom.css'],
// The shared foundation first (identical in the UsageBench repository),
// then this site's accent, palette and layout on top of it.
customCss: ['./src/styles/foundation.css', './src/styles/custom.css'],
logo: { src: './src/assets/mark.svg', alt: '' },
head: [
{ tag: 'meta', attrs: { property: 'og:image', content: socialCard } },
{ tag: 'meta', attrs: { property: 'og:image:width', content: '1200' } },
{ tag: 'meta', attrs: { property: 'og:image:height', content: '630' } },
{
tag: 'meta',
attrs: {
property: 'og:image:alt',
content: 'DataFlowBench: how well do static analyzers follow data flow?',
},
},
{ tag: 'meta', attrs: { name: 'twitter:image', content: socialCard } },
{
tag: 'script',
attrs: { type: 'application/ld+json' },
content: JSON.stringify({
'@context': 'https://schema.org',
'@type': 'WebSite',
name: 'DataFlowBench',
url: siteRoot.href,
publisher: { '@type': 'Organization', name: 'SlopCop', url: 'https://slopcop.com' },
}),
},
],
// Archived snapshot pages get their release named in their search
// description, so no two pages share one. See src/routeData.ts.
routeMiddleware: './src/routeData.ts',
expressiveCode: {
styleOverrides: {
borderRadius: '0px',
codeFontFamily: "'JetBrains Mono', ui-monospace, monospace",
uiFontFamily: "'Apfel Grotezk', ui-sans-serif, system-ui, sans-serif",
},
},
components: {
// Starlight's own footer, with the floating "back to top" control
// appended. The footer is the one slot that renders exactly once on
Expand Down
2 changes: 1 addition & 1 deletion docs/public/CNAME
Original file line number Diff line number Diff line change
@@ -1 +1 @@
dataflowbench.brokk.ai
dataflowbench.slopcop.com
13 changes: 7 additions & 6 deletions docs/public/favicon.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/public/og-image.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
15 changes: 15 additions & 0 deletions docs/src/assets/mark.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
114 changes: 57 additions & 57 deletions docs/src/components/AnalyzerLandscape.astro
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
vendorOrder,
} from '../data/snapshots';
import { analyzerLanguageSupport, type AnalyzerLanguageSupport } from '../data/analyzer-language-support';
import { percentAxis, placeLabels } from '../data/scatter-layout';
import ScatterTooltip from './ScatterTooltip.astro';

interface AnalyzerPoint {
Expand Down Expand Up @@ -67,28 +68,28 @@ const formatPercent = (value: number) => `${value.toFixed(1)}%`;

const width = 820;
const height = 470;
const padLeft = 66;
const padRight = 30;
const padTop = 34;
const padBottom = 62;
const padLeft = 60;
const padRight = 24;
const padTop = 28;
const padBottom = 56;
const plotWidth = width - padLeft - padRight;
const plotHeight = height - padTop - padBottom;
const languageAxisMax = 14;
// The vertical axis starts at the tens below the lowest analyzer rather than
// at 0, so the plot spends its height on the range the analyzers occupy.
const axis = percentAxis(points.map((point) => percent(point.correct, point.covered)));
const x = (languages: number) => padLeft + languages * plotWidth / languageAxisMax;
const y = (correct: number, covered: number) => padTop + (100 - percent(correct, covered)) * plotHeight / 100;
const y = (value: number) => padTop + (axis.max - value) * plotHeight / (axis.max - axis.min);
const xTicks = [0, 2, 4, 6, 8, 10, 12, 14];
const yTicks = [50, 60, 70, 80, 90, 100];
// Small, stable offsets keep labels readable without changing point geometry.
const labelOffsets: Record<string, { x: number; y: number; anchor?: 'start' | 'end' }> = {
bifrost: { x: -9, y: -11, anchor: 'end' },
codeql: { x: -9, y: 17, anchor: 'end' },
joern: { x: -9, y: -11, anchor: 'end' },
semgrep: { x: -9, y: 17, anchor: 'end' },
opentaint: { x: -10, y: -11, anchor: 'end' },
infer: { x: 9, y: -11 },
flowdroid: { x: 9, y: 17 },
pysa: { x: 9, y: 17 },
};
const plotted = points.map((point) => ({
...point,
px: x(point.languageSupport.languages.length),
py: y(percent(point.correct, point.covered)),
}));
const labels = placeLabels(
plotted.map((point) => ({ x: point.px, y: point.py, text: point.name })),
{ bounds: { left: padLeft + 4, top: 4, right: width - 4, bottom: height - padBottom - 4 }, radius: 8, fontSize: 13, charWidth: 0.58 },
);
---

<h2 id="landscape">Accuracy and language coverage — current snapshot</h2>
Expand All @@ -108,33 +109,26 @@ const labelOffsets: Record<string, { x: number; y: number; anchor?: 'start' | 'e
<svg class="landscape-chart" viewBox={`0 0 ${width} ${height}`} role="img" aria-labelledby="landscape-title landscape-desc">
<title id="landscape-title">Current analyzer accuracy by supported data-flow languages</title>
<desc id="landscape-desc">Scatter plot of all analyzers in DataFlowBench {currentSnapshot.version}. The horizontal axis is the documented number of languages in which each analyzer performs data-flow analysis. The vertical axis is decisive-correct assertions divided by every assertion in the benchmark kernels that analyzer covers, including non-answers in the denominator.</desc>
{yTicks.map((tick) => {
const tickY = padTop + (100 - tick) * plotHeight / 100;
return <>
<line class="grid" x1={padLeft} y1={tickY} x2={width - padRight} y2={tickY} />
<text class="tick" x={padLeft - 10} y={tickY + 4} text-anchor="end">{tick}%</text>
</>;
})}
{xTicks.map((tick) => {
const tickX = x(tick);
return <>
<line class="grid vertical" x1={tickX} y1={padTop} x2={tickX} y2={height - padBottom} />
<text class="tick" x={tickX} y={height - padBottom + 22} text-anchor="middle">{tick}</text>
</>;
})}
{axis.ticks.map((tick) => <>
<line class="grid" x1={padLeft} y1={y(tick)} x2={width - padRight} y2={y(tick)} />
<text class="tick" x={padLeft - 10} y={y(tick) + 4} text-anchor="end">{tick}%</text>
</>)}
{xTicks.map((tick) => <>
<line class="grid vertical" x1={x(tick)} y1={padTop} x2={x(tick)} y2={height - padBottom} />
<text class="tick" x={x(tick)} y={height - padBottom + 20} text-anchor="middle">{tick}</text>
</>)}
<line class="axis" x1={padLeft} y1={height - padBottom} x2={width - padRight} y2={height - padBottom} />
<line class="axis" x1={padLeft} y1={padTop} x2={padLeft} y2={height - padBottom} />
<text class="axis-title" x={(padLeft + width - padRight) / 2} y={height - 10} text-anchor="middle">languages with documented data-flow support</text>
<text class="axis-title" transform={`translate(17 ${(padTop + height - padBottom) / 2}) rotate(-90)`} text-anchor="middle">accuracy within covered kernels</text>
{points.map((point) => {
const pointX = x(point.languageSupport.languages.length);
const pointY = y(point.correct, point.covered);
const offset = labelOffsets[point.tool] ?? { x: 9, y: -9 };
<text class="axis-title" x={(padLeft + width - padRight) / 2} y={height - 12} text-anchor="middle">Languages with documented data-flow support</text>
<text class="axis-title" transform={`translate(16 ${(padTop + height - padBottom) / 2}) rotate(-90)`} text-anchor="middle">Accuracy within covered kernels</text>
{plotted.map((point, index) => {
const label = labels[index];
const accuracy = percent(point.correct, point.covered);
const tooltip = `${point.name} — x: ${point.languageSupport.languages.length} supported languages · y: ${formatPercent(accuracy)} accuracy`;
return <g class:list={['analyzer', point.colorClass]} tabindex="0" role="img" aria-label={tooltip} data-tooltip={tooltip}>
<circle class:list={['point', point.cohort]} cx={pointX} cy={pointY} r="7" />
<text class="point-label" x={pointX + offset.x} y={pointY + offset.y} text-anchor={offset.anchor ?? 'start'}>{point.name}</text>
{label.leader && <line class="leader" x1={point.px} y1={point.py} x2={label.x} y2={label.y - 4} />}
<circle class:list={['point', point.cohort]} cx={point.px} cy={point.py} r="6.5" />
<text class="point-label" x={label.x} y={label.y} text-anchor={label.anchor}>{point.name}</text>
</g>;
})}
</svg>
Expand All @@ -148,42 +142,48 @@ const labelOffsets: Record<string, { x: number; y: number; anchor?: 'start' | 'e

<details class="landscape-data">
<summary>Exact figures</summary>
<div class="table-wrap">
<div class="table-wrap ds-table">
<table>
<thead><tr><th>Analyzer</th><th>Scope</th><th>Supported languages</th><th>Benchmark kernels</th><th>Accuracy</th><th>Correct</th><th>Wrong</th><th>Incomplete</th></tr></thead>
<tbody>{points.map((point) => <tr>
<th scope="row"><span class:list={['vendor-dot', point.colorClass]}></span>{point.name}</th>
<td>{point.cohort}</td>
<td><a href={point.languageSupport.sourceUrl} title={point.languageSupport.source}>{point.languageSupport.languages.length}</a><small>{point.languageSupport.languages.join(', ')}</small></td>
<td>{point.kernels}/{kernelCount} ({formatPercent(percent(point.kernels, kernelCount))})</td>
<td>{point.correct}/{point.covered} ({formatPercent(percent(point.correct, point.covered))})</td>
<td class="num">{point.kernels}/{kernelCount} ({formatPercent(percent(point.kernels, kernelCount))})</td>
<td class="num">{point.correct}/{point.covered} ({formatPercent(percent(point.correct, point.covered))})</td>
<td>{point.correct}</td><td>{point.wrong}</td><td>{point.incomplete}</td>
</tr>)}</tbody>
</table>
</div>
</details>

<style>
.landscape-frame { position: relative; border: 1px solid var(--sl-color-hairline); border-radius: .75rem; padding: .75rem .5rem .25rem; background: var(--sl-color-bg-sidebar); overflow-x: auto; }
.landscape-frame { position: relative; margin-top: 1.25rem; border: 1px solid var(--ds-line); padding: 1rem .75rem .5rem; background: var(--ds-well); overflow-x: auto; }
.landscape-chart { display: block; width: 100%; min-width: 42rem; height: auto; }
.grid { stroke: var(--sl-color-hairline); stroke-width: 1; }
.grid.vertical { stroke-dasharray: 3 5; }
.axis { stroke: var(--sl-color-gray-4); stroke-width: 1.5; }
.tick, .axis-title { font-family: var(--__sl-font-mono); font-size: 11px; fill: var(--sl-color-gray-3); }
.axis-title { font-size: 12px; }
.point { fill: var(--vendor-rule); stroke: var(--sl-color-bg-sidebar); stroke-width: 2; }
.grid { stroke: var(--ds-line-soft); stroke-width: 1; }
.grid.vertical { stroke-dasharray: 2 4; }
.axis { stroke: var(--ds-line-strong); stroke-width: 1; }
.tick { font-family: var(--ds-font-mono); font-size: 11px; fill: var(--ds-text-3); }
.axis-title { font-family: var(--ds-font); font-size: 13px; font-weight: 500; fill: var(--ds-text-2); }
.point { fill: var(--vendor-rule); stroke: var(--ds-well); stroke-width: 2; }
.leader { stroke: var(--vendor-rule); stroke-width: 1; opacity: .55; }
.analyzer { cursor: default; outline: none; }
.analyzer:is(:hover, :focus-visible) .point { stroke: var(--sl-color-text); stroke-width: 3; }
.point.specialist { stroke-dasharray: 3 2; stroke-width: 3; }
.point-label { font-family: var(--__sl-font-mono); font-size: 12px; font-weight: 650; fill: var(--vendor-rule); paint-order: stroke; stroke: var(--sl-color-bg-sidebar); stroke-width: 4px; stroke-linejoin: round; }
.landscape-key { list-style: none; display: flex; gap: 1.25rem; margin: .65rem 0 0; padding: 0; color: var(--sl-color-gray-2); font-size: var(--sl-text-sm); }
.analyzer:is(:hover, :focus-visible) .point { stroke: var(--ds-text); stroke-width: 2.5; }
.analyzer:focus-visible .point-label { text-decoration: underline; }
/* A specialist is drawn as a ring, so the distinction survives without colour. */
.point.specialist { fill: var(--ds-well); stroke: var(--vendor-rule); stroke-width: 2.5; }
.point-label { font-family: var(--ds-font); font-size: 13px; font-weight: 500; fill: var(--vendor-rule); paint-order: stroke; stroke: var(--ds-well); stroke-width: 4px; stroke-linejoin: round; }
.landscape-key { list-style: none; display: flex; gap: 1.5rem; margin: .65rem 0 0; padding: 0; color: var(--ds-text-3); font-size: var(--sl-text-sm); }
.landscape-key li { display: flex; align-items: center; margin: 0; }
.shape { width: .7rem; height: .7rem; border-radius: 50%; margin-right: .4rem; background: var(--sl-color-gray-3); border: 2px solid var(--sl-color-bg); }
.shape.specialist { border: 2px dashed var(--sl-color-gray-3); background: transparent; }
.shape { width: .7rem; height: .7rem; border-radius: 50%; margin-right: .45rem; background: var(--ds-text-3); }
.shape.specialist { background: transparent; border: 2px solid var(--ds-text-3); }
.landscape-data { margin-top: .8rem; }
.table-wrap { overflow-x: auto; }
td small { display: block; min-width: 12rem; margin-top: .2rem; color: var(--sl-color-gray-3); line-height: 1.35; }
.vendor-dot { display: inline-block; width: .55rem; height: .55rem; border-radius: 50%; margin-right: .4rem; background: var(--vendor-rule); }
td small { display: block; min-width: 12rem; margin-top: .2rem; color: var(--ds-text-3); line-height: 1.35; }
/* A name and a fraction each read as one unit; the table pans before either
splits across lines. */
th[scope='row'], .num { white-space: nowrap; }
.vendor-dot { display: inline-block; width: .55rem; height: .55rem; border-radius: 50%; margin-right: .45rem; background: var(--vendor-rule); }
.v0 { --vendor-rule: var(--sl-color-accent-high); } .v1 { --vendor-rule: var(--sl-color-orange); } .v2 { --vendor-rule: var(--sl-color-purple); } .v3 { --vendor-rule: var(--sl-color-green); }
.v4 { --vendor-rule: var(--dfb-vendor-4); } .v5 { --vendor-rule: var(--dfb-vendor-5); } .v6 { --vendor-rule: var(--dfb-vendor-6); } .v7 { --vendor-rule: var(--dfb-vendor-7); }
</style>
38 changes: 20 additions & 18 deletions docs/src/components/AnalyzerProfiles.astro
Original file line number Diff line number Diff line change
Expand Up @@ -300,25 +300,27 @@ const cards = model.profiles.map((profile) => {
}
.profile-card {
margin: 0;
border: 1px solid var(--sl-color-hairline);
border-radius: 0.75rem;
padding: 0.75rem 0.75rem 0.6rem;
background: var(--sl-color-bg-sidebar);
border: 1px solid var(--ds-line);
border-radius: var(--ds-cut-control);
corner-shape: bevel;
padding: 0.95rem 1rem 0.85rem;
background: var(--ds-surface);
}
.profile-name {
display: flex;
align-items: center;
font-weight: 600;
font-weight: 500;
font-size: var(--sl-text-base);
color: var(--sl-color-white);
letter-spacing: -0.01em;
color: var(--ds-text);
margin: 0 0 0.25rem;
}
.vendor-dot {
display: inline-block;
width: 0.55rem;
height: 0.55rem;
border-radius: 50%;
margin-right: 0.4rem;
margin-right: 0.45rem;
background: var(--vendor-rule, var(--sl-color-accent));
}
.profile-radar {
Expand All @@ -328,11 +330,11 @@ const cards = model.profiles.map((profile) => {
}
.profile-radar .ring {
fill: none;
stroke: var(--sl-color-hairline);
stroke: var(--ds-line);
stroke-width: 1;
}
.profile-radar .spoke {
stroke: var(--sl-color-hairline);
stroke: var(--ds-line-soft);
stroke-width: 1;
}
.profile-radar .shape {
Expand All @@ -346,9 +348,10 @@ const cards = model.profiles.map((profile) => {
fill: var(--vendor-rule, var(--sl-color-accent));
}
.profile-radar .axis-label {
font-family: var(--__sl-font-mono);
font-size: 11px;
fill: var(--sl-color-gray-3);
font-family: var(--ds-font);
font-size: 11.5px;
font-weight: 500;
fill: var(--ds-text-3);
}
.readings {
display: grid;
Expand All @@ -358,26 +361,25 @@ const cards = model.profiles.map((profile) => {
font-size: var(--sl-text-xs);
}
.readings dt {
font-family: var(--__sl-font-mono);
color: var(--sl-color-gray-3);
color: var(--ds-text-3);
}
.readings dd {
margin: 0;
display: flex;
flex-direction: column;
line-height: 1.3;
color: var(--sl-color-gray-2);
color: var(--ds-text-2);
}
.readings .basis {
color: var(--sl-color-gray-3);
color: var(--ds-text-3);
}
.axis-notes {
font-size: var(--sl-text-sm);
color: var(--sl-color-gray-3);
color: var(--ds-text-3);
}
.caption {
font-size: var(--sl-text-sm);
color: var(--sl-color-gray-3);
color: var(--ds-text-3);
}
/* The same fixed vendor→colour identity map every other section uses. */
.v0 {
Expand Down
Loading
Loading