Loading benchmark results...
<style>
#benchmark-dashboard table {
width: 100%;
border-collapse: collapse;
font-size: 0.9rem;
}
#benchmark-dashboard th {
text-align: left;
padding: 0.6rem 0.8rem;
border-bottom: 2px solid var(--md-default-fg-color--lightest);
white-space: nowrap;
}
#benchmark-dashboard td {
padding: 0.5rem 0.8rem;
border-bottom: 1px solid var(--md-default-fg-color--lightest);
vertical-align: top;
}
#benchmark-dashboard tr:hover td {
background: var(--md-default-fg-color--lightest);
}
#benchmark-dashboard .status-success { color: #22863a; }
#benchmark-dashboard .status-failure { color: #cb2431; }
#benchmark-dashboard .error-msg {
color: var(--md-default-fg-color--light);
font-style: italic;
}
#benchmark-dashboard .section-title {
margin-top: 2.5rem;
margin-bottom: 0.5rem;
font-size: 1.2rem;
font-weight: 600;
}
#benchmark-dashboard .section-explanation {
font-size: 0.85rem;
opacity: 0.6;
margin: 0.3rem 0 1.5rem 0;
font-style: italic;
}
#benchmark-dashboard .avg-summary {
font-size: 1.05rem;
margin: 0.8rem 0;
line-height: 1.6;
}
#benchmark-dashboard .avg-summary strong {
font-size: 1.1rem;
}
#benchmark-dashboard .footnote {
font-size: 0.8rem;
opacity: 0.6;
font-style: italic;
margin: 0.5rem 0 1.5rem 0;
}
/* Hero section */
#benchmark-dashboard .hero-section { text-align: center; margin: 1rem 0 2.5rem 0; }
#benchmark-dashboard .hero-cards { display: flex; gap: 2rem; justify-content: center; flex-wrap: wrap; margin: 1.5rem 0; }
#benchmark-dashboard .hero-card { flex: 1; text-align: center; padding: 2rem; border-radius: 12px; background: var(--md-default-bg-color--light, #f5f5f5); min-width: 240px; border: 2px solid; }
#benchmark-dashboard .hero-card.factory { border-color: #4285f4; }
#benchmark-dashboard .hero-card.claude-code { border-color: #ff7043; }
#benchmark-dashboard .hero-score { font-size: 3rem; font-weight: 800; line-height: 1.1; }
#benchmark-dashboard .hero-label { font-size: 1.1rem; color: var(--md-default-fg-color--light); margin-bottom: 0.5rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; }
#benchmark-dashboard .hero-trend { font-size: 0.9rem; margin-top: 0.5rem; }
#benchmark-dashboard .hero-explanation { font-size: 0.85rem; opacity: 0.6; max-width: 600px; margin: 0 auto; }
/* Filter bar */
#benchmark-dashboard .filter-bar {
display: flex;
gap: 1rem;
margin: 0.5rem 0 1rem 0;
align-items: center;
flex-wrap: wrap;
}
#benchmark-dashboard .filter-bar select {
padding: 0.3rem 0.6rem;
border-radius: 4px;
border: 1px solid var(--md-default-fg-color--lightest);
background: var(--md-default-bg-color);
color: var(--md-default-fg-color);
font-size: 0.85rem;
}
#benchmark-dashboard .filter-bar label {
font-size: 0.85rem;
font-weight: 500;
}
/* Run history expandable rows */
#benchmark-dashboard .run-detail { background: rgba(255,255,255,0.02); }
#benchmark-dashboard .run-detail td:first-child { padding-left: 2rem; font-size: 0.85rem; opacity: 0.85; }
#benchmark-dashboard .run-detail.hidden { display: none; }
#benchmark-dashboard .toggle-details {
background: none;
border: 1px solid var(--md-default-fg-color--lightest);
border-radius: 4px;
cursor: pointer;
padding: 0.15rem 0.5rem;
font-size: 0.8rem;
color: var(--md-default-fg-color);
}
#benchmark-dashboard .toggle-details:hover {
background: var(--md-default-fg-color--lightest);
}
/* Trend indicators */
#benchmark-dashboard .trend-up { color: #22863a; }
#benchmark-dashboard .trend-down { color: #cb2431; }
#benchmark-dashboard .trend-neutral { color: #888; }
/* Trace analysis */
#benchmark-dashboard .trace-analysis-row td { padding: 0 0.8rem 0.8rem 2rem; }
#benchmark-dashboard .trace-content { font-family: var(--md-code-font-family, monospace); font-size: 0.8rem; background: rgba(128,128,128,0.1); padding: 0.8rem; border-radius: 4px; white-space: pre-wrap; max-height: 400px; overflow-y: auto; line-height: 1.5; }
#benchmark-dashboard .trace-link { font-size: 0.8rem; margin-left: 0.5rem; opacity: 0.7; }
#benchmark-dashboard .trace-link:hover { opacity: 1; }
/* Pagination controls */
#benchmark-dashboard .pagination { display: flex; justify-content: center; align-items: center; gap: 8px; margin: 16px 0; flex-wrap: wrap; }
#benchmark-dashboard .pagination button { background: #2a2a3e; color: #e0e0e0; border: 1px solid #444; border-radius: 6px; padding: 6px 12px; cursor: pointer; font-size: 0.85rem; }
#benchmark-dashboard .pagination button:hover:not(:disabled) { background: #3a3a5e; }
#benchmark-dashboard .pagination button:disabled { opacity: 0.4; cursor: not-allowed; }
#benchmark-dashboard .pagination button.active { background: #667eea; border-color: #667eea; }
#benchmark-dashboard .pagination .page-info { color: #aaa; font-size: 0.85em; }
</style>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
<script>
const REPO = 'akashgit/remote-factory';
const JSONL_URL = `https://raw.githubusercontent.com/${REPO}/benchmark-data/results.jsonl`;
function isDarkMode() {
return document.body.getAttribute('data-md-color-scheme') === 'slate';
}
function chartColors() {
const dark = isDarkMode();
return {
text: dark ? '#ccc' : '#333',
grid: dark ? '#444' : '#e0e0e0',
};
}
function formatDurationShort(seconds) {
if (seconds == null) return '—';
const m = Math.floor(seconds / 60);
const s = Math.round(seconds % 60);
return m > 0 ? m + 'm ' + s + 's' : s + 's';
}
function formatCost(usd) {
if (usd == null) return '—';
return '$' + usd.toFixed(2);
}
function formatDate(ts) {
if (!ts) return '—';
const d = parseTimestamp(ts);
if (!d) return ts;
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })
+ ' ' + d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
}
function parseTimestamp(ts) {
if (!ts) return null;
const iso = ts.replace(
/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/,
'$1-$2-$3T$4:$5:$6Z'
);
const d = new Date(iso);
return isNaN(d) ? null : d;
}
function statusIcon(resolved) {
return resolved
? 'PASS'
: 'FAIL';
}
function getSolver(r) {
return r.solver || r.details?.solver || 'unknown';
}
function isMainBranch(r) {
return r.ref === 'refs/heads/main';
}
function isPR(r) {
return (r.ref && r.ref.startsWith('refs/pull/')) || !!r.pr_number;
}
function formatRefLabel(r) {
if (!r.ref) return '—';
if (r.ref === 'refs/heads/main') return 'main';
if (r.ref.startsWith('refs/pull/'))
return 'PR #' + r.ref.replace('refs/pull/', '').replace('/merge', '');
return r.ref.replace('refs/heads/', '');
}
async function fetchJsonl() {
const resp = await fetch(JSONL_URL);
if (!resp.ok) return null;
const text = await resp.text();
const lines = text.trim().split('\n').filter(Boolean);
return lines.map(l => {
try { return JSON.parse(l); }
catch { return null; }
}).filter(Boolean);
}
function getLatestByCombo(results) {
const byKey = {};
for (const r of results) {
const solver = getSolver(r);
const key = r.benchmark + '|' + solver;
if (!byKey[key] || (r.timestamp > byKey[key].timestamp)) {
byKey[key] = r;
}
}
return byKey;
}
function groupByRunId(results) {
const runs = {};
for (const entry of results) {
const rid = entry.run_id || entry.timestamp || Math.random().toString();
if (!runs[rid]) {
runs[rid] = {
run_id: rid,
date: entry.timestamp,
ref: entry.ref,
commit: entry.commit,
run_url: entry.run_url,
trigger: entry.trigger,
jobs: []
};
}
runs[rid].jobs.push(entry);
}
for (const run of Object.values(runs)) {
run.passed = run.jobs.filter(j => j.resolved).length;
run.total = run.jobs.length;
run.totalCost = run.jobs.reduce((s, j) => s + (j.details?.cost_usd || 0), 0);
run.maxDuration = Math.max(...run.jobs.map(j => j.duration_seconds || 0));
run.branch = formatRefLabel(run);
run.passRate = run.total > 0 ? (run.passed / run.total * 100) : 0;
run.score = run.total > 0 ? run.passRate.toFixed(0) + '%' : '—';
}
return Object.values(runs).sort((a, b) =>
(b.date || '').localeCompare(a.date || '')
);
}
function computeRunAccuracy(jobs) {
const resolved = jobs.filter(j => j.resolved).length;
return jobs.length > 0 ? (resolved / jobs.length) * 100 : null;
}
function computeAccuracyPoints(mainResults) {
const runsByRunId = {};
for (const e of mainResults) {
const rid = e.run_id || e.timestamp;
if (!runsByRunId[rid]) runsByRunId[rid] = [];
runsByRunId[rid].push(e);
}
const factoryPoints = [];
const claudePoints = [];
for (const [, entries] of Object.entries(runsByRunId)) {
const date = parseTimestamp(entries[0].timestamp);
if (!date) continue;
const factoryJobs = entries.filter(e => getSolver(e) === 'factory');
const claudeJobs = entries.filter(e => getSolver(e) === 'claude-code');
if (factoryJobs.length > 0) {
factoryPoints.push({ x: date, y: computeRunAccuracy(factoryJobs) });
}
if (claudeJobs.length > 0) {
claudePoints.push({ x: date, y: computeRunAccuracy(claudeJobs) });
}
}
factoryPoints.sort((a, b) => a.x - b.x);
claudePoints.sort((a, b) => a.x - b.x);
return { factoryPoints, claudePoints };
}
function computeRunAverages(mainResults) {
const runsByRunId = {};
for (const e of mainResults) {
const rid = e.run_id || e.timestamp || '';
if (!runsByRunId[rid]) runsByRunId[rid] = [];
runsByRunId[rid].push(e);
}
const factoryDurationPoints = [];
const claudeDurationPoints = [];
const factoryCostPoints = [];
const claudeCostPoints = [];
for (const [, entries] of Object.entries(runsByRunId)) {
const date = parseTimestamp(entries[0].timestamp);
if (!date) continue;
const factoryJobs = entries.filter(e => getSolver(e) === 'factory');
const claudeJobs = entries.filter(e => getSolver(e) === 'claude-code');
if (factoryJobs.length > 0) {
const durs = factoryJobs.filter(j => j.duration_seconds != null);
const costs = factoryJobs.filter(j => j.details?.cost_usd != null);
if (durs.length > 0)
factoryDurationPoints.push({ x: date, y: durs.reduce((s, j) => s + j.duration_seconds, 0) / durs.length });
if (costs.length > 0)
factoryCostPoints.push({ x: date, y: costs.reduce((s, j) => s + j.details.cost_usd, 0) / costs.length });
}
if (claudeJobs.length > 0) {
const durs = claudeJobs.filter(j => j.duration_seconds != null);
const costs = claudeJobs.filter(j => j.details?.cost_usd != null);
if (durs.length > 0)
claudeDurationPoints.push({ x: date, y: durs.reduce((s, j) => s + j.duration_seconds, 0) / durs.length });
if (costs.length > 0)
claudeCostPoints.push({ x: date, y: costs.reduce((s, j) => s + j.details.cost_usd, 0) / costs.length });
}
}
const sortByDate = (a, b) => a.x - b.x;
factoryDurationPoints.sort(sortByDate);
claudeDurationPoints.sort(sortByDate);
factoryCostPoints.sort(sortByDate);
claudeCostPoints.sort(sortByDate);
return { factoryDurationPoints, claudeDurationPoints, factoryCostPoints, claudeCostPoints };
}
// Section 1: Hero — Headline Numbers
function renderHero(mainResults) {
const runsByRunId = {};
for (const e of mainResults) {
const rid = e.run_id || e.timestamp;
if (!runsByRunId[rid]) runsByRunId[rid] = [];
runsByRunId[rid].push(e);
}
const runIds = Object.keys(runsByRunId).sort((a, b) => {
const ta = runsByRunId[a][0].timestamp || '';
const tb = runsByRunId[b][0].timestamp || '';
return tb.localeCompare(ta);
});
if (runIds.length === 0) return '';
const latestEntries = runsByRunId[runIds[0]];
const priorEntries = runIds.length > 1 ? runsByRunId[runIds[1]] : [];
function solverAccuracy(entries, solver) {
const jobs = entries.filter(e => getSolver(e) === solver);
return computeRunAccuracy(jobs);
}
const factoryAcc = solverAccuracy(latestEntries, 'factory');
const claudeAcc = solverAccuracy(latestEntries, 'claude-code');
const prevFactoryAcc = solverAccuracy(priorEntries, 'factory');
const prevClaudeAcc = solverAccuracy(priorEntries, 'claude-code');
function trendHtml(current, prev) {
if (current == null) return '';
if (prev == null) return 'first run
';
const delta = current - prev;
if (Math.abs(delta) < 0.5) return '= no change
';
const sign = delta > 0 ? '+' : '';
const cls = delta > 0 ? 'trend-up' : 'trend-down';
const arrow = delta > 0 ? '▲' : '▼';
return `${arrow} ${sign}${delta.toFixed(0)}% vs prior run
`;
}
let html = '';
html += '
';
html += '
';
html += '
Factory
';
html += `
${factoryAcc != null ? factoryAcc.toFixed(0) + '%' : '—'}
`;
html += trendHtml(factoryAcc, prevFactoryAcc);
html += '
';
html += '
';
html += '
Claude Code
';
html += `
${claudeAcc != null ? claudeAcc.toFixed(0) + '%' : '—'}
`;
html += trendHtml(claudeAcc, prevClaudeAcc);
html += '
';
html += '
';
html += '
Latest main branch results, averaged across all benchmarks. Accuracy = percentage of benchmark tasks solved correctly (higher is better).
';
html += '
Accuracy Over Time
';
html += 'Average solve rate across all benchmarks for each CI run on the main branch.
';
html += '';
setTimeout(() => {
const ctx = document.getElementById('accuracy-trend-chart');
if (!ctx) return;
const colors = chartColors();
new Chart(ctx, {
type: 'line',
data: {
datasets: [
{
label: 'Factory',
data: factoryPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
tension: 0,
pointRadius: 4,
},
{
label: 'Claude Code',
data: claudePoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
tension: 0,
pointRadius: 4,
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: { labels: { color: colors.text } },
tooltip: {
callbacks: {
label: (item) => item.dataset.label + ': ' + item.raw.y.toFixed(0) + '%',
},
},
},
scales: {
x: {
type: 'time',
time: { unit: 'day', tooltipFormat: 'MMM d, yyyy HH:mm' },
title: { display: true, text: 'Date', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
y: {
min: 0,
max: 100,
title: { display: true, text: 'Accuracy (%)', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
},
},
});
}, 0);
return html;
}
// Section 3: Duration Trend — Line Chart Over Time
function renderDurationTrend(mainResults) {
const runAvgs = computeRunAverages(mainResults);
if (runAvgs.factoryDurationPoints.length === 0 && runAvgs.claudeDurationPoints.length === 0) return '';
let html = 'Average Duration Over Time
';
html += 'Average wall-clock time per benchmark task. Lower is better.
';
html += '';
setTimeout(() => {
const ctx = document.getElementById('duration-trend-chart');
if (!ctx) return;
const colors = chartColors();
new Chart(ctx, {
type: 'line',
data: {
datasets: [
{
label: 'Factory',
data: runAvgs.factoryDurationPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
tension: 0,
pointRadius: 4,
},
{
label: 'Claude Code',
data: runAvgs.claudeDurationPoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
tension: 0,
pointRadius: 4,
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: { labels: { color: colors.text } },
tooltip: {
callbacks: {
label: (item) => item.dataset.label + ': ' + formatDurationShort(item.raw.y),
},
},
},
scales: {
x: {
type: 'time',
time: { unit: 'day', tooltipFormat: 'MMM d, yyyy HH:mm' },
title: { display: true, text: 'Date', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
y: {
title: { display: true, text: 'Duration (seconds)', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
},
},
});
}, 0);
return html;
}
// Section 4: Cost Trend — Line Chart Over Time
function renderCostTrend(mainResults) {
const runAvgs = computeRunAverages(mainResults);
if (runAvgs.factoryCostPoints.length === 0 && runAvgs.claudeCostPoints.length === 0) return '';
const totalFactory = runAvgs.factoryCostPoints.reduce((s, p) => s + p.y, 0);
const totalClaude = runAvgs.claudeCostPoints.reduce((s, p) => s + p.y, 0);
let html = 'Average Cost Per Run Over Time
';
html += '';
html += `Total Spend: Factory ${formatCost(totalFactory)} | Claude Code ${formatCost(totalClaude)}`;
html += '
';
html += 'Average API cost per benchmark task. Vertex AI runs may show $0 when billing is at the platform level.
';
html += '';
setTimeout(() => {
const ctx = document.getElementById('cost-trend-chart');
if (!ctx) return;
const colors = chartColors();
new Chart(ctx, {
type: 'line',
data: {
datasets: [
{
label: 'Factory',
data: runAvgs.factoryCostPoints,
borderColor: '#4285f4',
backgroundColor: '#4285f4',
tension: 0,
pointRadius: 4,
},
{
label: 'Claude Code',
data: runAvgs.claudeCostPoints,
borderColor: '#ff7043',
backgroundColor: '#ff7043',
tension: 0,
pointRadius: 4,
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: { labels: { color: colors.text } },
tooltip: {
callbacks: {
label: (item) => item.dataset.label + ': ' + formatCost(item.raw.y),
},
},
},
scales: {
x: {
type: 'time',
time: { unit: 'day', tooltipFormat: 'MMM d, yyyy HH:mm' },
title: { display: true, text: 'Date', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
y: {
title: { display: true, text: 'Cost (USD)', color: colors.text },
ticks: { color: colors.text },
grid: { color: colors.grid },
},
},
},
});
}, 0);
return html;
}
// Section 5: Per-Benchmark Breakdown Table
function renderPerBenchmarkTable(mainResults) {
const byKey = getLatestByCombo(mainResults);
const combos = Object.entries(byKey).sort(([a], [b]) => a.localeCompare(b));
if (combos.length === 0) {
return 'Latest Results by Benchmark
'
+ 'No main branch results yet.
';
}
let html = 'Latest Results by Benchmark
';
html += 'Most recent main branch result for each benchmark and solver combination.
';
html += '';
html += '';
html += '';
html += '';
for (const [, r] of combos) {
const solver = getSolver(r);
const commit = r.commit ? r.commit.substring(0, 7) : '—';
const commitLink = r.commit
? `${commit}`
: commit;
const runLink = r.run_url
? `details`
: '—';
html += '';
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
const traceSummary = r.trace_summary || '';
const escaped = traceSummary.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"');
const truncated = escaped.length > 80 ? escaped.substring(0, 77) + '...' : escaped;
const traceHtml = r.trace_url
? '' + (truncated || 'trace') + ''
: (truncated || '—');
html += '';
html += ``;
html += ``;
html += '';
}
html += '
| Benchmark |
Solver |
Result |
Duration |
Cost |
Trace |
Commit |
Run |
| ${r.benchmark} |
${solver} |
${statusIcon(r.resolved)} ${(r.score * 100).toFixed(0)}% |
${formatDurationShort(r.duration_seconds)} |
${formatCost(r.details?.cost_usd)} |
' + traceHtml + ' |
${commitLink} |
${runLink} |
';
return html;
}
function formatTrendColumn(delta) {
if (!delta) return '—';
let parts = [];
const accVal = delta.accuracy;
if (Math.abs(accVal) >= 0.5) {
const sign = accVal > 0 ? '+' : '';
const cls = accVal > 0 ? 'trend-up' : 'trend-down';
const arrow = accVal > 0 ? '▲' : '▼';
parts.push(`${arrow}${sign}${accVal.toFixed(0)}% acc`);
}
const costVal = delta.cost;
if (Math.abs(costVal) >= 0.01) {
const sign = costVal > 0 ? '+' : '';
const cls = costVal > 0 ? 'trend-down' : 'trend-up';
const arrow = costVal > 0 ? '▲' : '▼';
parts.push(`${arrow}${sign}$${costVal.toFixed(2)} cost`);
}
const durVal = delta.duration;
if (Math.abs(durVal) >= 1) {
const sign = durVal > 0 ? '+' : '';
const cls = durVal > 0 ? 'trend-down' : 'trend-up';
const arrow = durVal > 0 ? '▲' : '▼';
parts.push(`${arrow}${sign}${formatDurationShort(Math.abs(durVal))} dur`);
}
return parts.length > 0 ? parts.join('
') : '= no change';
}
// Section 6: Run History — paginated with expandable details
let currentPage = 1;
const RUNS_PER_PAGE = 20;
let allSortedRuns = [];
function getFilteredRuns() {
const bv = document.getElementById('filter-benchmark')?.value || '';
const sv = document.getElementById('filter-solver')?.value || '';
return allSortedRuns.filter(run => {
const runBenchmarks = [...new Set(run.jobs.map(j => j.benchmark))];
const runSolvers = [...new Set(run.jobs.map(j => getSolver(j)))];
return (!bv || runBenchmarks.includes(bv)) && (!sv || runSolvers.includes(sv));
});
}
function renderPaginationControls(current, total, totalItems, startIdx, endIdx) {
if (totalItems === 0) return 'No matching runs';
if (total <= 1) return `Showing all ${totalItems} runs`;
let html = `Showing runs ${startIdx + 1}–${endIdx} of ${totalItems}`;
html += `← Prev`;
const pages = [];
pages.push(1);
if (current > 3) pages.push('...');
for (let p = Math.max(2, current - 1); p <= Math.min(total - 1, current + 1); p++) {
pages.push(p);
}
if (current < total - 2) pages.push('...');
if (total > 1) pages.push(total);
for (const p of pages) {
if (p === '...') {
html += '…';
} else {
html += `${p}`;
}
}
html += `Next →`;
return html;
}
function updateRunHistoryTable() {
const filteredRuns = getFilteredRuns();
const totalPages = Math.max(1, Math.ceil(filteredRuns.length / RUNS_PER_PAGE));
if (currentPage > totalPages) currentPage = totalPages;
const start = (currentPage - 1) * RUNS_PER_PAGE;
const pageRuns = filteredRuns.slice(start, start + RUNS_PER_PAGE);
const bv = document.getElementById('filter-benchmark')?.value || '';
const sv = document.getElementById('filter-solver')?.value || '';
let rowsHtml = '';
for (const run of pageRuns) {
const rid = run.run_id.replace(/[^a-zA-Z0-9_-]/g, '_');
const jobBenchmarks = [...new Set(run.jobs.map(j => j.benchmark))].join(',');
const jobSolvers = [...new Set(run.jobs.map(j => getSolver(j)))].join(',');
rowsHtml += ``;
rowsHtml += `${formatDate(run.date)}`;
rowsHtml += `${run.branch}`;
rowsHtml += `${run.score} (${run.passed}/${run.total})`;
rowsHtml += `${formatCost(run.totalCost)}`;
rowsHtml += `${formatDurationShort(run.maxDuration)}`;
rowsHtml += `${formatTrendColumn(run.delta)}`;
rowsHtml += `+`;
rowsHtml += '';
for (const j of run.jobs) {
const solver = getSolver(j);
const matchB = !bv || j.benchmark === bv;
const matchS = !sv || solver === sv;
if (!matchB || !matchS) continue;
const runLink = j.run_url ? `link` : '';
const traceLink = j.trace_url ? `trace` : '';
rowsHtml += ``;
rowsHtml += `${j.benchmark}`;
rowsHtml += `${solver}`;
rowsHtml += `${statusIcon(j.resolved)} ${(j.score * 100).toFixed(0)}%`;
rowsHtml += `${formatCost(j.details?.cost_usd)}`;
rowsHtml += `${formatDurationShort(j.duration_seconds)}`;
rowsHtml += ``;
rowsHtml += `${runLink}${traceLink ? ' ' + traceLink : ''}`;
rowsHtml += '';
if (j.trace_analysis) {
const analysisTraceLink = j.trace_url ? ' View in Langfuse →' : '';
rowsHtml += ``;
rowsHtml += 'Trace Analysis' + analysisTraceLink + '
';
rowsHtml += '' + j.trace_analysis.replace(//g, '>').replace(/\n/g, '
') + '
';
rowsHtml += 'Run History
';
html += 'All benchmark runs, newest first. Trend arrows compare each run against the immediately preceding run. Green = improvement, red = regression.
';
html += '';
html += 'Benchmark: All';
for (const b of benchmarks) html += `${b}`;
html += '';
html += 'Solver: All';
for (const s of solvers) html += `${s}`;
html += '';
html += '
';
html += '';
html += '';
html += '';
html += '';
html += '| Date | Branch | Pass Rate | Cost | Duration | Trend | |
|---|
';
html += '';
setTimeout(() => {
updateRunHistoryTable();
document.addEventListener('click', (e) => {
const btn = e.target.closest('.toggle-details');
if (!btn) return;
const runId = btn.dataset.target;
const details = document.querySelectorAll(`.run-detail[data-parent="${runId}"]`);
const isHidden = details[0]?.classList.contains('hidden');
details.forEach(row => row.classList.toggle('hidden'));
btn.textContent = isHidden ? '−' : '+';
});
document.addEventListener('click', (e) => {
const btn = e.target.closest('.page-btn');
if (!btn || btn.disabled) return;
const page = parseInt(btn.dataset.page);
if (page && page !== currentPage) {
currentPage = page;
updateRunHistoryTable();
}
});
const benchSelect = document.getElementById('filter-benchmark');
const solverSelect = document.getElementById('filter-solver');
if (benchSelect) benchSelect.addEventListener('change', () => { currentPage = 1; updateRunHistoryTable(); });
if (solverSelect) solverSelect.addEventListener('change', () => { currentPage = 1; updateRunHistoryTable(); });
}, 0);
return html;
}
// Section 7: PR Comparison
function renderPRComparison(allResults) {
const prResults = allResults.filter(isPR);
if (prResults.length === 0) return '';
const mainResults = allResults.filter(isMainBranch);
const mainLatest = getLatestByCombo(mainResults);
let html = 'Pull Request Comparison
';
html += 'Pull request benchmark results compared against the latest main branch baseline.
';
html += '';
html += '';
html += '';
html += '';
html += '';
html += '';
const sortedPR = [...prResults].sort((a, b) =>
(b.timestamp || '').localeCompare(a.timestamp || '')
);
for (const r of sortedPR) {
const solver = getSolver(r);
const key = r.benchmark + '|' + solver;
const baseline = mainLatest[key];
const prLabel = r.pr_number
? `#${r.pr_number}`
: r.ref ? r.ref.replace('refs/pull/', '').replace('/merge', '') : '—';
const prScore = r.score;
const mainScore = baseline ? baseline.score : null;
const scoreDelta = (prScore != null && mainScore != null) ? prScore - mainScore : null;
const prCost = r.details?.cost_usd;
const mainCost = baseline?.details?.cost_usd;
const costDelta = (prCost != null && mainCost != null) ? prCost - mainCost : null;
const prDur = r.duration_seconds;
const mainDur = baseline?.duration_seconds;
function formatDelta(val, invert) {
if (val == null) return '—';
const sign = val > 0 ? '+' : '';
const cls = val > 0
? (invert ? 'trend-down' : 'trend-up')
: val < 0
? (invert ? 'trend-up' : 'trend-down')
: 'trend-neutral';
return `${sign}${typeof val === 'number' && Math.abs(val) < 10 ? val.toFixed(2) : val}`;
}
html += '';
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += ``;
html += '';
}
html += '
| PR |
Benchmark |
Solver |
PR Score |
Main Score |
Delta |
PR Cost |
Main Cost |
Delta |
PR Duration |
Main Duration |
| ${prLabel} |
${r.benchmark} |
${solver} |
${prScore != null ? (prScore * 100).toFixed(0) + '%' : '—'} |
${mainScore != null ? (mainScore * 100).toFixed(0) + '%' : '—'} |
${formatDelta(scoreDelta, false)} |
${formatCost(prCost)} |
${formatCost(mainCost)} |
${formatDelta(costDelta != null ? Number(costDelta.toFixed(2)) : null, true)} |
${formatDurationShort(prDur)} |
${formatDurationShort(mainDur)} |
';
return html;
}
async function renderDashboard() {
const container = document.getElementById('benchmark-dashboard');
try {
const allResults = await fetchJsonl();
if (allResults && allResults.length > 0) {
const mainResults = allResults.filter(isMainBranch);
let html = '';
html += renderHero(mainResults);
html += renderAccuracyTrend(mainResults);
html += renderDurationTrend(mainResults);
html += renderCostTrend(mainResults);
html += renderPerBenchmarkTable(mainResults);
html += renderRunHistory(allResults);
html += renderPRComparison(allResults);
container.innerHTML = html;
return;
}
container.innerHTML = 'No benchmark data available yet.
'
+ 'Run the benchmark workflow to generate initial data: Actions → Benchmark CI → Run workflow
';
} catch (err) {
container.innerHTML = `${err.message}
`;
}
}
renderDashboard();
</script>