Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,19 @@ import fixture from "../public/benchmark-study.example.json" with { type: "json"
import {
parseBenchmarkStudyDashboard,
resolveBenchmarkStudyDashboardUrl,
selectBenchmarkStudyRun,
} from "../src/data/benchmark-study.js";

const packet = parseBenchmarkStudyDashboard(fixture);
const first = packet.runs[0]!;
const second = { ...first, case_id: "another-case" };
if (selectBenchmarkStudyRun([first, second], first.run_id, second.case_id) !== second) throw new Error("run selection lost its case");
if (selectBenchmarkStudyRun([first, second], first.run_id, "") !== undefined) throw new Error("ambiguous legacy link selected a run");
if (selectBenchmarkStudyRun([first], first.run_id, "") !== first) throw new Error("unique legacy link stopped resolving");
if (selectBenchmarkStudyRun([first], first.run_id, "missing-case") !== undefined) throw new Error("stale case link selected another case");
if (selectBenchmarkStudyRun([first], "missing-run", "") !== undefined) throw new Error("stale run link selected another run");
if (selectBenchmarkStudyRun([first, first], first.run_id, first.case_id) !== undefined) throw new Error("duplicate full identity selected a run");
if (selectBenchmarkStudyRun([first], "", "") !== first) throw new Error("unselected runs view lost its default");
if (packet.status !== "provisional") throw new Error("fixture must expose provisional coverage");
if (packet.campaign.intended_cell_denominator !== 8) throw new Error("campaign denominator drifted");
if (packet.campaign.selected_score_countable_cell_count !== 6) throw new Error("countable numerator drifted");
Expand Down
9 changes: 9 additions & 0 deletions apps/presentation/dashboard/src/data/benchmark-study.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,15 @@ export type BenchmarkStudyCase = BenchmarkStudyDashboard["cases"][number];
export type BenchmarkStudyRun = BenchmarkStudyDashboard["runs"][number];
export type BenchmarkStudyView = "campaign" | "arms" | "cases" | "runs";

export function selectBenchmarkStudyRun(
runs: BenchmarkStudyRun[], runId: string, caseId: string,
): BenchmarkStudyRun | undefined {
if (!runId && !caseId) return runs[0];
const matches = runs.filter((run) => run.run_id === runId && (!caseId || run.case_id === caseId));
// Legacy runId-only bookmarks are safe only when they identify exactly one run.
return matches.length === 1 ? matches[0] : undefined;
}

export function parseBenchmarkStudyDashboard(value: unknown): BenchmarkStudyDashboard {
return benchmarkStudyDashboardSchema.parse(value);
}
Expand Down
1 change: 1 addition & 0 deletions apps/presentation/dashboard/src/router.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ const benchmarkStudySearchSchema = z.object({
dashboardUrl: z.string().optional().default(""),
view: z.enum(["campaign", "arms", "cases", "runs"]).optional().default("campaign"),
runId: z.string().optional().default(""),
caseId: z.string().optional().default(""),
});

// Bookmarks remain valid, but the retired boards no longer ship a second UI.
Expand Down
21 changes: 11 additions & 10 deletions apps/presentation/dashboard/src/views/benchmark-study-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { benchmarkStudyRoute } from "../router";
import {
parseBenchmarkStudyDashboard,
resolveBenchmarkStudyDashboardUrl,
selectBenchmarkStudyRun,
type BenchmarkStudyArm,
type BenchmarkStudyCase,
type BenchmarkStudyDashboard,
Expand Down Expand Up @@ -190,7 +191,7 @@ function ArmsView({ packet }: { packet: BenchmarkStudyDashboard }) {
);
}

function CasesView({ packet, primaryMetric, onOpenRun }: { packet: BenchmarkStudyDashboard; primaryMetric: string; onOpenRun: (runId: string) => void }) {
function CasesView({ packet, primaryMetric, onOpenRun }: { packet: BenchmarkStudyDashboard; primaryMetric: string; onOpenRun: (runId: string, caseId: string) => void }) {
return (
<div className="benchmark-table-shell benchmark-wide-table">
<table>
Expand All @@ -208,7 +209,7 @@ function CasesView({ packet, primaryMetric, onOpenRun }: { packet: BenchmarkStud
{item.arms.map((cell) => (
<td key={cell.arm_id}>
{cell.selected_run_id && cell.score_countable ? (
<button className="benchmark-cell-link" onClick={() => onOpenRun(cell.selected_run_id!)} type="button">
<button className="benchmark-cell-link" onClick={() => onOpenRun(cell.selected_run_id!, item.case_id)} type="button">
<span>{primaryMetric}: {metricValue(cell.metrics[primaryMetric])}</span>
{packet.design.metric_catalog.filter((metric) => metric.metric_name !== primaryMetric).map((metric) => (
<small className="benchmark-cell-metric" key={metric.metric_name}>{metric.metric_name}: {metricValue(cell.metrics[metric.metric_name])}</small>
Expand Down Expand Up @@ -261,23 +262,23 @@ function RunDetail({ run, packet }: { run: BenchmarkStudyRun; packet: BenchmarkS
);
}

function RunsView({ packet, selectedRunId, onSelectRun }: { packet: BenchmarkStudyDashboard; selectedRunId: string; onSelectRun: (runId: string) => void }) {
const selected = packet.runs.find((run) => run.run_id === selectedRunId) ?? packet.runs[0];
function RunsView({ packet, selectedRunId, selectedCaseId, onSelectRun }: { packet: BenchmarkStudyDashboard; selectedRunId: string; selectedCaseId: string; onSelectRun: (runId: string, caseId: string) => void }) {
const selected = selectBenchmarkStudyRun(packet.runs, selectedRunId, selectedCaseId);
return (
<div className="benchmark-runs-layout">
<div className="benchmark-table-shell">
<table>
<thead><tr><th>Run</th><th>Case</th><th>Arm</th><th>Status</th><th>Countability</th></tr></thead>
<tbody>{packet.runs.map((run) => (
<tr aria-selected={run.run_id === selected?.run_id} key={run.run_id}>
<td><button className="benchmark-run-link" onClick={() => onSelectRun(run.run_id)} type="button">{run.run_id}</button></td>
<tr aria-selected={run === selected} key={JSON.stringify([run.case_id, run.run_id])}>
<td><button className="benchmark-run-link" onClick={() => onSelectRun(run.run_id, run.case_id)} type="button">{run.run_id}</button></td>
<td>{run.case_id}</td><td>{run.arm_id}</td><td>{run.status}</td>
<td>{run.countability.score_countable ? "Score-countable" : "Diagnostic only"}</td>
</tr>
))}</tbody>
</table>
</div>
{selected && <RunDetail packet={packet} run={selected} />}
{selected ? <RunDetail packet={packet} run={selected} /> : <p role="status">Select a run from the table to view its details.</p>}
</div>
);
}
Expand Down Expand Up @@ -323,7 +324,7 @@ export function BenchmarkStudyPage() {
if (!packet) return <main aria-busy="true" className="benchmark-page benchmark-loading"><Activity aria-hidden="true" /><h1>Reading benchmark study</h1><p>Validating the public-safe dashboard packet…</p></main>;

const primaryMetric = packet.design.metric_catalog.find((metric) => metric.role === "primary")?.metric_name ?? "primary";
const setView = (view: BenchmarkStudyView, runId = search.runId) => navigate({ search: (current) => ({ ...current, view, runId }) });
const setView = (view: BenchmarkStudyView, runId = search.runId, caseId = search.caseId) => navigate({ search: (current) => ({ ...current, view, runId, caseId }) });
const sourcePath = new URL(source.url).pathname;

return (
Expand Down Expand Up @@ -361,8 +362,8 @@ export function BenchmarkStudyPage() {
<div className="benchmark-content">
{search.view === "campaign" && <CampaignView packet={packet} primaryMetric={primaryMetric} />}
{search.view === "arms" && <ArmsView packet={packet} />}
{search.view === "cases" && <CasesView onOpenRun={(runId) => setView("runs", runId)} packet={packet} primaryMetric={primaryMetric} />}
{search.view === "runs" && <RunsView onSelectRun={(runId) => setView("runs", runId)} packet={packet} selectedRunId={search.runId} />}
{search.view === "cases" && <CasesView onOpenRun={(runId, caseId) => setView("runs", runId, caseId)} packet={packet} primaryMetric={primaryMetric} />}
{search.view === "runs" && <RunsView onSelectRun={(runId, caseId) => setView("runs", runId, caseId)} packet={packet} selectedRunId={search.runId} selectedCaseId={search.caseId} />}
</div>

<footer className="benchmark-footer">
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
54 changes: 50 additions & 4 deletions examples/dashboard-benchmark-study-browser-smoke.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import { spawn } from "node:child_process";
import { createRequire } from "node:module";
import { existsSync } from "node:fs";
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
Expand Down Expand Up @@ -97,13 +97,58 @@ async function assertCommonSurface(page, label) {
await assertNoHorizontalOverflow(page, label);
}

async function assertCaseScopedRunSelection(page, url) {
const fixture = JSON.parse(readFileSync(resolve(dashboardDir, "public/benchmark-study.example.json"), "utf8"));
const localIds = new Map(fixture.runs.map((run) => [run.run_id, run.arm_id]));
const packet = JSON.parse(JSON.stringify(fixture, (_key, value) => localIds.get(value) ?? value));
await page.route("**/case-local-runs.json", (route) => route.fulfill({ json: packet }));
const target = new URL(url);
target.search = new URLSearchParams({ dashboardUrl: "/case-local-runs.json", view: "cases" }).toString();
await page.goto(target.toString(), { waitUntil: "networkidle" });
const secondCase = page.getByRole("row").filter({ has: page.getByText("case-state-transition", { exact: true }) });
await secondCase.getByRole("button", { name: /Countable/ }).first().click();
const detail = page.getByRole("complementary", { name: "Run detail for goal_plain" });
await detail.waitFor();
if (process.env.LOOPX_BENCHMARK_STUDY_SCREENSHOT_DIR) {
mkdirSync(process.env.LOOPX_BENCHMARK_STUDY_SCREENSHOT_DIR, { recursive: true });
await page.screenshot({ path: resolve(process.env.LOOPX_BENCHMARK_STUDY_SCREENSHOT_DIR, "case-local-run.png"), fullPage: true });
}
if (!(await detail.innerText()).includes("case-state-transition · goal_plain")) {
throw new Error("case-cell navigation opened another case's reused run id");
}
if (new URL(page.url()).searchParams.get("caseId") !== "case-state-transition") {
throw new Error("run link lost its case identity");
}
await page.reload({ waitUntil: "networkidle" });
await detail.getByText("case-state-transition · goal_plain", { exact: true }).waitFor();
const firstCase = page.getByRole("row").filter({ has: page.getByRole("cell", { name: "case-api-contract", exact: true }) });
await firstCase.getByRole("button", { name: "goal_plain", exact: true }).click();
await detail.getByText("case-api-contract · goal_plain", { exact: true }).waitFor();

for (const caseId of ["", "missing-case"]) {
target.search = new URLSearchParams({ dashboardUrl: "/case-local-runs.json", view: "runs", runId: "goal_plain", caseId }).toString();
await page.goto(target.toString(), { waitUntil: "networkidle" });
await page.getByText("Select a run from the table to view its details.", { exact: true }).waitFor();
if (!caseId && process.env.LOOPX_BENCHMARK_STUDY_SCREENSHOT_DIR) {
await page.screenshot({ path: resolve(process.env.LOOPX_BENCHMARK_STUDY_SCREENSHOT_DIR, "ambiguous-run-link.png"), fullPage: true });
}
if (await page.getByRole("complementary", { name: /Run detail for/ }).count()) {
throw new Error("ambiguous or stale run link silently selected a different run");
}
}
// A unique legacy runId bookmark remains valid without a caseId parameter.
target.search = new URLSearchParams({ view: "runs", runId: "run-goal-state" }).toString();
await page.goto(target.toString(), { waitUntil: "networkidle" });
await page.getByRole("complementary").getByText("case-state-transition · goal_plain", { exact: true }).waitFor();
}

async function main() {
const { chromium } = loadPlaywright();
const server = startDashboardServer();
const server = process.env.LOOPX_BENCHMARK_STUDY_URL ? null : startDashboardServer();
let browser;
const pageErrors = [];
try {
const url = `http://127.0.0.1:${port}/benchmarks/study`;
const url = process.env.LOOPX_BENCHMARK_STUDY_URL ?? `http://127.0.0.1:${port}/benchmarks/study`;
await waitForDashboard(url);
browser = await launchBrowser(chromium);

Expand Down Expand Up @@ -149,11 +194,12 @@ async function main() {
await rejected.goto(`${url}?dashboardUrl=${encodeURIComponent("https://example.com/study.json")}`, { waitUntil: "networkidle" });
await rejected.getByText("Benchmark dashboard source must use same-origin local readback", { exact: true }).waitFor();

await assertCaseScopedRunSelection(desktop, url);
if (pageErrors.length) throw new Error(`browser errors: ${pageErrors.join(" | ")}`);
console.log("benchmark study browser smoke passed");
} finally {
await browser?.close();
server.kill("SIGTERM");
server?.kill("SIGTERM");
}
}

Expand Down
5 changes: 5 additions & 0 deletions loopx/capabilities/benchmark_toolkit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,11 @@ silently pooled. Unknown fields and path-like references fail closed.
Every non-baseline row names an exact `comparison_anchor_run_id`. Standard control
or treatment rows anchor to a baseline. Explore rows use `diagnostic_only` claim
scope and may anchor to the baseline or fixed standard arm they are examining.
Run identity is `(benchmark_id, study_id, case_id, run_id)`; an anchor inherits
the candidate's benchmark, study, and case. Run ids may be reused across those
scopes. Duplicate full identities must be reconciled before board readback.
Study provenance and case insights bind to this same identity. Dashboard run
links retain the case; a legacy run-id-only link resolves only when unambiguous.
Matched comparisons require compatible benchmark, study, case, model, primary
metric, comparison protocol, score countability, and treatment fidelity. Exact
protocol revisions remain visible as a warning even when a declared comparison
Expand Down
23 changes: 14 additions & 9 deletions loopx/capabilities/benchmark_toolkit/experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from ...domain_state import default_domain_state_file_path, upsert_domain_state_jsonl
from .experiment_identity import (
ARM_ROLES,
experiment_run_key,
experiment_token_text as _token,
)
from .factorial_contrast import (
Expand Down Expand Up @@ -476,13 +477,7 @@ def upsert_benchmark_experiment_board_row(
def _benchmark_experiment_board_row_key_tuple(
payload: Mapping[str, Any],
) -> tuple[str, str, str, str]:
key = benchmark_experiment_board_row_key(payload)
return (
key["benchmark_id"],
key["study_id"],
key["case_id"],
key["run_id"],
)
return experiment_run_key(benchmark_experiment_board_row_key(payload))


def _benchmark_experiment_board_observed_at(payload: Mapping[str, Any]) -> datetime:
Expand Down Expand Up @@ -730,9 +725,19 @@ def build_benchmark_experiment_board(
else []
)

by_run_id = {row["run_id"]: row for row in normalized}
by_run_key = {}
for row in normalized:
key = experiment_run_key(row)
if key in by_run_key:
raise ValueError("duplicate experiment-board run identity")
by_run_key[key] = row
comparisons = [
_build_comparison(row, by_run_id.get(row.get("comparison_anchor_run_id")))
_build_comparison(
row,
by_run_key.get(
experiment_run_key(row, run_id=row["comparison_anchor_run_id"])
),
)
for row in normalized
if row["arm_role"] != "baseline"
]
Expand Down
18 changes: 17 additions & 1 deletion loopx/capabilities/benchmark_toolkit/experiment_identity.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""One owner for the two vocabularies a benchmark experiment identity uses.
"""Shared benchmark experiment vocabulary and scoped run identity.

An arm role and a public-safe token are validated at the package boundary, at
the study projection, at the concurrency envelope and at the CLI that admits a
Expand All @@ -13,11 +13,15 @@
``experiment_token_text`` is the reject path that goes with the token shape.
Five modules carried a byte-identical private ``_token`` helper, so the shape and
its rejection message travelled together in five copies.

``experiment_run_key`` preserves the ledger's benchmark/study/case/run identity
when normalized rows, anchors, provenance and insights are joined.
"""

from __future__ import annotations

import re
from collections.abc import Mapping
from typing import Any

ARM_ROLES = frozenset({"baseline", "control", "treatment", "explore"})
Expand All @@ -26,6 +30,18 @@
EXPERIMENT_TOKEN_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,127}$")


def experiment_run_key(
row: Mapping[str, Any], *, run_id: str | None = None
) -> tuple[str, str, str, str]:
"""Identify a normalized board row or insight; anchors inherit the same scope."""
return (
row["benchmark_id"],
row["study_id"],
row["case_id"],
row["run_id"] if run_id is None else run_id,
)


def experiment_token_text(value: Any, *, field: str) -> str:
text = str(value or "").strip()
if not EXPERIMENT_TOKEN_PATTERN.fullmatch(text):
Expand Down
Loading
Loading