Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 111 additions & 0 deletions src/__tests__/unit/hooks/useEvalRunHistory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -759,3 +759,114 @@ describe("useEvalRunHistory — attemptRows", () => {
expect(keys).toEqual(["output-base", "output-r1", "eval-live"]);
});
});

// ─── Graph-stamped Stakwork project id (unique_source_id) ────────────────────

describe("useEvalRunHistory — unique_source_id join", () => {
const EVAL_SET_REF = "ref-usid-001";
const TASK_SLUG = "antitrust/task-1";

afterEach(() => {
vi.restoreAllMocks();
mockBuildHillClimbSeries.mockReset();
});

/** Concept-pipeline shape: no identity fields, unique_source_id on the output node. */
function conceptGraph(uniqueSourceId?: string | number) {
return {
nodes: [
makeTriggerNode("trigger-c1", false),
{
ref_id: "output-c1",
node_type: "EvalTriggerOutput",
date_added_to_graph: "1720000000",
properties: {
result: "partial",
score: 70 / 71,
n_passed: 70,
n_total: 71,
...(uniqueSourceId != null ? { unique_source_id: uniqueSourceId } : {}),
},
},
],
edges: [
{ source: EVAL_SET_REF, target: "trigger-c1", edge_type: "HAS_TRIGGER" },
{ source: "trigger-c1", target: "output-c1", edge_type: "HAS_OUTPUT" },
],
};
}

function renderUsid(routes: Record<string, unknown>) {
mockBuildHillClimbSeries.mockReturnValue([]);
mockFetch(routes);
return renderHook(() => useEvalRunHistory({ refId: EVAL_SET_REF, slug: TASK_SLUG }));
}

it("does NOT wear a run's status via the graph-stamped project id", async () => {
// The stamp is the CHILD re-runner's project id; hive run rows carry the
// PARENT orchestrator's. A projectId match is coincidence, never identity —
// so only the Stakwork link may use the stamp.
const graph = conceptGraph("152583201");
const { result } = renderUsid({
"fix-chain": makeFixChainResponse(graph.nodes, graph.edges),
"type=LEGAL_BENCHMARK_RUNNER": { runs: [] },
"type=LEGAL_BENCHMARK_EVAL": { runs: [] },
"type=LEGAL_BENCHMARK_RECURSION": {
runs: [
{
id: "rec-run-1",
projectId: 152583201,
status: "COMPLETED",
createdAt: "2026-08-19T10:00:00.000Z",
hasReport: true,
result: JSON.stringify({ taskSlug: TASK_SLUG, recursionId: "r-1" }),
},
],
},
});
await waitFor(() => expect(result.current.isLoading).toBe(false), { timeout: 5000 });

const row = result.current.attemptRows.find((r) => r.key === "output-c1")!;
expect(row.status).toBeNull();
expect(row.runId).toBeNull();
expect(row.hasReport).toBe(false);
// …but the lineage link survives
expect(row.projectId).toBe(152583201);
});

it("keeps the Stakwork link from the graph when no run row exists at all", async () => {
const graph = conceptGraph(152583201);
const { result } = renderUsid({
"fix-chain": makeFixChainResponse(graph.nodes, graph.edges),
"type=LEGAL_BENCHMARK_RUNNER": { runs: [] },
"type=LEGAL_BENCHMARK_EVAL": { runs: [] },
"type=LEGAL_BENCHMARK_RECURSION": { runs: [] },
});
await waitFor(() => expect(result.current.isLoading).toBe(false), { timeout: 5000 });

const row = result.current.attemptRows.find((r) => r.key === "output-c1")!;
expect(row.status).toBeNull();
expect(row.runId).toBeNull();
expect(row.hasReport).toBe(false);
// The graph-stamped project id still powers the super-admin link
expect(row.projectId).toBe(152583201);
});

it("reads unique_source_id from edge properties when nodes lack it", async () => {
const graph = conceptGraph(undefined);
graph.edges[1] = {
...graph.edges[1],
properties: { unique_source_id: "99887766" },
} as (typeof graph.edges)[number] & { properties: Record<string, unknown> };
const { result } = renderUsid({
"fix-chain": makeFixChainResponse(graph.nodes, graph.edges),
"type=LEGAL_BENCHMARK_RUNNER": { runs: [] },
"type=LEGAL_BENCHMARK_EVAL": { runs: [] },
"type=LEGAL_BENCHMARK_RECURSION": { runs: [] },
});
await waitFor(() => expect(result.current.isLoading).toBe(false), { timeout: 5000 });

const row = result.current.attemptRows.find((r) => r.key === "output-c1")!;
expect(row.projectId).toBe(99887766);
});
});
58 changes: 57 additions & 1 deletion src/hooks/useEvalRunHistory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ export interface AttemptRailRow {

const NON_TERMINAL_STATUSES = new Set(["PENDING", "IN_PROGRESS"]);

/**
* Stakwork project id from graph properties — same precedence as
* proposed-fixes/route.ts: `unique_source_id` (written by jarvis-backend at
* Stakwork dispatch) wins over legacy `project_id`.
*/
function projectIdFromProps(props: Record<string, unknown> | undefined): number | null {
if (!props) return null;
for (const key of ["unique_source_id", "project_id"]) {
const raw = props[key];
if (raw == null) continue;
const n = Number(raw);
if (Number.isFinite(n) && n > 0) return n;
}
return null;
}

/** Convert a Jarvis epoch-seconds string to ISO; null when unparseable. */
function graphEpochToIso(raw: string | undefined): string | null {
if (!raw) return null;
Expand Down Expand Up @@ -481,6 +497,32 @@ export function useEvalRunHistory(input: UseEvalRunHistoryInput): UseEvalRunHist
for (const o of t.outputs ?? []) triggerByOutputRef.set(o.ref_id, t);
}

// Graph-side Stakwork project id per attempt — the CHILD re-runner.
// Topology per recursion iteration: the cron's StakworkRun row stores
// the PARENT orchestrator's project id; that parent spawns a child
// workflow (new project id) which actually re-runs the eval and stamps
// `unique_source_id` = its own (child) id onto the graph it writes.
// So the stamp is true per-attempt identity — perfect for the Stakwork
// link — but it can never join hive's run rows, which only ever carry
// the parent's id. Status/report therefore join exclusively on
// result.evalTriggerRef.
// Precedence: output node → owning trigger node → any edge touching either.
const rawNodeByRef = new Map(fixChain.nodes.map((n) => [n.ref_id, n]));
const edgeProjectIdByRef = new Map<string, number>();
for (const e of fixChain.edges) {
const pid = projectIdFromProps(e.properties);
if (pid == null) continue;
if (!edgeProjectIdByRef.has(e.target)) edgeProjectIdByRef.set(e.target, pid);
if (!edgeProjectIdByRef.has(e.source)) edgeProjectIdByRef.set(e.source, pid);
}
const graphProjectIdFor = (outputRefId: string, triggerRefId: string | null): number | null =>
projectIdFromProps(rawNodeByRef.get(outputRefId)?.properties) ??
(triggerRefId ? projectIdFromProps(rawNodeByRef.get(triggerRefId)?.properties) : null) ??
edgeProjectIdByRef.get(outputRefId) ??
(triggerRefId ? edgeProjectIdByRef.get(triggerRefId) : null) ??
null;


const triggerJoinableRuns = [...runRows, ...evalRunRows];
const claimedRunIds = new Set<string>();

Expand Down Expand Up @@ -523,15 +565,29 @@ export function useEvalRunHistory(input: UseEvalRunHistoryInput): UseEvalRunHist
const chartedRows: AttemptRailRow[] = finalAttempts.map((attempt, index) => {
const trigger = triggerByOutputRef.get(attempt.ref_id) ?? null;
if (trigger) chartedTriggerRefs.add(trigger.ref_id);
const graphProjectId = graphProjectIdFor(attempt.ref_id, trigger?.ref_id ?? null);

// Status joins on result.evalTriggerRef only — hive rows carry the
// parent orchestrator's project id, the graph stamp carries the
// child re-runner's; a projectId join can never legitimately match
// (see topology note above).
const statusRun = trigger ? pickStatusRun(runsForTrigger(trigger.ref_id)) : null;

const passed = attempt.actualPassed ?? attempt.n_passed;
const total = attempt.n_total;
return rowFromRun(attempt.ref_id, statusRun, {
const row = rowFromRun(attempt.ref_id, statusRun, {
label: attempt.label ?? null,
attemptIndex: index,
score: passed != null && total != null ? { passed, total } : null,
graphTime: graphEpochToIso(attempt.date_added_to_graph),
});
// No hive run row exists for the child re-runner — but the graph
// stamp IS its project id, so the super-admin link opens the exact
// Stakwork execution that produced this attempt.
if (row.projectId == null && graphProjectId != null) {
row.projectId = graphProjectId;
}
return row;
});

// Identity triggers with no charted output yet — a dispatched run whose
Expand Down
2 changes: 2 additions & 0 deletions src/lib/harvey-lab/fix-chain-walker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ interface JarvisExpandResponse {
target: string;
ref_id?: string;
edge_type: string;
properties?: Record<string, unknown>;
}>;
}

Expand Down Expand Up @@ -229,6 +230,7 @@ async function fetchNodeForEdgeTypes(
source: e.source,
target: e.target,
edge_type: e.edge_type,
...(e.properties ? { properties: e.properties } : {}),
});
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/lib/harvey-lab/hill-climb-series.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ export interface SubgraphEdge {
source: string;
target: string;
edge_type: string;
/** Edge-level properties (e.g. unique_source_id written at Stakwork dispatch) */
properties?: Record<string, unknown>;
}

export interface SubgraphNode {
Expand Down
Loading