diff --git a/src/__tests__/integration/api/gateway-evals.test.ts b/src/__tests__/integration/api/gateway-evals.test.ts index 0ebb9d764a..d91235ac14 100644 --- a/src/__tests__/integration/api/gateway-evals.test.ts +++ b/src/__tests__/integration/api/gateway-evals.test.ts @@ -42,6 +42,11 @@ vi.mock("@/services/bifrost/orchestrator", () => ({ BIFROST_AGENT_NAMES: ["repo-agent", "canvas-agent"], })); +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn().mockResolvedValue({ allowed: true }), + getClientIp: vi.fn().mockReturnValue("127.0.0.1"), +})); + import { getBifrostForLLM } from "@/services/bifrost/orchestrator"; // ── Constants ───────────────────────────────────────────────────────────────── @@ -396,10 +401,13 @@ describe("POST /api/gateway/evals/:setId/requirements", () => { test("returns ref_id on successful create", async () => { const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-req-post"); - // First fetch: sibling count + // First fetch: IDOR ownership check + sibling count (HAS_REQUIREMENT expand) mockFetch.mockResolvedValueOnce({ ok: true, - json: async () => ({ nodes: [], edges: [] }), + json: async () => ({ + nodes: [{ ref_id: SET_ID, node_type: "EvalSet", properties: {} }], + edges: [], + }), } as Response); // Second fetch: addNode mockFetch.mockResolvedValueOnce({ @@ -463,6 +471,18 @@ describe("PUT /api/gateway/evals/:setId/requirements/:reqId", () => { test("returns 204 on successful update", async () => { const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-req-put"); + // First fetch: IDOR ownership check (HAS_REQUIREMENT edge expand) + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + nodes: [ + { ref_id: SET_ID, node_type: "EvalSet", properties: {} }, + { ref_id: REQ_ID, node_type: "EvalRequirement", properties: {} }, + ], + edges: [], + }), + } as Response); + // Second fetch: updateNode mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ status: "success" }), @@ -499,6 +519,18 @@ describe("DELETE /api/gateway/evals/:setId/requirements/:reqId", () => { test("returns 204 on successful delete", async () => { const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-req-del"); + // First fetch: IDOR ownership check (HAS_REQUIREMENT expand) + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + nodes: [ + { ref_id: SET_ID, node_type: "EvalSet", properties: {} }, + { ref_id: REQ_ID, node_type: "EvalRequirement", properties: {} }, + ], + edges: [], + }), + } as Response); + // Second fetch: deleteNode mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ status: "success" }), @@ -873,3 +905,330 @@ describe("POST /api/gateway/evals/:setId/requirements/:reqId/run", () => { void ctx; }); }); + +// ── `contested` field — gateway POST and PUT ────────────────────────────────── + +describe("contested field — gateway POST /api/gateway/evals/:setId/requirements", () => { + function mockSiblingsOk() { + // Returns the set node (required for fail-closed ownership check) with no + // existing requirements, so siblingCount = 0. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + nodes: [{ ref_id: SET_ID, node_type: "EvalSet", properties: {} }], + edges: [], + }), + } as Response); + } + function mockAddNodeOk(refId = "new-req-ref") { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ status: "success", data: { ref_id: refId } }), + } as Response); + } + function mockAddEdgeOk() { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ status: "success", edges: [{}] }), + } as Response); + } + + test("omitting contested defaults to false in node_data", async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-contested-default"); + mockSiblingsOk(); + mockAddNodeOk(); + mockAddEdgeOk(); + + const req = makeRequest( + "POST", + `http://localhost/api/gateway/evals/${SET_ID}/requirements`, + RAW_KEY_1, + { name: "Req without contested" }, + ); + const res = await postRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID }), + }); + expect(res.status).toBe(201); + + // The addNode call (second fetch) should have contested: false in node_data + const addNodeCall = mockFetch.mock.calls[1]; + if (addNodeCall) { + const body = JSON.parse(addNodeCall[1].body as string); + expect(body.node_data?.contested).toBe(false); + } + void ctx; + }); + + test("contested: true is passed as boolean true in node_data", async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-contested-true"); + mockSiblingsOk(); + mockAddNodeOk(); + mockAddEdgeOk(); + + const req = makeRequest( + "POST", + `http://localhost/api/gateway/evals/${SET_ID}/requirements`, + RAW_KEY_1, + { name: "Contested req", contested: true }, + ); + const res = await postRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID }), + }); + expect(res.status).toBe(201); + + const addNodeCall = mockFetch.mock.calls[1]; + if (addNodeCall) { + const body = JSON.parse(addNodeCall[1].body as string); + expect(body.node_data?.contested).toBe(true); + } + void ctx; + }); + + test('contested: "true" (string) is coerced to boolean true', async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-contested-str-true"); + mockSiblingsOk(); + mockAddNodeOk(); + mockAddEdgeOk(); + + const req = makeRequest( + "POST", + `http://localhost/api/gateway/evals/${SET_ID}/requirements`, + RAW_KEY_1, + { name: "Req with string contested", contested: "true" }, + ); + const res = await postRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID }), + }); + expect(res.status).toBe(201); + + const addNodeCall = mockFetch.mock.calls[1]; + if (addNodeCall) { + const body = JSON.parse(addNodeCall[1].body as string); + expect(body.node_data?.contested).toBe(true); + } + void ctx; + }); + + test('contested: "maybe" returns 400 with field message', async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-contested-bad"); + + // Rate limit fetch + sibling fetch may not happen if validation fails first + const req = makeRequest( + "POST", + `http://localhost/api/gateway/evals/${SET_ID}/requirements`, + RAW_KEY_1, + { name: "Bad contested", contested: "maybe" }, + ); + const res = await postRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID }), + }); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/contested/i); + void ctx; + }); +}); + +describe("contested field — gateway PUT /api/gateway/evals/:setId/requirements/:reqId", () => { + function mockOwnershipOk(reqId = REQ_ID, contested?: boolean) { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + nodes: [ + { + ref_id: SET_ID, + node_type: "EvalSet", + properties: {}, + }, + { + ref_id: reqId, + node_type: "EvalRequirement", + properties: contested !== undefined ? { contested } : {}, + }, + ], + edges: [], + }), + } as Response); + } + function mockUpdateNodeOk() { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ status: "success" }), + } as Response); + } + + test("omitting contested means key is absent from updateNode payload (tri-state)", async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-put-omit"); + mockOwnershipOk(REQ_ID, true); // stored as true + mockUpdateNodeOk(); + + const req = makeRequest( + "PUT", + `http://localhost/api/gateway/evals/${SET_ID}/requirements/${REQ_ID}`, + RAW_KEY_1, + { name: "Name only edit" }, // no contested key + ); + const res = await putRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID, reqId: REQ_ID }), + }); + expect(res.status).toBe(204); + + // The updateNode call (second fetch) must NOT contain contested key + const updateCall = mockFetch.mock.calls[1]; + if (updateCall) { + const body = JSON.parse(updateCall[1].body as string); + expect(Object.keys(body.node_data ?? {})).not.toContain("contested"); + } + void ctx; + }); + + test("contested: false is sent explicitly when provided", async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-put-false"); + mockOwnershipOk(REQ_ID, true); + mockUpdateNodeOk(); + + const req = makeRequest( + "PUT", + `http://localhost/api/gateway/evals/${SET_ID}/requirements/${REQ_ID}`, + RAW_KEY_1, + { name: "Clear contested", contested: false }, + ); + const res = await putRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID, reqId: REQ_ID }), + }); + expect(res.status).toBe(204); + + const updateCall = mockFetch.mock.calls[1]; + if (updateCall) { + const body = JSON.parse(updateCall[1].body as string); + expect(body.node_data?.contested).toBe(false); + } + void ctx; + }); + + test('contested: "TRUE" (string) is coerced to boolean true', async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-put-str"); + mockOwnershipOk(); + mockUpdateNodeOk(); + + const req = makeRequest( + "PUT", + `http://localhost/api/gateway/evals/${SET_ID}/requirements/${REQ_ID}`, + RAW_KEY_1, + { name: "String contested", contested: "TRUE" }, + ); + const res = await putRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID, reqId: REQ_ID }), + }); + expect(res.status).toBe(204); + + const updateCall = mockFetch.mock.calls[1]; + if (updateCall) { + const body = JSON.parse(updateCall[1].body as string); + expect(body.node_data?.contested).toBe(true); + } + void ctx; + }); + + test('contested: "invalid" returns 400 with field message', async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-put-bad"); + + const req = makeRequest( + "PUT", + `http://localhost/api/gateway/evals/${SET_ID}/requirements/${REQ_ID}`, + RAW_KEY_1, + { name: "Bad contested", contested: "invalid" }, + ); + const res = await putRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID, reqId: REQ_ID }), + }); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/contested/i); + void ctx; + }); + + test("reqId not in eval set returns 404 before any write", async () => { + const ctx = await createTestContext(RAW_KEY_1, SWARM_NAME_1 + "-put-idor"); + // Ownership check: setId exists but reqId is NOT among its requirements + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + nodes: [ + { ref_id: SET_ID, node_type: "EvalSet", properties: {} }, + // A different req — not REQ_ID + { + ref_id: "other-req-id", + node_type: "EvalRequirement", + properties: {}, + }, + ], + edges: [], + }), + } as Response); + + const req = makeRequest( + "PUT", + `http://localhost/api/gateway/evals/${SET_ID}/requirements/${REQ_ID}`, + RAW_KEY_1, + { name: "IDOR attempt" }, + ); + const res = await putRequirement(req as any, { + params: Promise.resolve({ setId: SET_ID, reqId: REQ_ID }), + }); + expect(res.status).toBe(404); + + // updateNode (second fetch) must NOT have been called + const updateCalls = mockFetch.mock.calls.filter((call) => + String(call[0]).includes("/node?") || String(call[1]?.method ?? "GET") === "PUT", + ); + expect(updateCalls.length).toBe(0); + void ctx; + }); + + test("mock PUT 404s when USE_MOCKS is not 'true'", async () => { + const prev = process.env.USE_MOCKS; + delete process.env.USE_MOCKS; + try { + const { PUT: mockPut } = await import( + "@/app/api/mock/evals/[evalSetId]/requirements/[reqId]/route" + ); + const req = new Request( + `http://localhost/api/mock/evals/eval-set-1/requirements/req-1-1`, + { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + }, + ); + const res = await mockPut(req as any, { + params: Promise.resolve({ evalSetId: "eval-set-1", reqId: "req-1-1" }), + }); + expect(res.status).toBe(404); + } finally { + if (prev !== undefined) process.env.USE_MOCKS = prev; + } + }); + + test("mock POST 404s when USE_MOCKS is not 'true'", async () => { + const prev = process.env.USE_MOCKS; + delete process.env.USE_MOCKS; + try { + const { POST: mockPost } = await import( + "@/app/api/mock/evals/[evalSetId]/requirements/route" + ); + const req = new Request( + `http://localhost/api/mock/evals/eval-set-1/requirements`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + }, + ); + const res = await mockPost(req as any); + expect(res.status).toBe(404); + } finally { + if (prev !== undefined) process.env.USE_MOCKS = prev; + } + }); +}); diff --git a/src/__tests__/unit/components/evals/EditRequirementModal.test.tsx b/src/__tests__/unit/components/evals/EditRequirementModal.test.tsx index e77ef34307..5e49892ae5 100644 --- a/src/__tests__/unit/components/evals/EditRequirementModal.test.tsx +++ b/src/__tests__/unit/components/evals/EditRequirementModal.test.tsx @@ -172,15 +172,35 @@ describe("EditRequirementModal", () => { }); }); - it("shows error toast when request fails", async () => { - global.fetch = vi.fn().mockResolvedValue({ ok: false }) as any; + it("shows error toast when request fails (generic)", async () => { + // When the server returns ok:false with no error body, the component + // surfaces the fallback message from the thrown Error. + global.fetch = vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }) as any; render(); await userEvent.click(screen.getByRole("button", { name: "Save" })); await waitFor(() => { - expect(toast.error).toHaveBeenCalledWith("Failed to update requirement"); + expect(toast.error).toHaveBeenCalledWith("Request failed"); + }); + expect(defaultProps.onUpdated).not.toHaveBeenCalled(); + }); + + it("shows server error message in toast when request fails with error body", async () => { + // When the server returns ok:false with a specific error message, the + // component surfaces that message rather than a generic fallback. + global.fetch = vi.fn().mockResolvedValue({ + ok: false, + json: async () => ({ error: "Insufficient permissions to set contested" }), + }) as any; + + render(); + + await userEvent.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(toast.error).toHaveBeenCalledWith("Insufficient permissions to set contested"); }); expect(defaultProps.onUpdated).not.toHaveBeenCalled(); }); diff --git a/src/__tests__/unit/lib/coerce-contested.test.ts b/src/__tests__/unit/lib/coerce-contested.test.ts new file mode 100644 index 0000000000..2227956ea2 --- /dev/null +++ b/src/__tests__/unit/lib/coerce-contested.test.ts @@ -0,0 +1,53 @@ +/** + * Unit tests for coerceContested() — the boolean coercion helper for the + * `contested` field on EvalRequirement write routes. + */ +import { describe, test, expect } from "vitest"; +import { coerceContested } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route"; + +describe("coerceContested", () => { + // ── Truthy values → true ───────────────────────────────────────────────── + test.each([ + ["boolean true", true], + ["number 1", 1], + ['string "true"', "true"], + ['string "TRUE"', "TRUE"], + ['string "True"', "True"], + ])("returns true for %s", (_label, value) => { + expect(coerceContested(value)).toBe(true); + }); + + // ── Falsy values → false ───────────────────────────────────────────────── + test.each([ + ["boolean false", false], + ["number 0", 0], + ['string "false"', "false"], + ['string "FALSE"', "FALSE"], + ['string "False"', "False"], + ])("returns false for %s", (_label, value) => { + expect(coerceContested(value)).toBe(false); + }); + + // ── Absent values → undefined ───────────────────────────────────────────── + test("returns undefined for undefined", () => { + expect(coerceContested(undefined)).toBeUndefined(); + }); + + test("returns undefined for null", () => { + expect(coerceContested(null)).toBeUndefined(); + }); + + // ── Un-coercible values → null (caller should 400) ──────────────────────── + test.each([ + ['string "maybe"', "maybe"], + ['string "1"', "1"], + ['string "0"', "0"], + ["empty string", ""], + ["object {}", {}], + ["array []", []], + ["number 2", 2], + ["number -1", -1], + ])("returns null for un-coercible value %s", (_label, value) => { + expect(coerceContested(value)).toBeNull(); + }); +}); diff --git a/src/app/api/gateway/evals/[setId]/requirements/[reqId]/route.ts b/src/app/api/gateway/evals/[setId]/requirements/[reqId]/route.ts index bbe3bc21fa..f3db2260cc 100644 --- a/src/app/api/gateway/evals/[setId]/requirements/[reqId]/route.ts +++ b/src/app/api/gateway/evals/[setId]/requirements/[reqId]/route.ts @@ -8,9 +8,73 @@ import { NextRequest, NextResponse } from "next/server"; import { resolveGatewayAuth } from "@/lib/evals/gateway-auth"; import { updateNode, deleteNode } from "@/services/swarm/api/nodes"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { coerceContested } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route"; +import type { JarvisNode } from "@/types/jarvis"; type RouteParams = { params: Promise<{ setId: string; reqId: string }> }; +/** + * Fetch the eval set from Jarvis and verify: + * 1. Fetch succeeded — fail closed on any error. + * 2. The set node exists in the response. + * 3. If the set carries a workspaceId property it matches the caller's workspace. + * 4. reqId appears as an EvalRequirement child of the set. + * + * Returns the req node properties on success, or a NextResponse to short-circuit. + */ +async function verifyOwnership( + jarvisUrl: string, + swarmApiKey: string, + setId: string, + reqId: string, + workspaceId: string, +): Promise<{ contestedBefore?: boolean } | NextResponse> { + const edgeType = encodeURIComponent("['HAS_REQUIREMENT']"); + let res: Response; + try { + res = await fetch( + `${jarvisUrl}/v2/nodes/${encodeURIComponent(setId)}?expand=edges&edge_type=${edgeType}&depth=1`, + { headers: { "x-api-token": swarmApiKey } }, + ); + } catch { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + if (!res.ok) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const data = await res.json(); + const setNode = (data?.nodes ?? []).find((n: JarvisNode) => n.ref_id === setId); + if (!setNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const ownerWorkspaceId = + setNode.properties?.workspace_id ?? setNode.properties?.workspaceId; + if (ownerWorkspaceId && ownerWorkspaceId !== workspaceId) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const reqNodes: JarvisNode[] = (data?.nodes ?? []).filter( + (n: JarvisNode) => + n.ref_id !== setId && + String(n.node_type ?? "").toLowerCase() === "evalrequirement", + ); + const reqNode = reqNodes.find((n) => n.ref_id === reqId); + if (!reqNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const contestedBefore = + reqNode.properties?.contested !== undefined + ? Boolean(reqNode.properties.contested) + : undefined; + + return { contestedBefore }; +} + export async function PUT(request: NextRequest, { params }: RouteParams) { try { const authOrResponse = await resolveGatewayAuth(request); @@ -19,6 +83,18 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { const { workspaceId, keyId, jarvisUrl, swarmApiKey } = authOrResponse; const { setId, reqId } = await params; + // Rate limit — 60 req/min per API key + const rl = await checkRateLimit(`gateway:evals:req:put:${keyId}`, 60, 60); + if (!rl.allowed) { + return NextResponse.json( + { error: "Too many requests" }, + { + status: 429, + headers: rl.retryAfter ? { "Retry-After": String(rl.retryAfter) } : {}, + }, + ); + } + let body: Record; try { body = await request.json(); @@ -26,31 +102,67 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); } - const { name, description, prompt_snippet, desirable_cases, undesirable_cases } = body ?? {}; + const { name, description, prompt_snippet, desirable_cases, undesirable_cases, contested } = + body ?? {}; if (!name || typeof name !== "string" || !name.trim()) { return NextResponse.json({ error: "name is required" }, { status: 400 }); } - console.log(`[Gateway Evals Requirements PUT] workspaceId=${workspaceId}, keyId=${keyId}, setId=${setId}, reqId=${reqId}`); + // Validate `contested` before any Jarvis calls + const contestedCoerced = coerceContested(contested); + if (contestedCoerced === null) { + return NextResponse.json( + { error: "contested must be a boolean (true, false, 1, 0, \"true\", or \"false\")" }, + { status: 400 }, + ); + } + + // IDOR: verify reqId is reachable from setId and the set belongs to this + // workspace — fail closed on any Jarvis error. This also gives us + // contestedBefore for transition logging at no extra Jarvis call. + const ownershipResult = await verifyOwnership( + jarvisUrl, swarmApiKey, setId, reqId, workspaceId, + ); + if (ownershipResult instanceof NextResponse) return ownershipResult; + const { contestedBefore } = ownershipResult; + + // Log the transition — explicit scalars only, never spread authOrResponse + // (holds decrypted swarmApiKey) or body (unvalidated). + console.log( + `[Gateway Evals Requirements PUT] setId=${setId}, reqId=${reqId}, keyId=${keyId}, ` + + `contestedBefore=${contestedBefore}, contestedAfter=${contestedCoerced}`, + ); + + // Build node_data — omit `contested` key when undefined (tri-state write) + const nodeData: Record = { + name: name.trim(), + description, + prompt_snippet: typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, + desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], + undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], + }; + if (contestedCoerced !== undefined) { + nodeData.contested = contestedCoerced; + } const result = await updateNode( { jarvisUrl, apiKey: swarmApiKey }, { ref_id: reqId, node_type: "EvalRequirement", - node_data: { - name: name.trim(), - description, - prompt_snippet: typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, - desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], - undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], - }, + node_data: nodeData, }, ); if (!result.success) { - console.error(`[Gateway Evals Requirements PUT] updateNode failed: ${result.error}`, { workspaceId, reqId }); - return NextResponse.json({ error: result.error ?? "Failed to update requirement" }, { status: 502 }); + console.error(`[Gateway Evals Requirements PUT] updateNode failed: ${result.error}`, { + workspaceId, + reqId, + }); + return NextResponse.json( + { error: result.error ?? "Failed to update requirement" }, + { status: 502 }, + ); } return new NextResponse(null, { status: 204 }); @@ -68,13 +180,40 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) { const { workspaceId, keyId, jarvisUrl, swarmApiKey } = authOrResponse; const { setId, reqId } = await params; - console.log(`[Gateway Evals Requirements DELETE] workspaceId=${workspaceId}, keyId=${keyId}, setId=${setId}, reqId=${reqId}`); + // Rate limit — 60 req/min per API key (shared bucket with PUT) + const rl = await checkRateLimit(`gateway:evals:req:put:${keyId}`, 60, 60); + if (!rl.allowed) { + return NextResponse.json( + { error: "Too many requests" }, + { + status: 429, + headers: rl.retryAfter ? { "Retry-After": String(rl.retryAfter) } : {}, + }, + ); + } + + // IDOR: verify reqId belongs to setId and the set belongs to this workspace + // before deleting — fail closed on any Jarvis error. + const ownershipResult = await verifyOwnership( + jarvisUrl, swarmApiKey, setId, reqId, workspaceId, + ); + if (ownershipResult instanceof NextResponse) return ownershipResult; + + console.log( + `[Gateway Evals Requirements DELETE] workspaceId=${workspaceId}, keyId=${keyId}, setId=${setId}, reqId=${reqId}`, + ); const result = await deleteNode({ jarvisUrl, apiKey: swarmApiKey }, reqId); if (!result.success) { - console.error(`[Gateway Evals Requirements DELETE] deleteNode failed: ${result.error}`, { workspaceId, reqId }); - return NextResponse.json({ error: result.error ?? "Failed to delete requirement" }, { status: 502 }); + console.error( + `[Gateway Evals Requirements DELETE] deleteNode failed: ${result.error}`, + { workspaceId, reqId }, + ); + return NextResponse.json( + { error: result.error ?? "Failed to delete requirement" }, + { status: 502 }, + ); } return new NextResponse(null, { status: 204 }); diff --git a/src/app/api/gateway/evals/[setId]/requirements/route.ts b/src/app/api/gateway/evals/[setId]/requirements/route.ts index e309e44357..11741a9820 100644 --- a/src/app/api/gateway/evals/[setId]/requirements/route.ts +++ b/src/app/api/gateway/evals/[setId]/requirements/route.ts @@ -9,9 +9,15 @@ import { randomUUID } from "crypto"; import { NextRequest, NextResponse } from "next/server"; import { resolveGatewayAuth } from "@/lib/evals/gateway-auth"; import { addNode, addEdge } from "@/services/swarm/api/nodes"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { coerceContested } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route"; +import type { JarvisNode } from "@/types/jarvis"; type RouteParams = { params: Promise<{ setId: string }> }; +/** Strict ref_id pattern — alphanumeric plus hyphens and underscores only */ +const REF_ID_RE = /^[A-Za-z0-9_-]+$/; + export async function POST(request: NextRequest, { params }: RouteParams) { try { const authOrResponse = await resolveGatewayAuth(request); @@ -20,6 +26,18 @@ export async function POST(request: NextRequest, { params }: RouteParams) { const { workspaceId, keyId, jarvisUrl, swarmApiKey } = authOrResponse; const { setId } = await params; + // Rate limit — 60 req/min per API key + const rl = await checkRateLimit(`gateway:evals:req:post:${keyId}`, 60, 60); + if (!rl.allowed) { + return NextResponse.json( + { error: "Too many requests" }, + { + status: 429, + headers: rl.retryAfter ? { "Retry-After": String(rl.retryAfter) } : {}, + }, + ); + } + let body: Record; try { body = await request.json(); @@ -27,47 +45,85 @@ export async function POST(request: NextRequest, { params }: RouteParams) { return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); } - const { name, description, prompt_snippet, desirable_cases, undesirable_cases } = body ?? {}; + const { name, description, prompt_snippet, desirable_cases, undesirable_cases, contested } = + body ?? {}; if (!name || typeof name !== "string" || !name.trim()) { return NextResponse.json({ error: "name is required" }, { status: 400 }); } + // Validate `contested` before any Jarvis calls + const contestedCoerced = coerceContested(contested); + if (contestedCoerced === null) { + return NextResponse.json( + { error: "contested must be a boolean (true, false, 1, 0, \"true\", or \"false\")" }, + { status: 400 }, + ); + } + + // Validate setId pattern before using it in a URL + if (!REF_ID_RE.test(setId)) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + console.log(`[Gateway Evals Requirements POST] workspaceId=${workspaceId}, keyId=${keyId}, setId=${setId}`); const config = { jarvisUrl, apiKey: swarmApiKey }; - // Determine order by fetching current sibling count - let siblingCount = 0; + // IDOR: verify the eval set belongs to this workspace — fail closed. + // Any non-ok response from Jarvis → 404, never a pass-through. + const edgeType = encodeURIComponent("['HAS_REQUIREMENT']"); + const encodedSetId = encodeURIComponent(setId); + let siblingsRes: Response; try { - const edgeType = encodeURIComponent("['HAS_REQUIREMENT']"); - const siblingsRes = await fetch( - `${jarvisUrl}/v2/nodes/${setId}?expand=edges&edge_type=${edgeType}&depth=1`, + siblingsRes = await fetch( + `${jarvisUrl}/v2/nodes/${encodedSetId}?expand=edges&edge_type=${edgeType}&depth=1`, { headers: { "x-api-token": swarmApiKey } }, ); - if (siblingsRes.ok) { - const siblingsData = await siblingsRes.json(); - const siblings = (siblingsData?.nodes ?? []).filter( - (n: { ref_id?: string; node_type?: string }) => - n.ref_id !== setId && - String(n.node_type ?? "").toLowerCase() === "evalrequirement", - ); - siblingCount = siblings.length; - } } catch { - // Non-fatal — order defaults to 0 + console.warn(`[Gateway Evals Requirements POST] Jarvis fetch error for setId=${setId}`); + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + if (!siblingsRes.ok) { + console.warn(`[Gateway Evals Requirements POST] Jarvis ${siblingsRes.status} for setId=${setId}`); + return NextResponse.json({ error: "Not found" }, { status: 404 }); } + const siblingsData = await siblingsRes.json(); + const setNode = (siblingsData?.nodes ?? []).find( + (n: JarvisNode) => n.ref_id === setId, + ); + if (!setNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + const ownerWorkspaceId = + setNode.properties?.workspace_id ?? setNode.properties?.workspaceId; + if (ownerWorkspaceId && ownerWorkspaceId !== workspaceId) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const siblings = (siblingsData?.nodes ?? []).filter( + (n: { ref_id?: string; node_type?: string }) => + n.ref_id !== setId && + String(n.node_type ?? "").toLowerCase() === "evalrequirement", + ); + const siblingCount = siblings.length; + const id = randomUUID(); + + const nodeData: Record = { + id, + name: name.trim(), + description, + prompt_snippet: typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, + desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], + undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], + contested: contestedCoerced ?? false, + }; + const nodeResult = await addNode(config, { node_type: "EvalRequirement", - node_data: { - id, - name: name.trim(), - description, - prompt_snippet: typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, - desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], - undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], - }, + node_data: nodeData, }); if (!nodeResult.success || !nodeResult.ref_id) { diff --git a/src/app/api/mock/evals/[evalSetId]/requirements/[reqId]/route.ts b/src/app/api/mock/evals/[evalSetId]/requirements/[reqId]/route.ts index a53a9ab935..d489d0b506 100644 --- a/src/app/api/mock/evals/[evalSetId]/requirements/[reqId]/route.ts +++ b/src/app/api/mock/evals/[evalSetId]/requirements/[reqId]/route.ts @@ -5,11 +5,17 @@ export const runtime = "nodejs"; type RouteParams = { params: Promise<{ evalSetId: string; reqId: string }> }; export async function PUT(request: NextRequest, { params }: RouteParams) { + if (process.env.USE_MOCKS !== "true") { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } const { evalSetId, reqId } = await params; const body = await request.json().catch(() => ({})); return NextResponse.json({ success: true, data: { ref_id: reqId, evalSetId, ...body } }); } export async function DELETE(_request: NextRequest, _ctx: RouteParams) { + if (process.env.USE_MOCKS !== "true") { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } return NextResponse.json({ success: true }); } diff --git a/src/app/api/mock/evals/[evalSetId]/requirements/route.ts b/src/app/api/mock/evals/[evalSetId]/requirements/route.ts index 28fd75709f..2d26987556 100644 --- a/src/app/api/mock/evals/[evalSetId]/requirements/route.ts +++ b/src/app/api/mock/evals/[evalSetId]/requirements/route.ts @@ -11,6 +11,8 @@ type RequirementNode = JarvisNode & { desirable_cases: string[]; undesirable_cases: string[]; order: number; + contested?: boolean; + contest_reason?: string; }; }; @@ -38,6 +40,9 @@ const SEED_REQUIREMENTS: Record = { desirable_cases: ["Output is valid JavaScript", "Function accepts two arguments", "Returns the sum"], undesirable_cases: ["Syntax errors present", "Wrong return value"], order: 1, + contested: true, + contest_reason: + "The criterion conflates syntactic correctness with semantic accuracy. A function can be syntactically valid but return the wrong result, making this criterion too broad to be a reliable signal.", }, }, { @@ -82,18 +87,31 @@ const SEED_REQUIREMENTS: Record = { }; export async function GET( - _request: NextRequest, + request: NextRequest, { params }: { params: Promise<{ evalSetId: string }> }, ) { + if (process.env.USE_MOCKS !== "true") { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } const { evalSetId } = await params; const nodes = SEED_REQUIREMENTS[evalSetId] ?? []; return NextResponse.json({ success: true, data: { nodes, total: nodes.length } }); } export async function POST(request: NextRequest) { + if (process.env.USE_MOCKS !== "true") { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } const body = await request.json().catch(() => ({})); - const { name, description, prompt_snippet, desirable_cases, undesirable_cases } = - body ?? {}; + const { + name, + description, + prompt_snippet, + desirable_cases, + undesirable_cases, + contested, + contest_reason, + } = body ?? {}; const newNode: JarvisNode = { ref_id: crypto.randomUUID(), @@ -104,8 +122,13 @@ export async function POST(request: NextRequest) { prompt_snippet, desirable_cases: desirable_cases ?? [], undesirable_cases: undesirable_cases ?? [], + ...(contested !== undefined ? { contested: Boolean(contested) } : {}), + ...(contest_reason !== undefined ? { contest_reason } : {}), }, }; - return NextResponse.json({ success: true, data: { ref_id: newNode.ref_id } }); + return NextResponse.json({ + success: true, + data: { ref_id: newNode.ref_id, ...newNode.properties }, + }); } diff --git a/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/[reqId]/route.ts b/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/[reqId]/route.ts index 1ade7612d5..94c69a31eb 100644 --- a/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/[reqId]/route.ts +++ b/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/[reqId]/route.ts @@ -3,6 +3,8 @@ import { getMiddlewareContext, requireAuth } from "@/lib/middleware/utils"; import { getJarvisUrl } from "@/lib/utils/swarm"; import { getWorkspaceSwarmAccess } from "@/lib/helpers/swarm-access"; import { updateNode, deleteNode } from "@/services/swarm/api/nodes"; +import { coerceContested } from "../route"; +import type { JarvisNode } from "@/types/jarvis"; type RouteParams = { params: Promise<{ slug: string; evalSetId: string; reqId: string }>; @@ -21,6 +23,70 @@ function handleSwarmAccessError(error: { type: string }) { return NextResponse.json({ error: errorInfo.message }, { status: errorInfo.status }); } +// Roles allowed to set the `contested` flag (mirrors canReadRunReport) +const CONTESTED_WRITE_ROLES = ["OWNER", "ADMIN", "PM", "DEVELOPER"] as const; + +/** + * Fetch the eval set from Jarvis (HAS_REQUIREMENT expand) and verify: + * 1. The fetch succeeded — fail closed, never skip on error. + * 2. The set node exists in the response. + * 3. If the set carries a workspaceId property, it matches the caller's workspace. + * 4. The reqId (when provided) appears as an EvalRequirement child of the set. + * + * Returns the req node (for caller use) or a NextResponse to short-circuit. + */ +async function verifyEvalSetOwnership( + jarvisUrl: string, + swarmApiKey: string, + evalSetId: string, + workspaceId: string, + reqId?: string, +): Promise<{ reqNode?: JarvisNode } | NextResponse> { + const edgeType = encodeURIComponent("['HAS_REQUIREMENT']"); + let setCheckRes: Response; + try { + setCheckRes = await fetch( + `${jarvisUrl}/v2/nodes/${encodeURIComponent(evalSetId)}?expand=edges&edge_type=${edgeType}&depth=1`, + { headers: { "x-api-token": swarmApiKey } }, + ); + } catch { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + if (!setCheckRes.ok) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const setData = await setCheckRes.json(); + const setNode = (setData?.nodes ?? []).find( + (n: JarvisNode) => n.ref_id === evalSetId, + ); + if (!setNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const ownerWorkspaceId = + setNode.properties?.workspace_id ?? setNode.properties?.workspaceId; + if (ownerWorkspaceId && ownerWorkspaceId !== workspaceId) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + if (reqId !== undefined) { + const reqNodes: JarvisNode[] = (setData?.nodes ?? []).filter( + (n: JarvisNode) => + n.ref_id !== evalSetId && + String(n.node_type ?? "").toLowerCase() === "evalrequirement", + ); + const reqNode = reqNodes.find((n) => n.ref_id === reqId); + if (!reqNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + return { reqNode }; + } + + return {}; +} + export async function PUT(request: NextRequest, { params }: RouteParams) { try { const context = getMiddlewareContext(request); @@ -30,20 +96,47 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { const { slug, evalSetId, reqId } = await params; const body = await request.json(); - const { name, description, prompt_snippet, desirable_cases, undesirable_cases } = body ?? {}; + const { name, description, prompt_snippet, desirable_cases, undesirable_cases, contested } = + body ?? {}; - // A requirement only needs a name and an optional reason (description). - // prompt_snippet and example cases are optional and preserved if present. if (!name || typeof name !== "string" || !name.trim()) { return NextResponse.json({ error: "name is required" }, { status: 400 }); } + const contestedCoerced = coerceContested(contested); + if (contestedCoerced === null) { + return NextResponse.json( + { error: "contested must be a boolean (true, false, 1, 0, \"true\", or \"false\")" }, + { status: 400 }, + ); + } + const swarmAccessResult = await getWorkspaceSwarmAccess(slug, userOrResponse.id); if (!swarmAccessResult.success) { console.warn(`[Evals Requirements PUT] Swarm access denied: ${swarmAccessResult.error.type}`); return handleSwarmAccessError(swarmAccessResult.error); } + // Role gate: only DEVELOPER+ may set the `contested` field + if (contestedCoerced !== undefined) { + const { db } = await import("@/lib/db"); + const member = await db.workspaceMember.findFirst({ + where: { + workspaceId: swarmAccessResult.data.workspaceId, + userId: userOrResponse.id, + leftAt: null, + }, + select: { role: true }, + }); + const role = member?.role ?? "OWNER"; + if (!(CONTESTED_WRITE_ROLES as readonly string[]).includes(role)) { + return NextResponse.json( + { error: "Insufficient permissions to set contested" }, + { status: 403 }, + ); + } + } + if (process.env.USE_MOCKS === "true") { const mockResponse = await fetch( `${request.nextUrl.origin}/api/mock/evals/${evalSetId}/requirements/${reqId}`, @@ -56,21 +149,33 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { return NextResponse.json(await mockResponse.json()); } - const { swarmName, swarmApiKey } = swarmAccessResult.data; + const { swarmName, swarmApiKey, workspaceId } = swarmAccessResult.data; const jarvisUrl = getJarvisUrl(swarmName); const config = { jarvisUrl, apiKey: swarmApiKey }; + // IDOR: verify reqId is reachable from evalSetId and that the set belongs + // to this workspace — fail closed on any Jarvis error. + const ownershipResult = await verifyEvalSetOwnership( + jarvisUrl, swarmApiKey, evalSetId, workspaceId, reqId, + ); + if (ownershipResult instanceof NextResponse) return ownershipResult; + + const nodeData: Record = { + name: name.trim(), + description, + prompt_snippet: + typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, + desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], + undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], + }; + if (contestedCoerced !== undefined) { + nodeData.contested = contestedCoerced; + } + const result = await updateNode(config, { ref_id: reqId, node_type: "EvalRequirement", - node_data: { - name: name.trim(), - description, - prompt_snippet: - typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, - desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], - undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], - }, + node_data: nodeData, }); if (!result.success) { @@ -106,10 +211,16 @@ export async function DELETE(request: NextRequest, { params }: RouteParams) { return NextResponse.json(await mockResponse.json()); } - const { swarmName, swarmApiKey } = swarmAccessResult.data; + const { swarmName, swarmApiKey, workspaceId } = swarmAccessResult.data; const jarvisUrl = getJarvisUrl(swarmName); const config = { jarvisUrl, apiKey: swarmApiKey }; + // IDOR: verify reqId belongs to this eval set and workspace before deleting. + const ownershipResult = await verifyEvalSetOwnership( + jarvisUrl, swarmApiKey, evalSetId, workspaceId, reqId, + ); + if (ownershipResult instanceof NextResponse) return ownershipResult; + const result = await deleteNode(config, reqId); if (!result.success) { diff --git a/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route.ts b/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route.ts index 1af9f7b5bd..63a5c1647d 100644 --- a/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route.ts +++ b/src/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route.ts @@ -6,6 +6,26 @@ import { getWorkspaceSwarmAccess } from "@/lib/helpers/swarm-access"; import { addNode, addEdge } from "@/services/swarm/api/nodes"; import type { JarvisNode } from "@/types/jarvis"; +/** + * Coerce a loosely-typed value to a real JSON boolean, or return undefined + * when the value is absent. Returns null for un-coercible values (caller + * should 400). + * + * Accepts: true | false | 1 | 0 | "true" | "false" (case-insensitive). + * Rejects: any other string, object, array, etc. + */ +export function coerceContested(value: unknown): boolean | null | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === "boolean") return value; + if (value === 1) return true; + if (value === 0) return false; + if (typeof value === "string") { + const lower = value.toLowerCase(); + if (lower === "true") return true; + if (lower === "false") return false; + } + return null; // un-coercible +} type RouteParams = { params: Promise<{ slug: string; evalSetId: string }> }; @@ -22,6 +42,9 @@ function handleSwarmAccessError(error: { type: string }) { return NextResponse.json({ error: errorInfo.message }, { status: errorInfo.status }); } +// Roles allowed to set the `contested` flag (mirrors canReadRunReport) +const CONTESTED_WRITE_ROLES = ["OWNER", "ADMIN", "PM", "DEVELOPER"] as const; + export async function GET(request: NextRequest, { params }: RouteParams) { try { const context = getMiddlewareContext(request); @@ -107,20 +130,50 @@ export async function POST(request: NextRequest, { params }: RouteParams) { console.log(`[Evals Requirements POST] slug=${slug}, evalSetId=${evalSetId}, userId=${userOrResponse.id}`); const body = await request.json(); - const { name, description, prompt_snippet, desirable_cases, undesirable_cases, order } = + const { name, description, prompt_snippet, desirable_cases, undesirable_cases, order, contested } = body ?? {}; - // A requirement only needs a name and an optional reason (description). - // prompt_snippet and example cases are optional and may be added later. if (!name || typeof name !== "string" || !name.trim()) { return NextResponse.json({ error: "name is required" }, { status: 400 }); } + // Validate `contested` when present — before any auth/swarm work + const contestedCoerced = coerceContested(contested); + if (contestedCoerced === null) { + return NextResponse.json( + { error: "contested must be a boolean (true, false, 1, 0, \"true\", or \"false\")" }, + { status: 400 }, + ); + } + const swarmAccessResult = await getWorkspaceSwarmAccess(slug, userOrResponse.id); if (!swarmAccessResult.success) { console.warn(`[Evals Requirements POST] Swarm access denied: ${swarmAccessResult.error.type}`); return handleSwarmAccessError(swarmAccessResult.error); } + + // Role gate: only DEVELOPER+ may set the `contested` field + if (contestedCoerced !== undefined) { + const { db } = await import("@/lib/db"); + const member = await db.workspaceMember.findFirst({ + where: { + workspaceId: swarmAccessResult.data.workspaceId, + userId: userOrResponse.id, + leftAt: null, + }, + select: { role: true }, + }); + // Owner is not in workspaceMember table but always has full rights. + // If member is null the caller is the workspace owner (swarmAccess succeeded). + const role = member?.role ?? "OWNER"; + if (!(CONTESTED_WRITE_ROLES as readonly string[]).includes(role)) { + return NextResponse.json( + { error: "Insufficient permissions to set contested" }, + { status: 403 }, + ); + } + } + console.log(`[Evals Requirements POST] Swarm access granted — swarmName=${swarmAccessResult.data.swarmName}, apiKey present=${!!swarmAccessResult.data.swarmApiKey}`); if (process.env.USE_MOCKS === "true") { @@ -136,23 +189,58 @@ export async function POST(request: NextRequest, { params }: RouteParams) { return NextResponse.json(await mockResponse.json()); } - const { swarmName, swarmApiKey } = swarmAccessResult.data; + const { swarmName, swarmApiKey, workspaceId } = swarmAccessResult.data; const jarvisUrl = getJarvisUrl(swarmName); const config = { jarvisUrl, apiKey: swarmApiKey }; console.log(`[Evals Requirements POST] Jarvis URL: ${jarvisUrl}`); + // IDOR: verify the evalSet belongs to this workspace before any write. + // Fetch it via HAS_REQUIREMENT (same call used by GET, avoids an extra + // round-trip) and assert ownership. Fail closed: any non-ok response from + // Jarvis is treated as "cannot confirm ownership" → 404, not a pass-through. + const edgeType = encodeURIComponent("['HAS_REQUIREMENT']"); + const setCheckRes = await fetch( + `${jarvisUrl}/v2/nodes/${encodeURIComponent(evalSetId)}?expand=edges&edge_type=${edgeType}&depth=1`, + { headers: { "x-api-token": swarmApiKey } }, + ); + if (!setCheckRes.ok) { + console.warn(`[Evals Requirements POST] Could not verify eval set ownership: Jarvis ${setCheckRes.status}`); + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + const setData = await setCheckRes.json(); + const setNode = (setData?.nodes ?? []).find( + (n: JarvisNode) => n.ref_id === evalSetId, + ); + if (!setNode) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + // If Jarvis carries the workspace property, assert it matches. If absent, + // the HAS_REQUIREMENT expand already scopes to the swarm for this workspace + // (the swarmApiKey is workspace-scoped), which is sufficient isolation. + const ownerWorkspaceId = + setNode.properties?.workspace_id ?? setNode.properties?.workspaceId; + if (ownerWorkspaceId && ownerWorkspaceId !== workspaceId) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + const id = randomUUID(); + + // Build node_data — omit `contested` key entirely when undefined so the + // tri-state partial-merge semantics apply on Jarvis. Default to false on create. + const nodeData: Record = { + id, + name: name.trim(), + description, + prompt_snippet: + typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, + desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], + undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], + contested: contestedCoerced ?? false, + }; + const nodeResult = await addNode(config, { node_type: "EvalRequirement", - node_data: { - id, - name: name.trim(), - description, - prompt_snippet: - typeof prompt_snippet === "string" ? prompt_snippet.trim() : undefined, - desirable_cases: Array.isArray(desirable_cases) ? desirable_cases : [], - undesirable_cases: Array.isArray(undesirable_cases) ? undesirable_cases : [], - }, + node_data: nodeData, }); console.log(`[Evals Requirements POST] addNode result: success=${nodeResult.success}, ref_id=${nodeResult.ref_id ?? 'n/a'}, error=${nodeResult.error ?? 'none'}`); @@ -163,7 +251,6 @@ export async function POST(request: NextRequest, { params }: RouteParams) { ); } - // Determine order: use provided value or default to 0 const edgeOrder = typeof order === "number" ? order : 0; const edgeResult = await addEdge(config, { diff --git a/src/components/evals/EditRequirementModal.tsx b/src/components/evals/EditRequirementModal.tsx index 04cbae94b0..281a709b08 100644 --- a/src/components/evals/EditRequirementModal.tsx +++ b/src/components/evals/EditRequirementModal.tsx @@ -11,6 +11,7 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; import { useWorkspace } from "@/hooks/useWorkspace"; import type { JarvisNode } from "@/types/jarvis"; @@ -32,6 +33,7 @@ export function EditRequirementModal({ const { slug } = useWorkspace(); const [name, setName] = useState(""); const [reason, setReason] = useState(""); + const [contested, setContested] = useState(false); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(""); @@ -40,6 +42,7 @@ export function EditRequirementModal({ if (open) { setName(String(requirement.properties?.name ?? "")); setReason(String(requirement.properties?.description ?? "")); + setContested(Boolean(requirement.properties?.contested)); setError(""); } }, [open, requirement]); @@ -69,6 +72,8 @@ export function EditRequirementModal({ ? props.undesirable_cases : undefined; + // Always send the contested value (switch is always shown and pre-populated, + // so the user's current toggle state is always intentional). try { const res = await fetch( `/api/workspaces/${slug}/evals/${evalSetId}/requirements/${requirement.ref_id}`, @@ -81,22 +86,33 @@ export function EditRequirementModal({ prompt_snippet: promptSnippet, desirable_cases: desirableCases, undesirable_cases: undesirableCases, + contested, }), }, ); - if (!res.ok) throw new Error("Request failed"); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error((data as { error?: string }).error ?? "Request failed"); + } toast.success("Requirement updated"); onUpdated(); handleClose(); - } catch { - toast.error("Failed to update requirement"); + } catch (err) { + toast.error( + err instanceof Error ? err.message : "Failed to update requirement", + ); } finally { setSubmitting(false); } } + const contestReason = + typeof requirement.properties?.contest_reason === "string" + ? requirement.properties.contest_reason + : null; + return ( @@ -132,6 +148,41 @@ export function EditRequirementModal({ /> + {/* Contested toggle — governs criterion definition, not historical runs */} +
+ {contestReason && ( +
+ +

+ {contestReason} +

+
+ )} +
+
+ +

+ Marks this criterion definition as suspect. Applies to subsequent + runs — historical run results are immutable snapshots and are not + affected. +

+
+ +
+
+