Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- AlterTable
ALTER TABLE "stakwork_runs" ADD COLUMN "report_bundle" JSONB,
ADD COLUMN "report_bundle_hash" TEXT,
ADD COLUMN "report_partial" BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN "report_url" TEXT,
ADD COLUMN "report_url_rejection_reason" TEXT,
ADD COLUMN "schema_unsupported" BOOLEAN NOT NULL DEFAULT false;
39 changes: 39 additions & 0 deletions prisma/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,25 @@ model Phase {
@@map("phases")
}

/// StakworkRun tracks AI workflow executions dispatched to the Stakwork platform.
///
/// ARTIFACT OWNERSHIP — two separate report artifacts, do NOT conflate:
///
/// 1. "Chat report" — an AI-written narrative produced post-run by the org-canvas agent.
/// Fields: reportStatus / reportConversationId / reportChatPath (all nested in `result`
/// JSON via mergeIntoRunResult in src/services/legal-benchmark-report.ts).
///
/// 2. "Run report bundle" — a structured JSON bundle produced by the Harvey LAB runner
/// and uploaded to S3. Fields: reportUrl (canonical bundle-pointer, server-side only),
/// reportBundle (sanitized projection persisted at ingest), reportBundleHash (SHA-256 of
/// the raw bundle for re-sanitize integrity), reportPartial (projection truncated due to
/// size), schemaUnsupported (schema_version not handled by this Hive version),
/// reportUrlRejectionReason (why a supplied report_url was rejected — never the URL).
///
/// reportUrl and webhookUrl are globally omitted from Prisma reads (src/lib/db.ts) and
/// must be accessed via an explicit select. webhookUrl embeds a raw HMAC run_token in its
/// query string; leaking it to clients collapses the run_token gate. See follow-up ticket
/// for moving the token to a dedicated hashed column.
model StakworkRun {
id String @id @default(cuid())
webhookUrl String @map("webhook_url")
Expand All @@ -1053,6 +1072,26 @@ model StakworkRun {
promptVersionId String? @map("prompt_version_id")
evalSetId String? @map("eval_set_id")
userId String? @map("user_id")

// ── Run report bundle fields (see ARTIFACT OWNERSHIP note above) ──────────
/// S3 URL of the raw report bundle. Server-side only — globally omitted from
/// Prisma reads. Never forwarded to clients. Captured at webhook ingest under
/// run_token + S3-allowlist gate.
reportUrl String? @map("report_url")
/// Sanitized, redacted projection of the bundle stored at ingest while the
/// presigned URL is still valid. The view path reads only this field.
reportBundle Json? @map("report_bundle")
/// SHA-256 hex digest of the raw bundle bytes, for future re-sanitize passes.
reportBundleHash String? @map("report_bundle_hash")
/// True when the projection was truncated (source_docs html bodies dropped)
/// because the serialized projection exceeded the size cap.
reportPartial Boolean @default(false) @map("report_partial")
/// True when the bundle's schema_version is not supported by this Hive version.
schemaUnsupported Boolean @default(false) @map("schema_unsupported")
/// Opaque reason code recorded when a supplied report_url failed validation
/// (never the URL itself, never the S3 host, never a query string).
reportUrlRejectionReason String? @map("report_url_rejection_reason")

agentLogs AgentLog[]
feature Feature? @relation(fields: [featureId], references: [id])
task Task? @relation(fields: [taskId], references: [id])
Expand Down
1 change: 1 addition & 0 deletions src/__tests__/support/mocks/prisma.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ const { db: dbMock, reset } = hoisted;

vi.mock("@/lib/db", () => ({
db: dbMock,
dbAdapter: dbMock,
}));

export { dbMock };
Expand Down
224 changes: 224 additions & 0 deletions src/__tests__/unit/services/report-url-omit.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,224 @@
/**
* Unit tests for the reportUrl/webhookUrl global Prisma omit and hasReport derivation.
*
* Test cases:
* 1. Global invariant: getStakworkRuns response never contains reportUrl, webhookUrl, or report_url
* 2. hasReport derives from reportBundle presence, NOT from reportUrl column
* 3. hasReport is false when reportBundle is null
* 4. hasReport is true when reportBundle is non-null
* 5. Explicit select for webhookUrl is opt-in — not a hard removal (the write path still works)
* 6. reportPartial derives from the reportPartial column, not from reportUrl
* 7. schemaUnsupported derives from the schemaUnsupported column
* 8. reportBundle is not forwarded to the mapped output (projection stripped, only flags)
*/

import { describe, test, expect, vi, beforeEach } from "vitest";

// ─── Stable mock references (hoisted) ────────────────────────────────────────

const mockDbWorkspaceFindUnique = vi.hoisted(() => vi.fn());
const mockDbStakworkRunCount = vi.hoisted(() => vi.fn());
const mockDbStakworkRunFindMany = vi.hoisted(() => vi.fn());

// ─── Module mocks ─────────────────────────────────────────────────────────────

vi.mock("@/lib/db", () => ({
db: {
workspace: {
findUnique: mockDbWorkspaceFindUnique,
},
stakworkRun: {
count: mockDbStakworkRunCount,
findMany: mockDbStakworkRunFindMany,
},
},
}));

// ─── Import subject under test ────────────────────────────────────────────────

import { getStakworkRuns } from "@/services/stakwork-run";
import { StakworkRunType, WorkflowStatus } from "@prisma/client";

// ─── Helpers ─────────────────────────────────────────────────────────────────

const WORKSPACE_ID = "ws-test-1";
const USER_ID = "user-1";

function makeWorkspace() {
return {
id: WORKSPACE_ID,
ownerId: USER_ID,
deleted: false,
members: [],
};
}

function makeDbRun(overrides: Record<string, unknown> = {}) {
return {
id: "run-1",
type: StakworkRunType.LEGAL_BENCHMARK_RUNNER,
status: WorkflowStatus.COMPLETED,
workspaceId: WORKSPACE_ID,
featureId: null,
projectId: 42,
dataType: "json",
decision: null,
feedback: null,
createdAt: new Date("2025-01-01T00:00:00Z"),
updatedAt: new Date("2025-01-01T01:00:00Z"),
taskId: null,
autoAccept: false,
promptVersionId: null,
evalSetId: null,
userId: null,
// Note: reportBundle, reportPartial, schemaUnsupported are selected by getStakworkRuns
reportBundle: null,
reportPartial: false,
schemaUnsupported: false,
// Note: webhookUrl and reportUrl are NOT selected (globally omitted)
feature: null,
...overrides,
};
}

const BASE_QUERY = {
workspaceId: WORKSPACE_ID,
limit: 20,
offset: 0,
includeResult: false,
};

// ─── Tests ───────────────────────────────────────────────────────────────────

describe("getStakworkRuns — reportUrl/webhookUrl omit + hasReport derivation", () => {
beforeEach(() => {
vi.resetAllMocks();
mockDbWorkspaceFindUnique.mockResolvedValue(makeWorkspace());
mockDbStakworkRunCount.mockResolvedValue(1);
});

test("1. response never contains reportUrl, webhookUrl, or report_url", async () => {
mockDbStakworkRunFindMany.mockResolvedValue([makeDbRun()]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);

const serialized = JSON.stringify(result);
expect(serialized).not.toContain("reportUrl");
expect(serialized).not.toContain("report_url");
expect(serialized).not.toContain("webhookUrl");
expect(serialized).not.toContain("webhook_url");

// Also verify at the object level on each run
for (const run of result.runs) {
const runObj = run as Record<string, unknown>;
expect(runObj).not.toHaveProperty("reportUrl");
expect(runObj).not.toHaveProperty("report_url");
expect(runObj).not.toHaveProperty("webhookUrl");
expect(runObj).not.toHaveProperty("webhook_url");
}
});

test("2. hasReport is false when reportBundle is null (derives from projection, not URL column)", async () => {
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({ reportBundle: null }),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs).toHaveLength(1);
expect(result.runs[0].hasReport).toBe(false);
});

test("3. hasReport is true when reportBundle is non-null", async () => {
const projection = { schema_version: 1, page_data: {}, source_docs: [] };
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({ reportBundle: projection }),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs).toHaveLength(1);
expect(result.runs[0].hasReport).toBe(true);
});

test("4. hasReport does NOT depend on the reportUrl column value", async () => {
// reportBundle is null → hasReport false, regardless of what reportUrl would be
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({ reportBundle: null }),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs[0].hasReport).toBe(false);

// Verify that the select clause passed to findMany does NOT include reportUrl or webhookUrl
const selectArg = mockDbStakworkRunFindMany.mock.calls[0][0].select;
expect(selectArg).not.toHaveProperty("reportUrl");
expect(selectArg).not.toHaveProperty("webhookUrl");
// But does include the projection fields
expect(selectArg).toHaveProperty("reportBundle", true);
expect(selectArg).toHaveProperty("reportPartial", true);
expect(selectArg).toHaveProperty("schemaUnsupported", true);
});

test("5. reportPartial flag derives from the reportPartial column", async () => {
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({
reportBundle: { schema_version: 1 },
reportPartial: true,
schemaUnsupported: false,
}),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs[0].reportPartial).toBe(true);
expect(result.runs[0].schemaUnsupported).toBe(false);
});

test("6. schemaUnsupported flag derives from the schemaUnsupported column", async () => {
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({
reportBundle: { schema_version: 99 },
reportPartial: false,
schemaUnsupported: true,
}),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs[0].schemaUnsupported).toBe(true);
expect(result.runs[0].reportPartial).toBe(false);
expect(result.runs[0].hasReport).toBe(true);
});

test("7. reportBundle column is NOT forwarded to the mapped output", async () => {
const projection = { schema_version: 1, source_docs: [{ id: "d1" }] };
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({ reportBundle: projection }),
]);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
const runObj = result.runs[0] as Record<string, unknown>;
// The raw projection must not leak to the response
expect(runObj).not.toHaveProperty("reportBundle");
// Only the derived flag should be present
expect(runObj.hasReport).toBe(true);
});

test("8. multiple runs mapped independently — each derives hasReport independently", async () => {
mockDbStakworkRunFindMany.mockResolvedValue([
makeDbRun({ id: "run-1", reportBundle: null }),
makeDbRun({ id: "run-2", reportBundle: { schema_version: 1 } }),
]);
mockDbStakworkRunCount.mockResolvedValue(2);

const result = await getStakworkRuns(BASE_QUERY, USER_ID);
expect(result.runs).toHaveLength(2);

const run1 = result.runs.find((r) => r.id === "run-1");
const run2 = result.runs.find((r) => r.id === "run-2");
expect(run1?.hasReport).toBe(false);
expect(run2?.hasReport).toBe(true);

// Neither run has webhookUrl or reportUrl in its output
const serialized = JSON.stringify(result);
expect(serialized).not.toContain("webhookUrl");
expect(serialized).not.toContain("reportUrl");
});
});
2 changes: 1 addition & 1 deletion src/app/api/stakwork/ai/generate/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export async function POST(request: NextRequest) {
status: run.status,
workspaceId: run.workspaceId,
featureId: run.featureId,
webhookUrl: run.webhookUrl,
// webhookUrl is globally omitted (embeds raw HMAC run_token)
projectId: run.projectId,
createdAt: run.createdAt,
},
Expand Down
14 changes: 14 additions & 0 deletions src/hooks/useLegalBenchmarkRunList.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ export interface BenchmarkRunListRow {
reportStatus?: string;
/** Relative link to the report chat, e.g. "/org/<login>?chat=<id>" */
reportChatPath?: string;
// Run report bundle flags — derived server-side from the persisted projection
/** True when a sanitized run report bundle is available for this run */
hasReport?: boolean;
/** True when the bundle was truncated due to size limits */
reportPartialBundle?: boolean;
/** True when the bundle's schema_version is unsupported by this Hive version */
schemaUnsupported?: boolean;
}

interface UseLegalBenchmarkRunListResult {
Expand Down Expand Up @@ -76,6 +83,9 @@ export function useLegalBenchmarkRunList(
result: string | null;
createdAt: string;
updatedAt: string;
hasReport?: boolean;
reportPartial?: boolean;
schemaUnsupported?: boolean;
}> = data.runs ?? [];

const mapped: BenchmarkRunListRow[] = rawRows.map((r) => {
Expand All @@ -97,6 +107,10 @@ export function useLegalBenchmarkRunList(
generateReport: parsed?.generateReport,
reportStatus: parsed?.reportStatus,
reportChatPath: parsed?.reportChatPath,
// Run report bundle flags from the server-side mapper
hasReport: r.hasReport ?? false,
reportPartialBundle: r.reportPartial ?? false,
schemaUnsupported: r.schemaUnsupported ?? false,
// Unified judge precedence: operator choice takes priority over runner-echoed value.
// Format mirrors stakwork-run.ts — if the server-side format string changes, update this line to match.
judgeNotes:
Expand Down
4 changes: 2 additions & 2 deletions src/lib/auth/nextauth.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { db } from "@/lib/db";
import { db, dbAdapter } from "@/lib/db";
import { EncryptionService } from "@/lib/encryption";
import { logger } from "@/lib/logger";
import { ensureMockWorkspaceForUser, ensureStakworkMockWorkspace, ensureMockOrgData, ensureMockLlmModels } from "@/utils/mockSetup";
Expand Down Expand Up @@ -180,7 +180,7 @@

export const authOptions: NextAuthOptions = {
// Only use PrismaAdapter when not using credentials provider
...(process.env.POD_URL ? {} : { adapter: PrismaAdapter(db) }),
...(process.env.POD_URL ? {} : { adapter: PrismaAdapter(dbAdapter) }),

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/lib/ai/graphWalkerTools.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/task-workflow.ts:5:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/lib/ai/filterReadonly.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/task-workflow.ts:5:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/api/user/profile.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/app/api/user/profile/route.ts:3:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/api/stakwork/user-journey.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/app/api/stakwork/user-journey/route.ts:3:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/services/task-workflow-createChatMessageAndTriggerStakwork.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/__tests__/unit/services/task-workflow-createChatMessageAndTriggerStakwork.test.ts:2:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/services/task-coordinator-dependencies.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/task-workflow.ts:5:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/services/report-url-omit.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/workflow-editor.ts:12:1 ❯ src/services/stakwork-run.ts:39:1 ❯ src/__tests__/unit/services/report-url-omit.test.ts:39:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/services/release-stale-task-pods.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/task-workflow.ts:5:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/api/swarm-stakgraph-ingest-route.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/app/api/swarm/stakgraph/ingest/route.ts:2:1

Check failure on line 183 in src/lib/auth/nextauth.ts

View workflow job for this annotation

GitHub Actions / unit-tests

src/__tests__/unit/api/legal-benchmark-eval-route.test.ts

Error: [vitest] No "dbAdapter" export is defined on the "@/lib/db" mock. Did you forget to return it from "vi.mock"? If you need to partially mock a module, you can use "importOriginal" helper inside: vi.mock(import("@/lib/db"), async (importOriginal) => { const actual = await importOriginal() return { ...actual, // your mocked methods } }) ❯ src/lib/auth/nextauth.ts:183:59 ❯ src/services/workflow-editor.ts:12:1 ❯ src/services/stakwork-run.ts:39:1 ❯ src/__tests__/unit/api/legal-benchmark-eval-route.test.ts:116:1
providers: getProviders(),
callbacks: {
async signIn({ user, account }) {
Expand Down
Loading
Loading