From b26961e144b888bdb4441e356b178e02563a76e5 Mon Sep 17 00:00:00 2001 From: "zealt-staging[bot]" <264479255+zealt-staging[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 04:19:27 +0000 Subject: [PATCH] Initialize repository with benchmark template and tasks --- .github/workflows/deploy.yml | 69 ++ .gitignore | 49 + .zealt/config.json | 5 + README.md | 47 +- plan.md | 136 +++ .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../type_safe_navigation_menu/instruction.md | 7 + .../(home)/components/leaderboard-table.tsx | 183 +++ site/app/(home)/page.tsx | 176 +++ site/app/globals.css | 149 +++ site/app/layout.tsx | 37 + .../trajectory/components/artifacts-panel.tsx | 265 ++++ .../trajectory/components/trajectory-page.tsx | 404 ++++++ .../tasks/[name]/[jobId]/trajectory/page.tsx | 414 +++++++ site/app/tasks/components/back-to-top.tsx | 41 + site/app/tasks/components/multi-select.tsx | 104 ++ .../tasks/components/tasks-page-client.tsx | 724 +++++++++++ site/app/tasks/page.tsx | 142 +++ site/bun.lock | 1082 +++++++++++++++++ site/components.json | 23 + site/components/pending-review-card.tsx | 44 + site/components/query-provider.tsx | 21 + site/components/theme-provider.tsx | 27 + site/components/theme-toggle.tsx | 32 + site/components/ui/badge.tsx | 48 + site/components/ui/button.tsx | 64 + site/components/ui/checkbox.tsx | 32 + site/components/ui/command.tsx | 184 +++ site/components/ui/dialog.tsx | 158 +++ site/components/ui/drawer.tsx | 131 ++ site/components/ui/hover-card.tsx | 44 + site/components/ui/popover.tsx | 89 ++ site/components/ui/scroll-area.tsx | 59 + site/components/ui/select.tsx | 190 +++ site/components/ui/sheet.tsx | 153 +++ site/components/ui/skeleton.tsx | 16 + site/components/ui/tabs.tsx | 69 ++ site/lib/http-error.ts | 8 + site/lib/utils.ts | 6 + site/next.config.ts | 17 + site/package.json | 34 + site/postcss.config.mjs | 7 + site/scripts/check-trajectory.ts | 118 ++ site/scripts/compute-tasks.ts | 252 ++++ site/tsconfig.json | 41 + site/types/result.d.ts | 30 + site/zealt | 1 + 51 files changed, 5971 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 .zealt/config.json create mode 100644 plan.md create mode 100644 scratchpad/pending-tasks/headless_table_initialization_and_typing/instruction.md create mode 100644 scratchpad/pending-tasks/optimistic_updates_with_tanstack_query/instruction.md create mode 100644 scratchpad/pending-tasks/resolving_query_ssr_hydration_mismatch/instruction.md create mode 100644 scratchpad/pending-tasks/server_function_creation_in_tanstack_start/instruction.md create mode 100644 scratchpad/pending-tasks/synchronized_url_search_parameters/instruction.md create mode 100644 scratchpad/pending-tasks/type_safe_navigation_menu/instruction.md create mode 100644 site/app/(home)/components/leaderboard-table.tsx create mode 100644 site/app/(home)/page.tsx create mode 100644 site/app/globals.css create mode 100644 site/app/layout.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/page.tsx create mode 100644 site/app/tasks/components/back-to-top.tsx create mode 100644 site/app/tasks/components/multi-select.tsx create mode 100644 site/app/tasks/components/tasks-page-client.tsx create mode 100644 site/app/tasks/page.tsx create mode 100644 site/bun.lock create mode 100644 site/components.json create mode 100644 site/components/pending-review-card.tsx create mode 100644 site/components/query-provider.tsx create mode 100644 site/components/theme-provider.tsx create mode 100644 site/components/theme-toggle.tsx create mode 100644 site/components/ui/badge.tsx create mode 100644 site/components/ui/button.tsx create mode 100644 site/components/ui/checkbox.tsx create mode 100644 site/components/ui/command.tsx create mode 100644 site/components/ui/dialog.tsx create mode 100644 site/components/ui/drawer.tsx create mode 100644 site/components/ui/hover-card.tsx create mode 100644 site/components/ui/popover.tsx create mode 100644 site/components/ui/scroll-area.tsx create mode 100644 site/components/ui/select.tsx create mode 100644 site/components/ui/sheet.tsx create mode 100644 site/components/ui/skeleton.tsx create mode 100644 site/components/ui/tabs.tsx create mode 100644 site/lib/http-error.ts create mode 100644 site/lib/utils.ts create mode 100644 site/next.config.ts create mode 100644 site/package.json create mode 100644 site/postcss.config.mjs create mode 100644 site/scripts/check-trajectory.ts create mode 100644 site/scripts/compute-tasks.ts create mode 100644 site/tsconfig.json create mode 100644 site/types/result.d.ts create mode 120000 site/zealt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..828b276 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,69 @@ +name: Deploy Next.js site to Pages + +on: + push: + branches: ["main"] + pull_request: + types: [opened, reopened, synchronize, closed] + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --frozen-lockfile + working-directory: site + + - name: Setup Pages + id: setup_pages + uses: actions/configure-pages@v5 + + - name: Set base uri + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}/pr-preview/pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" + else + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}" >> "$GITHUB_ENV" + fi + + - name: Build with Next.js + run: bun run build + working-directory: site + + - name: Deploy preview + if: github.event_name == 'pull_request' + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site/out + + - name: Deploy production + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + clean-exclude: pr-preview/ + force: false + folder: site/out diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0cf2fa4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +__pycache__/ +*.py[cod] +*$py.class + +# testing +coverage + +# next.js +.next/ + +# The `out` directory should not be ignored by version control +out/ + +# production +build + +# misc +.DS_Store +*.pem +*~ +\#* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.zealt/config.json b/.zealt/config.json new file mode 100644 index 0000000..dd30f00 --- /dev/null +++ b/.zealt/config.json @@ -0,0 +1,5 @@ +{ + "title": "TanStack Benchmark", + "description": "Performance results of AI coding models on TanStack tasks, measuring success rate and execution time with high precision.", + "github_repo": "https://github.com/kweizh/tanstack-benchmark" +} \ No newline at end of file diff --git a/README.md b/README.md index 1b872c5..dc6b88c 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# tanstack-benchmark \ No newline at end of file + +# TanStack Benchmark + +This repository contains benchmarks for evaluating AI models on **TanStack**. + +You can view the evaluation reports at [https://kweizh.github.io/tanstack-benchmark/](https://kweizh.github.io/tanstack-benchmark/). + +## Project Structure + +- `tasks/`: Contains the benchmark tasks, each with its own instructions. +- `jobs/`: Stores the results of benchmark runs. +- `site/`: A Next.js application to visualize benchmark results. + +## Getting Started + +This benchmark is evaluated using the [Harbor framework](https://github.com/harbor-framework/harbor) and the [Pochi agent](https://github.com/TabbyML/pochi). + +### Running Evaluation + +You can run the evaluation using the Harbor CLI. Here is an example: + +```bash +harbor run \ + --agent codex \ + --model "gpt-5.2-codex" \ + --env daytona \ + --path ./tasks \ + --n-attempts 1 \ + --max-retries 5 \ + --n-concurrent 5 \ + --retry-include RuntimeError \ + --retry-include DaytonaError \ + --retry-include AgentTimeoutError +``` + +### Evaluation Details + +Before starting the evaluation, you should set the necessary environment variables for your chosen agent. +For example, if using Pochi, you should export `POCHI_API_KEY`. + +Evaluation can be run locally with Docker (default), or using [Daytona.io](https://www.daytona.io/) by setting `--env daytona`. + +When running with Daytona, please note that Daytona blocks some network access for tier 1 and tier 2 users. If you meet any network issues, please refer to [Daytona network limits](https://www.daytona.io/docs/en/network-limits/). + +--- +Generated by [Zealt](https://github.com/TabbyML/zealt) diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..2501f3f --- /dev/null +++ b/plan.md @@ -0,0 +1,136 @@ +# TanStack Benchmark Research Report + +## 1. Library Overview + +**Description**: TanStack is a collection of high-quality, open-source headless libraries for web development. It focuses on the "hard parts" of application development: state management, routing, data grids, and forms. The ecosystem's flagship is **TanStack Start**, a full-stack React framework that integrates these libraries into a cohesive, type-safe development experience. + +**Ecosystem Role**: TanStack provides the foundational "engine" for modern web apps. Unlike opinionated UI kits, TanStack libraries are "headless," providing logic and state without markup, allowing developers to use any UI library (e.g., Tailwind, Shadcn UI) while maintaining strict type safety from the database to the browser. + +**Project Setup**: +The recommended way to initialize a full-stack project is via the TanStack CLI: +```bash +npx @tanstack/cli@latest create +``` +Standard project structure for TanStack Start: +- `app/routes/`: File-based routing directory. +- `app/routeTree.gen.ts`: Automatically generated type-safe route tree. +- `app/ssr.tsx` & `app/client.tsx`: Entry points for server and client. +- `app/router.tsx`: Shared router configuration. + +--- + +## 2. Core Primitives & APIs + +### TanStack Query (Server State) +- **Concept**: Manages asynchronous state (fetching, caching, synchronization). +- **Core APIs**: `useQuery`, `useMutation`, `queryOptions`. +- **Code Snippet**: +```typescript +const postsQuery = queryOptions({ + queryKey: ['posts'], + queryFn: () => fetch('/api/posts').then(r => r.json()), +}) + +function Posts() { + const { data } = useQuery(postsQuery) + return +} +``` +- **Docs**: [TanStack Query Reference](https://tanstack.com/query/latest/docs/framework/react/reference/useQuery) + +### TanStack Router (Type-Safe Routing) +- **Concept**: File-based routing with 100% type safety for paths, params, and search state. +- **Core APIs**: `createFileRoute`, `Link`, `useLoaderData`. +- **Code Snippet**: +```typescript +// routes/posts.$postId.tsx +export const Route = createFileRoute('/posts/$postId')({ + loader: ({ params }) => fetchPost(params.postId), + component: PostComponent, +}) + +function PostComponent() { + const data = Route.useLoaderData() + return
{data.title}
+} +``` +- **Docs**: [TanStack Router Guide](https://tanstack.com/router/latest/docs/routing/file-based-routing) + +### TanStack Table (Headless Data Grid) +- **Concept**: Logic engine for complex tables (sorting, filtering, pagination). +- **Core APIs**: `useReactTable`, `createColumnHelper`. +- **Code Snippet**: +```typescript +const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), +}) + +// Render using table.getHeaderGroups() and table.getRowModel().rows +``` +- **Docs**: [TanStack Table Core APIs](https://tanstack.com/table/latest/docs/api/core/table) + +### TanStack Start (Full-Stack) +- **Concept**: SSR, Streaming, and Server Functions (RPCs). +- **Core APIs**: `createServerFn`, `createFileRoute`. +- **Code Snippet**: +```typescript +const updateCount = createServerFn({ method: 'POST' }) + .validator((d: number) => d) + .handler(async ({ data }) => { + // Server-side logic (DB update, etc.) + return { success: true } + }) +``` +- **Docs**: [TanStack Start Overview](https://tanstack.com/start/latest/docs/framework/react/overview) + +--- + +## 3. Real-World Use Cases & Templates + +- **SaaS Admin Dashboards**: Combining **Table** for data grids, **Query** for server state, and **Router** for deeply nested layouts and URL-driven filters. +- **E-commerce Product Filters**: Using **Router's Search Param Validation** (Zod-integrated) to manage complex filtering states in the URL. +- **AI-Powered Chat Apps**: Using **TanStack AI** for streaming responses and tool-calling with approval workflows. +- **Template**: [Trellaux](https://github.com/TanStack/router/tree/main/examples/react/start-trellaux) - A full-stack Trello clone showing Start, Query, and complex drag-and-drop state. + +--- + +## 4. Developer Friction Points + +1. **Router Type Generation**: The `routeTree.gen.ts` file must be generated via a background watcher. AI agents often struggle to trigger this generation or understand that the file is missing during initial setup. +2. **SSR Hydration with Query**: Misconfiguring `staleTime` or `gcTime` during SSR can lead to "Hydration Mismatch" errors where the server and client data differ. +3. **Table Column Typing**: Defining complex columns with custom cell renderers and meta-data requires deep understanding of TypeScript generics, often leading to "Type instantiation is excessively deep" errors. +4. **Headless Complexity**: The "Headless" nature means no default UI. Implementing a basic accessible table or form requires significant boilerplate (e.g., mapping over header groups). + +--- + +## 5. Evaluation Ideas + +### Simple +- Create a type-safe navigation menu with active link highlighting using TanStack Router. +- Implement a basic "Todo" list that fetches and creates items using TanStack Query. + +### Intermediate +- Build a paginated data table with TanStack Table including server-side sorting. +- Create a multi-step registration form with TanStack Form and Zod validation. +- Implement a "Search" page where all filters (query, category, price range) are synced to the URL via TanStack Router. + +### Complex +- Build a full-stack "Counter" app in TanStack Start using Server Functions and SQLite. +- Implement an "Optimistic Update" flow for a nested comment system using TanStack Query. +- Create an AI Chat interface using TanStack AI that includes a "Tool Approval" step for database writes. + +--- + +## 6. Sources + +1. [TanStack Official Site](https://tanstack.com/) - Main ecosystem hub. +2. [TanStack Query Docs](https://tanstack.com/query/latest/docs) - Server-state management reference. +3. [TanStack Router Docs](https://tanstack.com/router/latest/docs) - Type-safe routing and URL state reference. +4. [TanStack Start Docs](https://tanstack.com/start/latest/docs) - Full-stack framework and SSR reference. +5. [TanStack Table Docs](https://tanstack.com/table/latest/docs) - Headless table engine reference. +6. [TanStack Form Docs](https://tanstack.com/form/latest/docs) - Headless form state reference. +7. [TanStack AI Overview](https://tanstack.com/ai/latest/docs/getting-started/overview) - AI SDK and tool calling reference. +8. [TanStack GitHub Repository](https://github.com/TanStack) - Source code and community examples. \ No newline at end of file diff --git a/scratchpad/pending-tasks/headless_table_initialization_and_typing/instruction.md b/scratchpad/pending-tasks/headless_table_initialization_and_typing/instruction.md new file mode 100644 index 0000000..707f0cb --- /dev/null +++ b/scratchpad/pending-tasks/headless_table_initialization_and_typing/instruction.md @@ -0,0 +1,8 @@ +TanStack Table is a headless logic engine, requiring strict typing for complex columns and manual boilerplate mapping to render the UI grid without "excessively deep" type instantiation errors. + +You need to instantiate a data table using `useReactTable` and `createColumnHelper` for a specific `Employee` interface, and build the standard HTML table markup to display the data. + +**Constraints:** +- Must render the standard HTML ``, ``, and `` structure by manually mapping over `table.getHeaderGroups()` and `table.getRowModel().rows`. +- Must create at least one custom cell renderer (e.g., formatting a date or rendering an Action button). +- Do NOT use any pre-built UI library table components (like MUI DataGrid or AG Grid). \ No newline at end of file diff --git a/scratchpad/pending-tasks/optimistic_updates_with_tanstack_query/instruction.md b/scratchpad/pending-tasks/optimistic_updates_with_tanstack_query/instruction.md new file mode 100644 index 0000000..ed75dd0 --- /dev/null +++ b/scratchpad/pending-tasks/optimistic_updates_with_tanstack_query/instruction.md @@ -0,0 +1,8 @@ +TanStack Query manages asynchronous server state and allows optimistic updates for a snappy user experience during data mutations. + +You need to implement a `useMutation` hook for adding a new "Post" that optimistically updates the local cache before the network request finishes, and rolls back if the network request fails. + +**Constraints:** +- Must synchronously update the cache array for the `['posts']` query key inside the `onMutate` callback. +- Must implement the rollback logic in the `onError` callback using the context returned from `onMutate`. +- Must trigger a background refetch via `onSettled` to ensure synchronization. \ No newline at end of file diff --git a/scratchpad/pending-tasks/resolving_query_ssr_hydration_mismatch/instruction.md b/scratchpad/pending-tasks/resolving_query_ssr_hydration_mismatch/instruction.md new file mode 100644 index 0000000..7dc8113 --- /dev/null +++ b/scratchpad/pending-tasks/resolving_query_ssr_hydration_mismatch/instruction.md @@ -0,0 +1,8 @@ +When combining TanStack Query with Server-Side Rendering (SSR), misconfiguring caching durations often leads to "Hydration Mismatch" errors where server HTML and initial client state differ. + +You need to configure the global `QueryClient` initialization in a TanStack Start `app.tsx` file to properly align server and client caching behavior and prevent immediate refetching on hydration. + +**Constraints:** +- Must set a default `staleTime` strictly greater than `0` (e.g., `60 * 1000`) in the default options to prevent instant invalidation. +- Must conditionally initialize the `QueryClient` so it is not shared across users during SSR, but remains a singleton on the client. +- Do NOT alter any specific component's `useQuery` configurations; apply the fix at the root provider level. \ No newline at end of file diff --git a/scratchpad/pending-tasks/server_function_creation_in_tanstack_start/instruction.md b/scratchpad/pending-tasks/server_function_creation_in_tanstack_start/instruction.md new file mode 100644 index 0000000..ecedc21 --- /dev/null +++ b/scratchpad/pending-tasks/server_function_creation_in_tanstack_start/instruction.md @@ -0,0 +1,8 @@ +TanStack Start supports full-stack RPCs via Server Functions, allowing secure server-side logic and database operations to be called directly from client components. + +You need to create a `updateCount` server function in a TanStack Start application using `createServerFn` that accepts an increment amount, executes dummy server-side logic, and returns a success payload. + +**Constraints:** +- Must use the `.validator()` method to ensure the payload type is strictly an integer. +- Must configure the server function method strictly as `POST`. +- Do NOT write directly to a real database; use a mock asynchronous return. \ No newline at end of file diff --git a/scratchpad/pending-tasks/synchronized_url_search_parameters/instruction.md b/scratchpad/pending-tasks/synchronized_url_search_parameters/instruction.md new file mode 100644 index 0000000..6aa6a4d --- /dev/null +++ b/scratchpad/pending-tasks/synchronized_url_search_parameters/instruction.md @@ -0,0 +1,8 @@ +TanStack Router allows deep integration with validation libraries like Zod to manage complex, type-safe filtering states directly in the URL search params. + +You need to create a file route definition for `/products` that strictly types and validates URL search parameters (`category` as a string, `inStock` as a boolean) using the `validateSearch` option. + +**Constraints:** +- Must use Zod (`z.object`) to validate the search parameters. +- Must provide fallback default values (`category` defaults to `"all"`, `inStock` defaults to `true`) if the parameters are omitted from the URL. +- Do NOT generate the `routeTree.gen.ts` file manually. \ No newline at end of file diff --git a/scratchpad/pending-tasks/type_safe_navigation_menu/instruction.md b/scratchpad/pending-tasks/type_safe_navigation_menu/instruction.md new file mode 100644 index 0000000..f99bf51 --- /dev/null +++ b/scratchpad/pending-tasks/type_safe_navigation_menu/instruction.md @@ -0,0 +1,7 @@ +TanStack Router provides file-based routing and 100% type safety for paths, ensuring broken links are caught at compile time. + +You need to implement a navigation menu component `Nav.tsx` utilizing TanStack Router's `` component to navigate between `/`, `/posts`, and `/settings` in a standard React environment. + +**Constraints:** +- Must apply a specific CSS class `active-link` to the currently active route using the `activeProps` API. +- Do NOT use `react-router-dom` or standard HTML `` tags for internal routing. \ No newline at end of file diff --git a/site/app/(home)/components/leaderboard-table.tsx b/site/app/(home)/components/leaderboard-table.tsx new file mode 100644 index 0000000..d10b1f0 --- /dev/null +++ b/site/app/(home)/components/leaderboard-table.tsx @@ -0,0 +1,183 @@ +"use client"; + +import { useState, useMemo, useEffect } from "react"; +import { useRouter } from "next/navigation"; +import { Search, Trophy, ListTree } from "lucide-react"; +import { clsx, type ClassValue } from "clsx"; +import { twMerge } from "tailwind-merge"; +import Link from "next/link"; +import zealtConfig from "@/zealt/config.json"; + +function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} + +export interface LeaderboardEntry { + id: string; + model: string; + rawModel: string; + agent: string; + passedEvals: number; + successRate: number; + avgLatency: number; + isNew?: boolean; +} + +function ProgressBar({ value, colorClass }: { value: number; colorClass: string }) { + return ( +
+
+
+ ); +} + +function ScoreCell({ value }: { value: number }) { + let colorClass = "bg-primary"; + let textClass = "text-muted-foreground"; + + if (value >= 90) { + colorClass = "bg-emerald-500"; + textClass = "text-emerald-500 font-bold"; + } else if (value >= 75) { + colorClass = "bg-blue-500"; + textClass = "text-blue-500 font-medium"; + } else if (value >= 60) { + colorClass = "bg-amber-500"; + textClass = "text-amber-500"; + } else { + colorClass = "bg-red-500"; + textClass = "text-red-500"; + } + + return ( +
+ {value}% + +
+ ); +} + +export default function LeaderboardTable({ data }: { data: LeaderboardEntry[] }) { + const router = useRouter(); + const [devMode, setDevMode] = useState(process.env.NODE_ENV === "development"); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevMode(isDev); + }, []); + + const filteredData = useMemo(() => { + let processedData = data; + const config = zealtConfig as { pending_models?: string[] }; + + if (!devMode && config.pending_models && config.pending_models.length > 0) { + processedData = processedData.filter((item) => + !config.pending_models!.includes(item.rawModel) + ); + } + + if (searchQuery) { + const query = searchQuery.toLowerCase(); + processedData = processedData.filter(item => + item.model.toLowerCase().includes(query) + ); + } + + return processedData; + }, [data, searchQuery, devMode]); + + return ( + <> + {/* Controls & Filters */} +
+

+ Model Performance +

+ +
+ + + View Tasks + + +
+ + setSearchQuery(e.target.value)} + className="pl-9 pr-4 py-2 bg-card border border-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary/20 w-full sm:w-64 transition-all" + /> +
+
+
+ + {/* Leaderboard Table */} +
+
+
+ + + + + + + + + + {filteredData.length === 0 ? ( + + + + ) : ( + filteredData.map((row, index) => ( + router.push(`./tasks?model=${encodeURIComponent(row.model)}`)} + className="group hover:bg-secondary/30 transition-colors duration-200 cursor-pointer" + > + + + + + + )))} + +
ModelPassedAvg DurationSuccess Rate
+ No results found matching your search. +
+ #{index + 1} +
+ + {row.model} + {index === 0 && } + {row.isNew && ( + + NEW + + )} + +
+
+ {row.passedEvals} + + {row.avgLatency > 0 ? `${row.avgLatency.toFixed(1)}s` : '-'} + +
+ +
+
+ + + + ); +} diff --git a/site/app/(home)/page.tsx b/site/app/(home)/page.tsx new file mode 100644 index 0000000..d617123 --- /dev/null +++ b/site/app/(home)/page.tsx @@ -0,0 +1,176 @@ +import { Suspense } from "react"; +import { Github, Terminal, ClipboardList, ListTree } from "lucide-react"; +import Link from "next/link"; +import tasksData from "@/zealt/tasks.json"; +import pendingTasksData from "@/zealt/pending-tasks.json"; +import zealtConfig from "@/zealt/config.json"; +import PendingReviewCard from "@/components/pending-review-card"; +import LeaderboardTable, { type LeaderboardEntry } from "./components/leaderboard-table"; + +type TaskTrial = { + agent: string; + model: string; + passed: boolean; + latency_sec: number | null; +}; + +type TaskValue = { + trials?: TaskTrial[]; +}; + +type PendingTasksValue = { + 'pending-tasks'?: number; +}; + +export default function Home() { + const totalTasks = Object.keys(tasksData as Record).length; + const hasTasks = totalTasks > 0; + const pendingSampleCases = Math.max( + 0, + Number((pendingTasksData as PendingTasksValue)['pending-tasks'] ?? 0), + ); + + // Process tasks.json to compute leaderboard stats directly on the server + const statsMap = new Map(); + + Object.values(tasksData as Record).forEach((taskValue) => { + let trials: TaskTrial[] = []; + if (Array.isArray(taskValue)) { + trials = taskValue as TaskTrial[]; + } else if (typeof taskValue === "object" && taskValue !== null) { + const task = taskValue as TaskValue; + trials = Array.isArray(task.trials) ? task.trials : []; + } + + trials.forEach((trial) => { + // Simplify model name + const modelName = trial.model.split('/').pop() || trial.model; + const agentName = trial.agent.charAt(0).toUpperCase() + trial.agent.slice(1); + + const key = `${modelName}-${agentName}`; + + if (!statsMap.has(key)) { + statsMap.set(key, { + passed: 0, + total: 0, + totalLatency: 0, + latencyCount: 0, + model: modelName, + rawModel: trial.model, + agent: agentName + }); + } + + const stats = statsMap.get(key); + if (!stats) { + return; + } + stats.total += 1; + if (trial.passed) { + stats.passed += 1; + } + if (trial.latency_sec) { + stats.totalLatency += trial.latency_sec; + stats.latencyCount += 1; + } + }); + }); + + const data: LeaderboardEntry[] = Array.from(statsMap.values()) + .map((stats, index) => { + const successRate = stats.total > 0 ? Math.round((stats.passed / stats.total) * 100) : 0; + const avgLatency = stats.latencyCount > 0 ? stats.totalLatency / stats.latencyCount : 0; + return { + id: String(index + 1), + model: stats.model, + rawModel: stats.rawModel, + agent: stats.agent, + passedEvals: stats.passed, + successRate: successRate, + avgLatency: avgLatency, + }; + }) + .sort((a, b) => b.successRate - a.successRate); + + // Re-assign IDs based on sorted order and adjust isNew + data.forEach((item, index) => { + item.id = String(index + 1); + item.isNew = index === 0; // Keeping the original visual effect for the top item + }); + + return ( +
+ {/* Background Gradient Effect */} +
+ +
+ {/* Header Section */} +
+
+ + Live Benchmarks +
+ +

+ {zealtConfig.title} +

+ +

+ {zealtConfig.description} +

+ +
+ + + View on GitHub + + {data.length > 0 && ( + <> +
+ + + Total tasks: {totalTasks} + + + )} +
+ + + Last run: {new Date().toLocaleDateString()} + +
+
+ + {!hasTasks ? ( + + ) : data.length === 0 ? ( +
+

No evaluation data yet

+
+ + + View Tasks + +
+
+ ) : ( + // Client Component for Interactive Table + Loading leaderboard...
}> + + + )} +
+ + ); +} diff --git a/site/app/globals.css b/site/app/globals.css new file mode 100644 index 0000000..437c041 --- /dev/null +++ b/site/app/globals.css @@ -0,0 +1,149 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); + --color-background: var(--background); + --color-foreground: var(--foreground); + --color-card: var(--card); + --color-card-foreground: var(--card-foreground); + --color-popover: var(--popover); + --color-popover-foreground: var(--popover-foreground); + --color-primary: var(--primary); + --color-primary-foreground: var(--primary-foreground); + --color-secondary: var(--secondary); + --color-secondary-foreground: var(--secondary-foreground); + --color-muted: var(--muted); + --color-muted-foreground: var(--muted-foreground); + --color-accent: var(--accent); + --color-accent-foreground: var(--accent-foreground); + --color-destructive: var(--destructive); + --color-border: var(--border); + --color-input: var(--input); + --color-ring: var(--ring); + --color-chart-1: var(--chart-1); + --color-chart-2: var(--chart-2); + --color-chart-3: var(--chart-3); + --color-chart-4: var(--chart-4); + --color-chart-5: var(--chart-5); + --color-sidebar: var(--sidebar); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.141 0.005 285.823); + --card: oklch(1 0 0); + --card-foreground: oklch(0.141 0.005 285.823); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.141 0.005 285.823); + --primary: oklch(0.21 0.006 285.885); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.967 0.001 286.375); + --secondary-foreground: oklch(0.21 0.006 285.885); + --muted: oklch(0.967 0.001 286.375); + --muted-foreground: oklch(0.552 0.016 285.938); + --accent: oklch(0.967 0.001 286.375); + --accent-foreground: oklch(0.21 0.006 285.885); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.92 0.004 286.32); + --input: oklch(0.92 0.004 286.32); + --ring: oklch(0.705 0.015 286.067); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.141 0.005 285.823); + --sidebar-primary: oklch(0.21 0.006 285.885); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.967 0.001 286.375); + --sidebar-accent-foreground: oklch(0.21 0.006 285.885); + --sidebar-border: oklch(0.92 0.004 286.32); + --sidebar-ring: oklch(0.705 0.015 286.067); +} + +.dark { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.21 0.006 285.885); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.92 0.004 286.32); + --primary-foreground: oklch(0.21 0.006 285.885); + --secondary: oklch(0.274 0.006 286.033); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --accent: oklch(0.274 0.006 286.033); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.552 0.016 285.938); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.21 0.006 285.885); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.274 0.006 286.033); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.552 0.016 285.938); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} + +@layer utilities { + .custom-scrollbar { + scrollbar-width: thin; + scrollbar-color: var(--border) transparent; + } + + .custom-scrollbar::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + .custom-scrollbar::-webkit-scrollbar-track { + background: transparent; + } + + .custom-scrollbar::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; + } + + .custom-scrollbar::-webkit-scrollbar-thumb:hover { + background: color-mix(in oklch, var(--foreground) 30%, transparent); + } +} \ No newline at end of file diff --git a/site/app/layout.tsx b/site/app/layout.tsx new file mode 100644 index 0000000..3ee8c0a --- /dev/null +++ b/site/app/layout.tsx @@ -0,0 +1,37 @@ +import "./globals.css"; + +import type { Metadata } from 'next'; +import { Inter } from "next/font/google"; +import zealtConfig from "@/zealt/config.json"; +import { ThemeProvider } from "@/components/theme-provider"; +import { ThemeToggle } from "@/components/theme-toggle"; +import { AppQueryProvider } from "@/components/query-provider"; + +const inter = Inter({ subsets: ["latin"] }); + +export const metadata: Metadata = { + title: zealtConfig.title, + description: zealtConfig.description, +}; + + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + +
+ +
+ {children} +
+
+ + + ); +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx new file mode 100644 index 0000000..bf9202c --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx @@ -0,0 +1,265 @@ +"use client"; + +import { useMemo } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { ChevronRight, File, Folder } from "lucide-react"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + fetchLogText, + LogContentSkeleton, + LogErrorView, + getLogErrorMessage, +} from "./trajectory-page"; + +export type ArtifactNode = { + name: string; + type: "file" | "dir"; + path: string; + children?: ArtifactNode[]; +}; + +export type ArtifactNodeWithUrl = + | { name: string; type: "file"; path: string; url: string } + | { name: string; type: "dir"; path: string; children: ArtifactNodeWithUrl[] }; + +type ArtifactsPanelProps = { + artifactTree: ArtifactNodeWithUrl[]; +}; + +export function ArtifactsPanel({ artifactTree }: ArtifactsPanelProps) { + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + + const queryArtifact = searchParams.get("artifact"); + const queryPath = searchParams.get("path"); + + const activeArtifact = useMemo(() => { + if (queryArtifact) { + const found = artifactTree.find((node) => node.name === queryArtifact); + if (found) return found; + } + return artifactTree[0] ?? null; + }, [artifactTree, queryArtifact]); + + const updateParams = (next: { artifact?: string | null; path?: string | null }) => { + const params = new URLSearchParams(searchParams.toString()); + if (next.artifact === null) { + params.delete("artifact"); + } else if (next.artifact !== undefined) { + params.set("artifact", next.artifact); + } + if (next.path === null) { + params.delete("path"); + } else if (next.path !== undefined) { + params.set("path", next.path); + } + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + + const handleSelectArtifact = (name: string) => { + updateParams({ artifact: name, path: null }); + }; + + if (!activeArtifact) { + return ( +
+ No artifacts available. +
+ ); + } + + return ( +
+
+
+ {artifactTree.map((node) => ( + + ))} +
+
+ +
+ {activeArtifact.type === "file" ? ( + + ) : ( + updateParams({ path: path ?? null })} + /> + )} +
+
+ ); +} + +type DirectoryArtifactViewProps = { + artifact: Extract; + currentPath: string | null; + onPathChange: (path: string | null) => void; +}; + +function DirectoryArtifactView({ artifact, currentPath, onPathChange }: DirectoryArtifactViewProps) { + // Resolve the active path. If currentPath isn't valid for this artifact, fall back to the artifact root. + const resolved = useMemo(() => resolvePath(artifact, currentPath), [artifact, currentPath]); + const { folder, selectedFile, breadcrumbDirs } = resolved; + + return ( +
+
+ {breadcrumbDirs.map((dir, idx) => { + const isLast = idx === breadcrumbDirs.length - 1 && !selectedFile; + return ( + + + {(idx < breadcrumbDirs.length - 1 || selectedFile) && ( + + )} + + ); + })} + {selectedFile && ( + {selectedFile.name} + )} +
+ +
+
+ {folder.children.length === 0 && ( + Empty directory. + )} + {folder.children.map((child) => { + const isActive = child.type === "file" && selectedFile?.path === child.path; + return ( + + ); + })} +
+
+ +
+ {selectedFile ? ( + + ) : ( +
+ Select a file to view its contents. +
+ )} +
+
+ ); +} + +function FileViewer({ url }: { url: string }) { + const query = useQuery({ + queryKey: ["artifact-file", url], + queryFn: () => fetchLogText(url), + }); + + return ( + +
+ {query.isPending || query.isFetching ? ( + + ) : query.isError ? ( + void query.refetch()} + /> + ) : query.data ? ( +
+            {query.data}
+          
+ ) : ( +

Empty file.

+ )} +
+
+ ); +} + +type ResolvedPath = { + folder: Extract; + selectedFile: Extract | null; + breadcrumbDirs: { name: string; path: string }[]; +}; + +function resolvePath( + root: Extract, + targetPath: string | null, +): ResolvedPath { + const breadcrumbDirs: { name: string; path: string }[] = [{ name: root.name, path: root.path }]; + let folder = root; + let selectedFile: Extract | null = null; + + if (!targetPath || !targetPath.startsWith(`${root.path}/`)) { + return { folder, selectedFile, breadcrumbDirs }; + } + + const remainder = targetPath.slice(root.path.length + 1); + const segments = remainder.split("/").filter(Boolean); + + let currentPath = root.path; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]; + currentPath = `${currentPath}/${segment}`; + const child = folder.children.find((c) => c.name === segment); + if (!child) { + // Path doesn't exist; bail and show what we resolved so far. + return { folder, selectedFile: null, breadcrumbDirs }; + } + if (child.type === "dir") { + folder = child; + breadcrumbDirs.push({ name: child.name, path: child.path }); + } else { + // File at the end of the path. + selectedFile = child; + // Don't push the file as a breadcrumb dir; the caller renders it separately. + break; + } + } + + return { folder, selectedFile, breadcrumbDirs }; +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx new file mode 100644 index 0000000..0fab8fc --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx @@ -0,0 +1,404 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useTheme } from "next-themes"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Button } from "@/components/ui/button"; +import { HttpError } from "@/lib/http-error"; +import { ArtifactsPanel, type ArtifactNodeWithUrl } from "./artifacts-panel"; + +export type TabConfig = { + value: string; + label: React.ReactNode; +}; + +type TrajectoryPageProps = { + trajectoryUrl: string; + browserVerificationUrls: { name: string; url: string }[]; + fallbackUrl: string; + stderrLogUrl: string | null; + verifierLogUrl: string | null; + tabsConfig: TabConfig[]; + artifactTree?: ArtifactNodeWithUrl[]; +}; + +export async function fetchLogText(url: string): Promise { + let response: Response; + try { + response = await fetch(url, { cache: "force-cache" }); + } catch (_e) { + throw new HttpError("Network request failed."); + } + + if (!response.ok) { + throw new HttpError(`Request failed with status ${response.status}.`, { status: response.status }); + } + + return response.text(); +} + +export function TrajectoryPage({ + trajectoryUrl, + browserVerificationUrls, + fallbackUrl, + stderrLogUrl, + verifierLogUrl, + tabsConfig, + artifactTree, +}: TrajectoryPageProps) { + const { resolvedTheme } = useTheme(); + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + const [mounted, setMounted] = useState(false); + const [iframeLoading, setIframeLoading] = useState(true); + const [devModeEnabled, setDevModeEnabled] = useState(process.env.NODE_ENV === "development"); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevModeEnabled(isDev); + }, []); + + const visibleTabsConfig = useMemo( + () => + devModeEnabled + ? tabsConfig + : tabsConfig.filter((t) => t.value !== "artifacts"), + [tabsConfig, devModeEnabled], + ); + const validTabs = visibleTabsConfig.map((t) => t.value); + const [activeTab, setActiveTab] = useState(() => { + const queryTab = searchParams.get("tab"); + return queryTab && validTabs.includes(queryTab) ? queryTab : validTabs[0]; + }); + + const handleTabChange = (value: string) => { + setActiveTab(value); + const params = new URLSearchParams(searchParams.toString()); + params.set("tab", value); + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + const [browserIframeLoading, setBrowserIframeLoading] = useState>({}); + const [activeBrowserVerificationTab, setActiveBrowserVerificationTab] = useState( + browserVerificationUrls[0]?.name || "", + ); + + const iframeTheme = mounted && resolvedTheme === "light" ? "light" : "dark"; + + const iframeUrl = useMemo(() => { + const url = new URL(trajectoryUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [trajectoryUrl, iframeTheme]); + + const activeBrowserVerificationBaseUrl = useMemo(() => { + const testCase = browserVerificationUrls.find((tc) => tc.name === activeBrowserVerificationTab); + return testCase ? testCase.url : null; + }, [browserVerificationUrls, activeBrowserVerificationTab]); + + const activeBrowserVerificationUrl = useMemo(() => { + if (!activeBrowserVerificationBaseUrl) { + return null; + } + + const url = new URL(activeBrowserVerificationBaseUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [activeBrowserVerificationBaseUrl, iframeTheme]); + + useEffect(() => { + setMounted(true); + }, []); + + useEffect(() => { + if (!mounted || !activeBrowserVerificationBaseUrl) { + return; + } + + setBrowserIframeLoading((prev) => ({ ...prev, [activeBrowserVerificationTab]: true })); + }, [activeBrowserVerificationBaseUrl, activeBrowserVerificationTab, mounted]); + + const stderrQuery = useQuery({ + queryKey: ["trajectory-stderr", stderrLogUrl], + enabled: Boolean(stderrLogUrl), + queryFn: async () => { + if (!stderrLogUrl) { + return null; + } + + return fetchLogText(stderrLogUrl); + }, + }); + + const verifierQuery = useQuery({ + queryKey: ["trajectory-verifier", verifierLogUrl], + enabled: Boolean(verifierLogUrl), + queryFn: async () => { + if (!verifierLogUrl) { + return null; + } + + return fetchLogText(verifierLogUrl); + }, + }); + + const handleIframeLoad = () => { + setIframeLoading(false); + }; + + const handleBrowserIframeLoad = (testCaseName: string) => { + setBrowserIframeLoading((prev) => ({ ...prev, [testCaseName]: false })); + }; + + const handleIframeError = () => { + window.location.replace(fallbackUrl); + }; + + const renderLogContent = ( + text: string | null | undefined, + isLoading: boolean, + isError: boolean, + error: unknown, + onRetry: () => void, + emptyMessage: string, + ) => { + if (isLoading) { + return ; + } + + if (isError) { + return ; + } + + if (!text) { + return

{emptyMessage}

; + } + + return ( +
+        {text}
+      
+ ); + }; + + return ( +
+
+ +
+ + {visibleTabsConfig.map((tab) => ( + + {tab.label} + + ))} + +
+ + +
+ +
+ {mounted && ( +