From 655bf1714d80937edef84ccef4dca77e4bbed380 Mon Sep 17 00:00:00 2001 From: "zealt-staging[bot]" <264479255+zealt-staging[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:49:38 +0000 Subject: [PATCH] Initialize repository with benchmark template and tasks --- .github/workflows/deploy.yml | 69 ++ .gitignore | 49 + .zealt/config.json | 5 + README.md | 47 +- plan.md | 103 ++ .../instruction.md | 9 + .../instruction.md | 9 + .../instruction.md | 9 + .../instruction.md | 9 + .../instruction.md | 9 + .../instruction.md | 9 + .../(home)/components/leaderboard-table.tsx | 183 +++ site/app/(home)/page.tsx | 176 +++ site/app/globals.css | 149 +++ site/app/layout.tsx | 37 + .../trajectory/components/artifacts-panel.tsx | 265 ++++ .../trajectory/components/trajectory-page.tsx | 404 ++++++ .../tasks/[name]/[jobId]/trajectory/page.tsx | 414 +++++++ site/app/tasks/components/back-to-top.tsx | 41 + site/app/tasks/components/multi-select.tsx | 104 ++ .../tasks/components/tasks-page-client.tsx | 724 +++++++++++ site/app/tasks/page.tsx | 142 +++ site/bun.lock | 1082 +++++++++++++++++ site/components.json | 23 + site/components/pending-review-card.tsx | 44 + site/components/query-provider.tsx | 21 + site/components/theme-provider.tsx | 27 + site/components/theme-toggle.tsx | 32 + site/components/ui/badge.tsx | 48 + site/components/ui/button.tsx | 64 + site/components/ui/checkbox.tsx | 32 + site/components/ui/command.tsx | 184 +++ site/components/ui/dialog.tsx | 158 +++ site/components/ui/drawer.tsx | 131 ++ site/components/ui/hover-card.tsx | 44 + site/components/ui/popover.tsx | 89 ++ site/components/ui/scroll-area.tsx | 59 + site/components/ui/select.tsx | 190 +++ site/components/ui/sheet.tsx | 153 +++ site/components/ui/skeleton.tsx | 16 + site/components/ui/tabs.tsx | 69 ++ site/lib/http-error.ts | 8 + site/lib/utils.ts | 6 + site/next.config.ts | 17 + site/package.json | 34 + site/postcss.config.mjs | 7 + site/scripts/check-trajectory.ts | 118 ++ site/scripts/compute-tasks.ts | 252 ++++ site/tsconfig.json | 41 + site/types/result.d.ts | 30 + site/zealt | 1 + 51 files changed, 5945 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 .zealt/config.json create mode 100644 plan.md create mode 100644 scratchpad/pending-tasks/basic_aggregation_and_sorting_chart/instruction.md create mode 100644 scratchpad/pending-tasks/focus_and_context_interactive_chart/instruction.md create mode 100644 scratchpad/pending-tasks/independent_facet_scales_with_shared_selection/instruction.md create mode 100644 scratchpad/pending-tasks/large_dataset_handling_with_vegafusion/instruction.md create mode 100644 scratchpad/pending-tasks/layered_scatter_plot_with_regression_line/instruction.md create mode 100644 scratchpad/pending-tasks/tri_view_cross_filtering_dashboard/instruction.md create mode 100644 site/app/(home)/components/leaderboard-table.tsx create mode 100644 site/app/(home)/page.tsx create mode 100644 site/app/globals.css create mode 100644 site/app/layout.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/page.tsx create mode 100644 site/app/tasks/components/back-to-top.tsx create mode 100644 site/app/tasks/components/multi-select.tsx create mode 100644 site/app/tasks/components/tasks-page-client.tsx create mode 100644 site/app/tasks/page.tsx create mode 100644 site/bun.lock create mode 100644 site/components.json create mode 100644 site/components/pending-review-card.tsx create mode 100644 site/components/query-provider.tsx create mode 100644 site/components/theme-provider.tsx create mode 100644 site/components/theme-toggle.tsx create mode 100644 site/components/ui/badge.tsx create mode 100644 site/components/ui/button.tsx create mode 100644 site/components/ui/checkbox.tsx create mode 100644 site/components/ui/command.tsx create mode 100644 site/components/ui/dialog.tsx create mode 100644 site/components/ui/drawer.tsx create mode 100644 site/components/ui/hover-card.tsx create mode 100644 site/components/ui/popover.tsx create mode 100644 site/components/ui/scroll-area.tsx create mode 100644 site/components/ui/select.tsx create mode 100644 site/components/ui/sheet.tsx create mode 100644 site/components/ui/skeleton.tsx create mode 100644 site/components/ui/tabs.tsx create mode 100644 site/lib/http-error.ts create mode 100644 site/lib/utils.ts create mode 100644 site/next.config.ts create mode 100644 site/package.json create mode 100644 site/postcss.config.mjs create mode 100644 site/scripts/check-trajectory.ts create mode 100644 site/scripts/compute-tasks.ts create mode 100644 site/tsconfig.json create mode 100644 site/types/result.d.ts create mode 120000 site/zealt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..828b276 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,69 @@ +name: Deploy Next.js site to Pages + +on: + push: + branches: ["main"] + pull_request: + types: [opened, reopened, synchronize, closed] + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --frozen-lockfile + working-directory: site + + - name: Setup Pages + id: setup_pages + uses: actions/configure-pages@v5 + + - name: Set base uri + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}/pr-preview/pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" + else + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}" >> "$GITHUB_ENV" + fi + + - name: Build with Next.js + run: bun run build + working-directory: site + + - name: Deploy preview + if: github.event_name == 'pull_request' + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site/out + + - name: Deploy production + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + clean-exclude: pr-preview/ + force: false + folder: site/out diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0cf2fa4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +__pycache__/ +*.py[cod] +*$py.class + +# testing +coverage + +# next.js +.next/ + +# The `out` directory should not be ignored by version control +out/ + +# production +build + +# misc +.DS_Store +*.pem +*~ +\#* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.zealt/config.json b/.zealt/config.json new file mode 100644 index 0000000..0730a11 --- /dev/null +++ b/.zealt/config.json @@ -0,0 +1,5 @@ +{ + "title": "Altair Benchmark", + "description": "Performance results of AI coding models on Altair tasks, measuring success rate and execution time with high precision.", + "github_repo": "https://github.com/kweizh/altair-benchmark" +} \ No newline at end of file diff --git a/README.md b/README.md index 30bba36..e79c667 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# altair-benchmark \ No newline at end of file + +# Altair Benchmark + +This repository contains benchmarks for evaluating AI models on **Altair**. + +You can view the evaluation reports at [https://kweizh.github.io/altair-benchmark/](https://kweizh.github.io/altair-benchmark/). + +## Project Structure + +- `tasks/`: Contains the benchmark tasks, each with its own instructions. +- `jobs/`: Stores the results of benchmark runs. +- `site/`: A Next.js application to visualize benchmark results. + +## Getting Started + +This benchmark is evaluated using the [Harbor framework](https://github.com/harbor-framework/harbor) and the [Pochi agent](https://github.com/TabbyML/pochi). + +### Running Evaluation + +You can run the evaluation using the Harbor CLI. Here is an example: + +```bash +harbor run \ + --agent codex \ + --model "gpt-5.2-codex" \ + --env daytona \ + --path ./tasks \ + --n-attempts 1 \ + --max-retries 5 \ + --n-concurrent 5 \ + --retry-include RuntimeError \ + --retry-include DaytonaError \ + --retry-include AgentTimeoutError +``` + +### Evaluation Details + +Before starting the evaluation, you should set the necessary environment variables for your chosen agent. +For example, if using Pochi, you should export `POCHI_API_KEY`. + +Evaluation can be run locally with Docker (default), or using [Daytona.io](https://www.daytona.io/) by setting `--env daytona`. + +When running with Daytona, please note that Daytona blocks some network access for tier 1 and tier 2 users. If you meet any network issues, please refer to [Daytona network limits](https://www.daytona.io/docs/en/network-limits/). + +--- +Generated by [Zealt](https://github.com/TabbyML/zealt) diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..ffe123f --- /dev/null +++ b/plan.md @@ -0,0 +1,103 @@ +# Evaluation Dataset Research: Altair (Vega-Altair) + +## 1. Library Overview +* **Description**: Altair is a declarative statistical visualization library for Python, built on top of the [Vega-Lite](https://vega.github.io/vega-lite/) grammar. It allows users to describe visualizations in terms of data transformations and visual encodings rather than low-level imperative drawing commands. +* **Ecosystem Role**: It is the standard declarative plotting library for the Python data science stack (Pandas, Polars, NumPy). It integrates deeply with Jupyter, VS Code, and Streamlit. +* **Project Setup**: + ```bash + pip install altair vega_datasets + # For large datasets (optional but recommended) + pip install vegafusion[all] + ``` + +## 2. Core Primitives & APIs + +### Key Objects +* **`alt.Chart(data)`**: The fundamental object. Data can be a Pandas DataFrame, Polars DataFrame, or a URL string pointing to a JSON/CSV file. +* **`mark_*()`**: Defines the geometry (e.g., `mark_point()`, `mark_bar()`, `mark_line()`, `mark_area()`, `mark_rect()`, `mark_text()`). +* **`encode()`**: Maps data columns to visual channels (e.g., `x`, `y`, `color`, `size`, `shape`, `tooltip`). +* **`add_params()`**: (v5+) Replaces `add_selection`. Used to add interactive parameters like selections to a chart. + +### Code Examples + +#### Basic Chart with Shorthand Types +Altair uses a shorthand syntax for data types: `:Q` (Quantitative), `:N` (Nominal), `:O` (Ordinal), `:T` (Temporal). +```python +import altair as alt +from vega_datasets import data + +cars = data.cars() +chart = alt.Chart(cars).mark_point().encode( + x='Horsepower:Q', + y='Miles_per_Gallon:Q', + color='Origin:N', + tooltip=['Name', 'Origin'] +).interactive() # Enables zoom/pan +``` + +#### Composition (Layering & Concatenation) +* **Layering (`+`)**: Overlays charts. +* **Horizontal Concatenation (`|`)**: Side-by-side. +* **Vertical Concatenation (`&`)**: Top-to-bottom. +```python +base = alt.Chart(cars).encode(x='Horsepower:Q') +layers = base.mark_bar() + base.mark_rule(color='red').transform_aggregate(x='mean(Horsepower)') +concat = (chart1 | chart2).properties(title="Side by Side") +``` + +#### Advanced Interaction (v5+ Syntax) +Using `selection_interval` for cross-filtering. +```python +brush = alt.selection_interval() + +points = alt.Chart(cars).mark_point().encode( + x='Horsepower:Q', + y='Miles_per_Gallon:Q', + color=alt.when(brush).then('Origin:N').otherwise(alt.value('lightgray')) +).add_params(brush) + +bars = alt.Chart(cars).mark_bar().encode( + y='Origin:N', + x='count()', + color='Origin:N' +).transform_filter(brush) + +dashboard = points & bars +``` + +#### Transformations +```python +chart.transform_calculate( + Efficiency='datum.Miles_per_Gallon / datum.Weight' +).transform_filter( + alt.datum.Efficiency > 0.01 +) +``` + +## 3. Real-World Use Cases & Templates +* **Interactive Dashboards**: Linked views where selecting data in one plot (e.g., a map or timeline) filters the others. +* **Statistical Exploration**: Visualizing distributions with binned histograms and box plots. +* **Geographic Mapping**: Using `mark_geoshape` with TopoJSON data. +* **Integration with Streamlit**: Using `st.altair_chart` to build data apps with bidirectional communication (selections in Altair updating Streamlit state). + +## 4. Developer Friction Points +1. **MaxRowsError**: By default, Altair limits datasets to 5,000 rows to prevent browser crashes. + * *Solution*: Use `alt.data_transformers.disable_max_rows()` or `alt.data_transformers.enable('vegafusion')`. +2. **Date Handling**: Passing Python `datetime` objects in selections or filters can sometimes fail if not converted correctly by the underlying Vega-Lite engine. +3. **Complex Faceting**: Faceted charts with `resolve_scale(y='independent')` can be tricky when combined with shared selections across facets. +4. **Encoding Conflicts**: Forgetting to specify data types (e.g., `:O` vs `:N`) can lead to unexpected axis sorting or color scales. + +## 5. Evaluation Ideas +* **Simple**: Create a bar chart of average MPG by Origin with sorted bars and custom tooltips. +* **Intermediate**: Build a layered plot showing a scatter plot of data points with a regression line (using `transform_regression`). +* **Intermediate**: Implement an interactive "Focus + Context" chart (a small overview chart with a brush that controls the X-axis of a larger detail chart). +* **Complex**: Create a cross-filtering dashboard with three linked views (Scatter, Histogram, and Heatmap) using `selection_point` and `selection_interval`. +* **Complex**: Design a geographic map of US airports where clicking an airport highlights its connections on the same map (using `transform_lookup`). +* **Edge Case**: Handle a 50,000-row dataset by configuring `VegaFusion` and implementing an aggregated heatmap to avoid browser lag. + +## 6. Sources +1. [Official Vega-Altair Documentation](https://altair-viz.github.io/): Main reference for API and User Guide. +2. [Altair GitHub Repository](https://github.com/vega/altair): Source for issues and release notes. +3. [Vega-Lite Documentation](https://vega.github.io/vega-lite/): Documentation for the underlying grammar. +4. [VegaFusion Documentation](https://vegafusion.io/): Solutions for large dataset scaling. +5. [Streamlit Altair Integration](https://docs.streamlit.io/develop/api-reference/charts/st.altair_chart): Guide for interactive web apps. \ No newline at end of file diff --git a/scratchpad/pending-tasks/basic_aggregation_and_sorting_chart/instruction.md b/scratchpad/pending-tasks/basic_aggregation_and_sorting_chart/instruction.md new file mode 100644 index 0000000..39d6838 --- /dev/null +++ b/scratchpad/pending-tasks/basic_aggregation_and_sorting_chart/instruction.md @@ -0,0 +1,9 @@ +Visualizing aggregate metrics and standardizing encodings is a fundamental first step in exploratory data analysis with Altair. Altair's shorthand syntax simplifies defining data types, but explicit sorting and tooltips require careful configuration. + +You need to create a bar chart showing the average "Miles_per_Gallon" by "Origin" using the `cars` dataset from `vega_datasets` in a standard Python environment. + +**Constraints:** +- Must explicitly use Altair shorthand types (e.g., `:Q` for Quantitative, `:N` for Nominal). +- Bars MUST be sorted in descending order based on the average MPG. +- Tooltips must be added to display both the "Origin" and the computed average MPG. +- Save the resulting chart specification to a file named `bar_chart.json`. \ No newline at end of file diff --git a/scratchpad/pending-tasks/focus_and_context_interactive_chart/instruction.md b/scratchpad/pending-tasks/focus_and_context_interactive_chart/instruction.md new file mode 100644 index 0000000..6e6d0b8 --- /dev/null +++ b/scratchpad/pending-tasks/focus_and_context_interactive_chart/instruction.md @@ -0,0 +1,9 @@ +Interactive "Focus + Context" charts allow users to zoom in on specific temporal or quantitative data regions without losing sight of the overall trend, leveraging Altair's selection APIs. + +You need to implement an interactive chart using Altair v5+ syntax where a small overview area chart includes an interval brush that dynamically controls the X-axis domain of a larger detailed line chart. + +**Constraints:** +- MUST use Altair v5+ syntax, specifically `alt.selection_interval()` and `add_params()` (do NOT use the deprecated `add_selection`). +- The two charts must be vertically concatenated using the `&` operator. +- The X-axis of the detail chart must strictly bind to the brush parameter from the overview chart. +- Save the resulting chart specification to `focus_context.json`. \ No newline at end of file diff --git a/scratchpad/pending-tasks/independent_facet_scales_with_shared_selection/instruction.md b/scratchpad/pending-tasks/independent_facet_scales_with_shared_selection/instruction.md new file mode 100644 index 0000000..82cdf60 --- /dev/null +++ b/scratchpad/pending-tasks/independent_facet_scales_with_shared_selection/instruction.md @@ -0,0 +1,9 @@ +Combining faceted charts with shared interactive selections often leads to axis scale conflicts or unexpected behaviors if scales are not properly resolved. + +You need to create a faceted scatter plot (faceted by "Origin" into separate columns) of the `cars` dataset where a legend-bound point selection highlights specific "Cylinders" across all facets simultaneously. + +**Constraints:** +- You MUST ensure the Y-axis is optimized for each facet by using `resolve_scale(y='independent')`. +- You MUST use `alt.selection_point(fields=['Cylinders'], bind='legend')` to create the interactivity. +- The opacity of points not matching the legend selection must drop to `0.2` across all facets. +- Save the resulting chart specification to `faceted_shared.json`. \ No newline at end of file diff --git a/scratchpad/pending-tasks/large_dataset_handling_with_vegafusion/instruction.md b/scratchpad/pending-tasks/large_dataset_handling_with_vegafusion/instruction.md new file mode 100644 index 0000000..10344bc --- /dev/null +++ b/scratchpad/pending-tasks/large_dataset_handling_with_vegafusion/instruction.md @@ -0,0 +1,9 @@ +Altair restricts datasets over 5,000 rows by default (throwing a `MaxRowsError`) to prevent browser crashes from massive JSON payloads. Addressing this is critical for production-grade data science pipelines. + +You need to process a synthesized Pandas DataFrame containing 50,000 rows and visualize it as an aggregated 2D heatmap showing record counts. + +**Constraints:** +- You MUST explicitly bypass the 5,000-row limit by configuring the environment with `alt.data_transformers.enable('vegafusion')`. +- Do NOT use `alt.data_transformers.disable_max_rows()` as it risks browser lock-up. +- The visualization must aggregate the data into bins on both the X and Y axes within Altair (do not pre-bin in Pandas). +- Save the resulting visualization as `heatmap_large.html`. \ No newline at end of file diff --git a/scratchpad/pending-tasks/layered_scatter_plot_with_regression_line/instruction.md b/scratchpad/pending-tasks/layered_scatter_plot_with_regression_line/instruction.md new file mode 100644 index 0000000..a99b7ea --- /dev/null +++ b/scratchpad/pending-tasks/layered_scatter_plot_with_regression_line/instruction.md @@ -0,0 +1,9 @@ +Altair excels at visual composition, allowing developers to overlay analytical transformations directly on top of raw data visualizations using the layering operator (`+`). + +You need to build a layered chart containing a base scatter plot of "Horsepower" (X-axis) versus "Miles_per_Gallon" (Y-axis) from the `cars` dataset, and overlay a linear regression line in a Python script. + +**Constraints:** +- The regression line MUST be calculated natively within Altair using `transform_regression`. +- The scatter plot points and the regression line must be distinct colors (e.g., blue for points, red for the line). +- Do NOT pre-calculate the regression line using Pandas or NumPy. +- Save the rendered chart to a file named `regression_chart.html`. \ No newline at end of file diff --git a/scratchpad/pending-tasks/tri_view_cross_filtering_dashboard/instruction.md b/scratchpad/pending-tasks/tri_view_cross_filtering_dashboard/instruction.md new file mode 100644 index 0000000..835bf5c --- /dev/null +++ b/scratchpad/pending-tasks/tri_view_cross_filtering_dashboard/instruction.md @@ -0,0 +1,9 @@ +Dashboards often require linked views where selecting or brushing data in one visual updates the subset of data displayed in the others, providing deep multi-dimensional exploration. + +You need to create a cross-filtering dashboard comprising three linked views (a scatter plot, a bar chart, and a histogram) using the `cars` dataset. + +**Constraints:** +- Apply an interval selection brush (`alt.selection_interval()`) to the scatter plot. +- The bar chart and histogram MUST dynamically filter their displayed data based on the scatter plot's brush using `transform_filter`. +- Unselected points in the scatter plot should turn `lightgray` using an `alt.when().then().otherwise()` condition. +- Save the complete dashboard layout to `dashboard.html`. \ No newline at end of file diff --git a/site/app/(home)/components/leaderboard-table.tsx b/site/app/(home)/components/leaderboard-table.tsx new file mode 100644 index 0000000..d10b1f0 --- /dev/null +++ b/site/app/(home)/components/leaderboard-table.tsx @@ -0,0 +1,183 @@ +"use client"; + +import { useState, useMemo, useEffect } from "react"; +import { useRouter } from "next/navigation"; +import { Search, Trophy, ListTree } from "lucide-react"; +import { clsx, type ClassValue } from "clsx"; +import { twMerge } from "tailwind-merge"; +import Link from "next/link"; +import zealtConfig from "@/zealt/config.json"; + +function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} + +export interface LeaderboardEntry { + id: string; + model: string; + rawModel: string; + agent: string; + passedEvals: number; + successRate: number; + avgLatency: number; + isNew?: boolean; +} + +function ProgressBar({ value, colorClass }: { value: number; colorClass: string }) { + return ( +
+
+
+ ); +} + +function ScoreCell({ value }: { value: number }) { + let colorClass = "bg-primary"; + let textClass = "text-muted-foreground"; + + if (value >= 90) { + colorClass = "bg-emerald-500"; + textClass = "text-emerald-500 font-bold"; + } else if (value >= 75) { + colorClass = "bg-blue-500"; + textClass = "text-blue-500 font-medium"; + } else if (value >= 60) { + colorClass = "bg-amber-500"; + textClass = "text-amber-500"; + } else { + colorClass = "bg-red-500"; + textClass = "text-red-500"; + } + + return ( +
+ {value}% + +
+ ); +} + +export default function LeaderboardTable({ data }: { data: LeaderboardEntry[] }) { + const router = useRouter(); + const [devMode, setDevMode] = useState(process.env.NODE_ENV === "development"); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevMode(isDev); + }, []); + + const filteredData = useMemo(() => { + let processedData = data; + const config = zealtConfig as { pending_models?: string[] }; + + if (!devMode && config.pending_models && config.pending_models.length > 0) { + processedData = processedData.filter((item) => + !config.pending_models!.includes(item.rawModel) + ); + } + + if (searchQuery) { + const query = searchQuery.toLowerCase(); + processedData = processedData.filter(item => + item.model.toLowerCase().includes(query) + ); + } + + return processedData; + }, [data, searchQuery, devMode]); + + return ( + <> + {/* Controls & Filters */} +
+

+ Model Performance +

+ +
+ + + View Tasks + + +
+ + setSearchQuery(e.target.value)} + className="pl-9 pr-4 py-2 bg-card border border-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary/20 w-full sm:w-64 transition-all" + /> +
+
+
+ + {/* Leaderboard Table */} +
+
+ + + + + + + + + + + {filteredData.length === 0 ? ( + + + + ) : ( + filteredData.map((row, index) => ( + router.push(`./tasks?model=${encodeURIComponent(row.model)}`)} + className="group hover:bg-secondary/30 transition-colors duration-200 cursor-pointer" + > + + + + + + )))} + +
ModelPassedAvg DurationSuccess Rate
+ No results found matching your search. +
+ #{index + 1} +
+ + {row.model} + {index === 0 && } + {row.isNew && ( + + NEW + + )} + +
+ 		+ {row.passedEvals} + + {row.avgLatency > 0 ? `${row.avgLatency.toFixed(1)}s` : '-'} + +
+ +
+
+
+
+ + ); +} diff --git a/site/app/(home)/page.tsx b/site/app/(home)/page.tsx new file mode 100644 index 0000000..d617123 --- /dev/null +++ b/site/app/(home)/page.tsx @@ -0,0 +1,176 @@ +import { Suspense } from "react"; +import { Github, Terminal, ClipboardList, ListTree } from "lucide-react"; +import Link from "next/link"; +import tasksData from "@/zealt/tasks.json"; +import pendingTasksData from "@/zealt/pending-tasks.json"; +import zealtConfig from "@/zealt/config.json"; +import PendingReviewCard from "@/components/pending-review-card"; +import LeaderboardTable, { type LeaderboardEntry } from "./components/leaderboard-table"; + +type TaskTrial = { + agent: string; + model: string; + passed: boolean; + latency_sec: number | null; +}; + +type TaskValue = { + trials?: TaskTrial[]; +}; + +type PendingTasksValue = { + 'pending-tasks'?: number; +}; + +export default function Home() { + const totalTasks = Object.keys(tasksData as Record).length; + const hasTasks = totalTasks > 0; + const pendingSampleCases = Math.max( + 0, + Number((pendingTasksData as PendingTasksValue)['pending-tasks'] ?? 0), + ); + + // Process tasks.json to compute leaderboard stats directly on the server + const statsMap = new Map(); + + Object.values(tasksData as Record).forEach((taskValue) => { + let trials: TaskTrial[] = []; + if (Array.isArray(taskValue)) { + trials = taskValue as TaskTrial[]; + } else if (typeof taskValue === "object" && taskValue !== null) { + const task = taskValue as TaskValue; + trials = Array.isArray(task.trials) ? task.trials : []; + } + + trials.forEach((trial) => { + // Simplify model name + const modelName = trial.model.split('/').pop() || trial.model; + const agentName = trial.agent.charAt(0).toUpperCase() + trial.agent.slice(1); + + const key = `${modelName}-${agentName}`; + + if (!statsMap.has(key)) { + statsMap.set(key, { + passed: 0, + total: 0, + totalLatency: 0, + latencyCount: 0, + model: modelName, + rawModel: trial.model, + agent: agentName + }); + } + + const stats = statsMap.get(key); + if (!stats) { + return; + } + stats.total += 1; + if (trial.passed) { + stats.passed += 1; + } + if (trial.latency_sec) { + stats.totalLatency += trial.latency_sec; + stats.latencyCount += 1; + } + }); + }); + + const data: LeaderboardEntry[] = Array.from(statsMap.values()) + .map((stats, index) => { + const successRate = stats.total > 0 ? Math.round((stats.passed / stats.total) * 100) : 0; + const avgLatency = stats.latencyCount > 0 ? stats.totalLatency / stats.latencyCount : 0; + return { + id: String(index + 1), + model: stats.model, + rawModel: stats.rawModel, + agent: stats.agent, + passedEvals: stats.passed, + successRate: successRate, + avgLatency: avgLatency, + }; + }) + .sort((a, b) => b.successRate - a.successRate); + + // Re-assign IDs based on sorted order and adjust isNew + data.forEach((item, index) => { + item.id = String(index + 1); + item.isNew = index === 0; // Keeping the original visual effect for the top item + }); + + return ( +
+ {/* Background Gradient Effect */} +
+ +
+ {/* Header Section */} +
+
+ + Live Benchmarks +
+ +

+ {zealtConfig.title} +

+ +

+ {zealtConfig.description} +

+ +
+ + + View on GitHub + + {data.length > 0 && ( + <> +
+ + + Total tasks: {totalTasks} + + + )} +
+ + + Last run: {new Date().toLocaleDateString()} + +
+
+ + {!hasTasks ? ( + + ) : data.length === 0 ? ( +
+

No evaluation data yet

+
+ + + View Tasks + +
+
+ ) : ( + // Client Component for Interactive Table + Loading leaderboard...
}> + + + )} +
+
+ ); +} diff --git a/site/app/globals.css b/site/app/globals.css new file mode 100644 index 0000000..437c041 --- /dev/null +++ b/site/app/globals.css @@ -0,0 +1,149 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); + --color-background: var(--background); + --color-foreground: var(--foreground); + --color-card: var(--card); + --color-card-foreground: var(--card-foreground); + --color-popover: var(--popover); + --color-popover-foreground: var(--popover-foreground); + --color-primary: var(--primary); + --color-primary-foreground: var(--primary-foreground); + --color-secondary: var(--secondary); + --color-secondary-foreground: var(--secondary-foreground); + --color-muted: var(--muted); + --color-muted-foreground: var(--muted-foreground); + --color-accent: var(--accent); + --color-accent-foreground: var(--accent-foreground); + --color-destructive: var(--destructive); + --color-border: var(--border); + --color-input: var(--input); + --color-ring: var(--ring); + --color-chart-1: var(--chart-1); + --color-chart-2: var(--chart-2); + --color-chart-3: var(--chart-3); + --color-chart-4: var(--chart-4); + --color-chart-5: var(--chart-5); + --color-sidebar: var(--sidebar); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.141 0.005 285.823); + --card: oklch(1 0 0); + --card-foreground: oklch(0.141 0.005 285.823); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.141 0.005 285.823); + --primary: oklch(0.21 0.006 285.885); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.967 0.001 286.375); + --secondary-foreground: oklch(0.21 0.006 285.885); + --muted: oklch(0.967 0.001 286.375); + --muted-foreground: oklch(0.552 0.016 285.938); + --accent: oklch(0.967 0.001 286.375); + --accent-foreground: oklch(0.21 0.006 285.885); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.92 0.004 286.32); + --input: oklch(0.92 0.004 286.32); + --ring: oklch(0.705 0.015 286.067); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.141 0.005 285.823); + --sidebar-primary: oklch(0.21 0.006 285.885); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.967 0.001 286.375); + --sidebar-accent-foreground: oklch(0.21 0.006 285.885); + --sidebar-border: oklch(0.92 0.004 286.32); + --sidebar-ring: oklch(0.705 0.015 286.067); +} + +.dark { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.21 0.006 285.885); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.92 0.004 286.32); + --primary-foreground: oklch(0.21 0.006 285.885); + --secondary: oklch(0.274 0.006 286.033); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --accent: oklch(0.274 0.006 286.033); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.552 0.016 285.938); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.21 0.006 285.885); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.274 0.006 286.033); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.552 0.016 285.938); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} + +@layer utilities { + .custom-scrollbar { + scrollbar-width: thin; + scrollbar-color: var(--border) transparent; + } + + .custom-scrollbar::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + .custom-scrollbar::-webkit-scrollbar-track { + background: transparent; + } + + .custom-scrollbar::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; + } + + .custom-scrollbar::-webkit-scrollbar-thumb:hover { + background: color-mix(in oklch, var(--foreground) 30%, transparent); + } +} \ No newline at end of file diff --git a/site/app/layout.tsx b/site/app/layout.tsx new file mode 100644 index 0000000..3ee8c0a --- /dev/null +++ b/site/app/layout.tsx @@ -0,0 +1,37 @@ +import "./globals.css"; + +import type { Metadata } from 'next'; +import { Inter } from "next/font/google"; +import zealtConfig from "@/zealt/config.json"; +import { ThemeProvider } from "@/components/theme-provider"; +import { ThemeToggle } from "@/components/theme-toggle"; +import { AppQueryProvider } from "@/components/query-provider"; + +const inter = Inter({ subsets: ["latin"] }); + +export const metadata: Metadata = { + title: zealtConfig.title, + description: zealtConfig.description, +}; + + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + +
+ +
+ {children} + + + + + ); +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx new file mode 100644 index 0000000..bf9202c --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx @@ -0,0 +1,265 @@ +"use client"; + +import { useMemo } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { ChevronRight, File, Folder } from "lucide-react"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + fetchLogText, + LogContentSkeleton, + LogErrorView, + getLogErrorMessage, +} from "./trajectory-page"; + +export type ArtifactNode = { + name: string; + type: "file" | "dir"; + path: string; + children?: ArtifactNode[]; +}; + +export type ArtifactNodeWithUrl = + | { name: string; type: "file"; path: string; url: string } + | { name: string; type: "dir"; path: string; children: ArtifactNodeWithUrl[] }; + +type ArtifactsPanelProps = { + artifactTree: ArtifactNodeWithUrl[]; +}; + +export function ArtifactsPanel({ artifactTree }: ArtifactsPanelProps) { + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + + const queryArtifact = searchParams.get("artifact"); + const queryPath = searchParams.get("path"); + + const activeArtifact = useMemo(() => { + if (queryArtifact) { + const found = artifactTree.find((node) => node.name === queryArtifact); + if (found) return found; + } + return artifactTree[0] ?? null; + }, [artifactTree, queryArtifact]); + + const updateParams = (next: { artifact?: string | null; path?: string | null }) => { + const params = new URLSearchParams(searchParams.toString()); + if (next.artifact === null) { + params.delete("artifact"); + } else if (next.artifact !== undefined) { + params.set("artifact", next.artifact); + } + if (next.path === null) { + params.delete("path"); + } else if (next.path !== undefined) { + params.set("path", next.path); + } + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + + const handleSelectArtifact = (name: string) => { + updateParams({ artifact: name, path: null }); + }; + + if (!activeArtifact) { + return ( +
+ No artifacts available. +
+ ); + } + + return ( +
+
+
+ {artifactTree.map((node) => ( + + ))} +
+
+ +
+ {activeArtifact.type === "file" ? ( + + ) : ( + updateParams({ path: path ?? null })} + /> + )} +
+
+ ); +} + +type DirectoryArtifactViewProps = { + artifact: Extract; + currentPath: string | null; + onPathChange: (path: string | null) => void; +}; + +function DirectoryArtifactView({ artifact, currentPath, onPathChange }: DirectoryArtifactViewProps) { + // Resolve the active path. If currentPath isn't valid for this artifact, fall back to the artifact root. + const resolved = useMemo(() => resolvePath(artifact, currentPath), [artifact, currentPath]); + const { folder, selectedFile, breadcrumbDirs } = resolved; + + return ( +
+
+ {breadcrumbDirs.map((dir, idx) => { + const isLast = idx === breadcrumbDirs.length - 1 && !selectedFile; + return ( + + + {(idx < breadcrumbDirs.length - 1 || selectedFile) && ( + + )} + + ); + })} + {selectedFile && ( + {selectedFile.name} + )} +
+ +
+
+ {folder.children.length === 0 && ( + Empty directory. + )} + {folder.children.map((child) => { + const isActive = child.type === "file" && selectedFile?.path === child.path; + return ( + + ); + })} +
+
+ +
+ {selectedFile ? ( + + ) : ( +
+ Select a file to view its contents. +
+ )} +
+
+ ); +} + +function FileViewer({ url }: { url: string }) { + const query = useQuery({ + queryKey: ["artifact-file", url], + queryFn: () => fetchLogText(url), + }); + + return ( + +
+ {query.isPending || query.isFetching ? ( + + ) : query.isError ? ( + void query.refetch()} + /> + ) : query.data ? ( + 
+            {query.data}
+
+ ) : ( +

Empty file.

+ )} +
+ + ); +} + +type ResolvedPath = { + folder: Extract; + selectedFile: Extract | null; + breadcrumbDirs: { name: string; path: string }[]; +}; + +function resolvePath( + root: Extract, + targetPath: string | null, +): ResolvedPath { + const breadcrumbDirs: { name: string; path: string }[] = [{ name: root.name, path: root.path }]; + let folder = root; + let selectedFile: Extract | null = null; + + if (!targetPath || !targetPath.startsWith(`${root.path}/`)) { + return { folder, selectedFile, breadcrumbDirs }; + } + + const remainder = targetPath.slice(root.path.length + 1); + const segments = remainder.split("/").filter(Boolean); + + let currentPath = root.path; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]; + currentPath = `${currentPath}/${segment}`; + const child = folder.children.find((c) => c.name === segment); + if (!child) { + // Path doesn't exist; bail and show what we resolved so far. + return { folder, selectedFile: null, breadcrumbDirs }; + } + if (child.type === "dir") { + folder = child; + breadcrumbDirs.push({ name: child.name, path: child.path }); + } else { + // File at the end of the path. + selectedFile = child; + // Don't push the file as a breadcrumb dir; the caller renders it separately. + break; + } + } + + return { folder, selectedFile, breadcrumbDirs }; +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx new file mode 100644 index 0000000..0fab8fc --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx @@ -0,0 +1,404 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useTheme } from "next-themes"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Button } from "@/components/ui/button"; +import { HttpError } from "@/lib/http-error"; +import { ArtifactsPanel, type ArtifactNodeWithUrl } from "./artifacts-panel"; + +export type TabConfig = { + value: string; + label: React.ReactNode; +}; + +type TrajectoryPageProps = { + trajectoryUrl: string; + browserVerificationUrls: { name: string; url: string }[]; + fallbackUrl: string; + stderrLogUrl: string | null; + verifierLogUrl: string | null; + tabsConfig: TabConfig[]; + artifactTree?: ArtifactNodeWithUrl[]; +}; + +export async function fetchLogText(url: string): Promise { + let response: Response; + try { + response = await fetch(url, { cache: "force-cache" }); + } catch (_e) { + throw new HttpError("Network request failed."); + } + + if (!response.ok) { + throw new HttpError(`Request failed with status ${response.status}.`, { status: response.status }); + } + + return response.text(); +} + +export function TrajectoryPage({ + trajectoryUrl, + browserVerificationUrls, + fallbackUrl, + stderrLogUrl, + verifierLogUrl, + tabsConfig, + artifactTree, +}: TrajectoryPageProps) { + const { resolvedTheme } = useTheme(); + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + const [mounted, setMounted] = useState(false); + const [iframeLoading, setIframeLoading] = useState(true); + const [devModeEnabled, setDevModeEnabled] = useState(process.env.NODE_ENV === "development"); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevModeEnabled(isDev); + }, []); + + const visibleTabsConfig = useMemo( + () => + devModeEnabled + ? tabsConfig + : tabsConfig.filter((t) => t.value !== "artifacts"), + [tabsConfig, devModeEnabled], + ); + const validTabs = visibleTabsConfig.map((t) => t.value); + const [activeTab, setActiveTab] = useState(() => { + const queryTab = searchParams.get("tab"); + return queryTab && validTabs.includes(queryTab) ? queryTab : validTabs[0]; + }); + + const handleTabChange = (value: string) => { + setActiveTab(value); + const params = new URLSearchParams(searchParams.toString()); + params.set("tab", value); + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + const [browserIframeLoading, setBrowserIframeLoading] = useState>({}); + const [activeBrowserVerificationTab, setActiveBrowserVerificationTab] = useState( + browserVerificationUrls[0]?.name || "", + ); + + const iframeTheme = mounted && resolvedTheme === "light" ? "light" : "dark"; + + const iframeUrl = useMemo(() => { + const url = new URL(trajectoryUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [trajectoryUrl, iframeTheme]); + + const activeBrowserVerificationBaseUrl = useMemo(() => { + const testCase = browserVerificationUrls.find((tc) => tc.name === activeBrowserVerificationTab); + return testCase ? testCase.url : null; + }, [browserVerificationUrls, activeBrowserVerificationTab]); + + const activeBrowserVerificationUrl = useMemo(() => { + if (!activeBrowserVerificationBaseUrl) { + return null; + } + + const url = new URL(activeBrowserVerificationBaseUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [activeBrowserVerificationBaseUrl, iframeTheme]); + + useEffect(() => { + setMounted(true); + }, []); + + useEffect(() => { + if (!mounted || !activeBrowserVerificationBaseUrl) { + return; + } + + setBrowserIframeLoading((prev) => ({ ...prev, [activeBrowserVerificationTab]: true })); + }, [activeBrowserVerificationBaseUrl, activeBrowserVerificationTab, mounted]); + + const stderrQuery = useQuery({ + queryKey: ["trajectory-stderr", stderrLogUrl], + enabled: Boolean(stderrLogUrl), + queryFn: async () => { + if (!stderrLogUrl) { + return null; + } + + return fetchLogText(stderrLogUrl); + }, + }); + + const verifierQuery = useQuery({ + queryKey: ["trajectory-verifier", verifierLogUrl], + enabled: Boolean(verifierLogUrl), + queryFn: async () => { + if (!verifierLogUrl) { + return null; + } + + return fetchLogText(verifierLogUrl); + }, + }); + + const handleIframeLoad = () => { + setIframeLoading(false); + }; + + const handleBrowserIframeLoad = (testCaseName: string) => { + setBrowserIframeLoading((prev) => ({ ...prev, [testCaseName]: false })); + }; + + const handleIframeError = () => { + window.location.replace(fallbackUrl); + }; + + const renderLogContent = ( + text: string | null | undefined, + isLoading: boolean, + isError: boolean, + error: unknown, + onRetry: () => void, + emptyMessage: string, + ) => { + if (isLoading) { + return ; + } + + if (isError) { + return ; + } + + if (!text) { + return

{emptyMessage}

; + } + + return ( + 
+        {text}
+
+ ); + }; + + return ( +
+
+ +
+ + {visibleTabsConfig.map((tab) => ( + + {tab.label} + + ))} + +
+ + +
+ +
+ {mounted && ( +