diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..828b276a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,69 @@ +name: Deploy Next.js site to Pages + +on: + push: + branches: ["main"] + pull_request: + types: [opened, reopened, synchronize, closed] + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --frozen-lockfile + working-directory: site + + - name: Setup Pages + id: setup_pages + uses: actions/configure-pages@v5 + + - name: Set base uri + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}/pr-preview/pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" + else + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}" >> "$GITHUB_ENV" + fi + + - name: Build with Next.js + run: bun run build + working-directory: site + + - name: Deploy preview + if: github.event_name == 'pull_request' + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site/out + + - name: Deploy production + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + clean-exclude: pr-preview/ + force: false + folder: site/out diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0cf2fa4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +__pycache__/ +*.py[cod] +*$py.class + +# testing +coverage + +# next.js +.next/ + +# The `out` directory should not be ignored by version control +out/ + +# production +build + +# misc +.DS_Store +*.pem +*~ +\#* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.zealt/config.json b/.zealt/config.json new file mode 100644 index 00000000..039553ef --- /dev/null +++ b/.zealt/config.json @@ -0,0 +1,5 @@ +{ + "title": "Typesense Benchmark", + "description": "Performance results of AI coding models on Typesense tasks, measuring success rate and execution time with high precision.", + "github_repo": "https://github.com/kweizh/typesense-benchmark" +} \ No newline at end of file diff --git a/README.md b/README.md index 6cfc27fd..b06fe11e 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# typesense-benchmark \ No newline at end of file + +# Typesense Benchmark + +This repository contains benchmarks for evaluating AI models on **Typesense**. + +You can view the evaluation reports at [https://kweizh.github.io/typesense-benchmark/](https://kweizh.github.io/typesense-benchmark/). + +## Project Structure + +- `tasks/`: Contains the benchmark tasks, each with its own instructions. +- `jobs/`: Stores the results of benchmark runs. +- `site/`: A Next.js application to visualize benchmark results. + +## Getting Started + +This benchmark is evaluated using the [Harbor framework](https://github.com/harbor-framework/harbor) and the [Pochi agent](https://github.com/TabbyML/pochi). + +### Running Evaluation + +You can run the evaluation using the Harbor CLI. Here is an example: + +```bash +harbor run \ + --agent codex \ + --model "gpt-5.2-codex" \ + --env daytona \ + --path ./tasks \ + --n-attempts 1 \ + --max-retries 5 \ + --n-concurrent 5 \ + --retry-include RuntimeError \ + --retry-include DaytonaError \ + --retry-include AgentTimeoutError +``` + +### Evaluation Details + +Before starting the evaluation, you should set the necessary environment variables for your chosen agent. +For example, if using Pochi, you should export `POCHI_API_KEY`. + +Evaluation can be run locally with Docker (default), or using [Daytona.io](https://www.daytona.io/) by setting `--env daytona`. + +When running with Daytona, please note that Daytona blocks some network access for tier 1 and tier 2 users. If you meet any network issues, please refer to [Daytona network limits](https://www.daytona.io/docs/en/network-limits/). + +--- +Generated by [Zealt](https://github.com/TabbyML/zealt) diff --git a/plan.md b/plan.md new file mode 100644 index 00000000..7155990e --- /dev/null +++ b/plan.md @@ -0,0 +1,341 @@ +# Typesense Research Report & Benchmark Task Design + +This report provides a deep technical analysis of the **Typesense** search engine, its core primitives, APIs, real-world integration patterns, common developer friction points, and a suite of self-contained, container-friendly evaluation tasks designed specifically for autonomous coding agents. + +--- + +## 1. Library Overview + +### Description +**Typesense** is a modern, open-source, typo-tolerant search engine optimized for sub-50ms, instant search-as-you-type developer experiences. It is written in C++ and designed from the ground up to store its search indices entirely in memory (with a raw data backup on disk via RocksDB), achieving high performance and throughput. + +### Ecosystem Role +Typesense serves as a fast, user-facing search index positioned between heavy-duty analytical search engines like Elasticsearch and costly proprietary SaaS solutions like Algolia. It is frequently synced with primary databases (such as PostgreSQL, MongoDB, DynamoDB, Supabase, or Firebase) to power instant auto-complete bars, faceted navigation, geosearch, and AI-powered semantic/hybrid search (RAG) with built-in or custom vector embedding pipelines. + +### Project Setup (Non-Interactive, Standalone Binary) +In non-interactive Docker environments (such as constrained test runners or agent sandboxes) where Docker-in-Docker (DinD) is unavailable or restricted, Typesense can be run directly as a **standalone native Linux binary**. This eliminates the requirement of external container runtimes or system daemons. + +#### Step-by-Step Standalone Initialization (AMD64 Linux) +```bash +# 1. Download the pre-compiled standalone Linux binary (using v26.0 as a stable target) +curl -O https://dl.typesense.org/releases/26.0/typesense-server-26.0-linux-amd64.tar.gz + +# 2. Extract the archive +tar -xzf typesense-server-26.0-linux-amd64.tar.gz + +# 3. Create a local data directory to persist raw document data +mkdir -p ./typesense-data + +# 4. Start the Typesense server in the background +export TYPESENSE_API_KEY=xyz +./typesense-server \ + --data-dir="$(pwd)"/typesense-data \ + --api-key=$TYPESENSE_API_KEY \ + --port=8108 \ + --enable-cors & + +# 5. Wait for the server to become healthy +until curl -s http://localhost:8108/health | grep -q '"ok":true'; do + echo "Waiting for Typesense..." + sleep 1 +done + +echo "Typesense is up and running!" +``` + +--- + +## 2. Core Primitives & APIs + +Typesense is structured around a few key primitives that mirror relational database concepts: +* **Collection**: Roughly equivalent to a database table. It has a name, a schema (defining fields, types, and index options), and contains multiple documents. +* **Document**: An individual JSON record indexed inside a collection. +* **Alias**: A virtual pointer to a physical collection, allowing zero-downtime schema migrations or reindexing. +* **Key**: Fine-grained API keys with scoped permissions (e.g., search-only, tenant-restricted). + +### Key APIs & Documentation Links + +1. **Collections API**: [Collections Reference](https://typesense.org/docs/30.2/api/collections.html) + * Used to define, retrieve, update, clone, and drop collections. +2. **Documents API**: [Documents Reference](https://typesense.org/docs/30.2/api/documents.html) + * Used to index, retrieve, update, upsert, delete, import, and export individual or batch documents. +3. **Search API**: [Search Reference](https://typesense.org/docs/30.2/api/search.html) + * Supports full-text query, filtering (`filter_by`), sorting (`sort_by`), faceting (`facet_by`), grouping (`group_by`), and pagination. +4. **Vector Search API**: [Vector Search Reference](https://typesense.org/docs/30.2/api/vector-search.html) + * Enables nearest-neighbor (KNN) query, hybrid search combining keyword and vector queries, and auto-embedding generation. +5. **JOINs API**: [JOINs Reference](https://typesense.org/docs/30.2/api/joins.html) + * Enables cross-collection joins for one-to-one, one-to-many, and many-to-many relations. +6. **Collection Alias API**: [Collection Alias Reference](https://typesense.org/docs/30.2/api/collection-alias.html) + * Creates virtual names for collections to allow zero-downtime swaps. + +--- + +### Deep Dive: 1. Collection Creation & Schema Definition +A collection is created by defining its schema. Schema fields can be explicitly defined, dynamically detected (`.*` with type `auto`), or mixed. + +* **SDK Versions Used**: `typesense` (Python SDK) `v1.8.0`, `typesense` (Node.js SDK) `v1.8.2` + +#### Python Snippet (Explicit Schema with Stemming & Faceting) +```python +import typesense + +client = typesense.Client({ + 'nodes': [{ + 'host': 'localhost', + 'port': '8108', + 'protocol': 'http' + }], + 'api_key': 'xyz', + 'connection_timeout_seconds': 2 +}) + +schema = { + 'name': 'products', + 'fields': [ + {'name': 'product_name', 'type': 'string', 'facet': False}, + {'name': 'category', 'type': 'string', 'facet': True}, + {'name': 'price', 'type': 'float', 'facet': False}, + {'name': 'tags', 'type': 'string[]', 'facet': True, 'optional': True}, + {'name': 'description', 'type': 'string', 'stem': True, 'optional': True} + ], + 'default_sorting_field': 'price' +} + +client.collections.create(schema) +``` + +#### Node.js Equivalent +```javascript +const Typesense = require('typesense'); + +const client = new Typesense.Client({ + nodes: [{ host: 'localhost', port: '8108', protocol: 'http' }], + apiKey: 'xyz', + connectionTimeoutSeconds: 2 +}); + +const schema = { + name: 'products', + fields: [ + { name: 'product_name', type: 'string', facet: false }, + { name: 'category', type: 'string', facet: true }, + { name: 'price', type: 'float', facet: false }, + { name: 'tags', type: 'string[]', facet: true, optional: true }, + { name: 'description', type: 'string', stem: true, optional: true } + ], + default_sorting_field: 'price' +}; + +client.collections().create(schema); +``` + +#### CLI / Shell Equivalent +```bash +curl "http://localhost:8108/collections" \ + -X POST \ + -H "Content-Type: application/json" \ + -H "X-TYPESENSE-API-KEY: xyz" \ + -d '{ + "name": "products", + "fields": [ + {"name": "product_name", "type": "string", "facet": false}, + {"name": "category", "type": "string", "facet": true}, + {"name": "price", "type": "float", "facet": false}, + {"name": "tags", "type": "string[]", "facet": true, "optional": true}, + {"name": "description", "type": "string", "stem": true, "optional": true} + ], + "default_sorting_field": "price" + }' +``` + +--- + +### Deep Dive: 2. Document Bulk Import & Dirty Data Handling +When importing high volumes of data, using single-document inserts is highly inefficient. The `import` endpoint accepts JSONLines (JSONL) and features robust "dirty data" coercion parameters. + +#### Python Snippet (Bulk Import with Coercion) +```python +documents = [ + {"id": "1", "product_name": "Wireless Mouse", "category": "Electronics", "price": 29.99, "tags": ["pc", "accessory"]}, + {"id": "2", "product_name": "Mechanical Keyboard", "category": "Electronics", "price": "89.99", "tags": ["pc", "gaming"]} # price is a string here +] + +# We increase the connection timeout for imports to avoid client-side timeouts +import_client = typesense.Client({ + 'nodes': [{'host': 'localhost', 'port': '8108', 'protocol': 'http'}], + 'api_key': 'xyz', + 'connection_timeout_seconds': 300 +}) + +# Use coerce_or_reject to attempt to convert "89.99" (string) into 89.99 (float) +results = import_client.collections['products'].documents.import_( + documents, + {'action': 'upsert', 'dirty_values': 'coerce_or_reject'} +) + +print(results) +# Output will be a list of dicts/JSONL strings indicating success/failure for each row: +# [{'success': True}, {'success': True}] +``` + +#### Node.js Equivalent +```javascript +const documents = [ + { id: "1", product_name: "Wireless Mouse", category: "Electronics", price: 29.99, tags: ["pc", "accessory"] }, + { id: "2", product_name: "Mechanical Keyboard", category: "Electronics", price: "89.99", tags: ["pc", "gaming"] } +]; + +client.collections('products').documents().import(documents, { + action: 'upsert', + dirty_values: 'coerce_or_reject' +}); +``` + +#### CLI / Shell Equivalent +```bash +curl "http://localhost:8108/collections/products/documents/import?action=upsert&dirty_values=coerce_or_reject" \ + -X POST \ + -H "X-TYPESENSE-API-KEY: xyz" \ + -H "Content-Type: text/plain" \ + -d '{"id": "1", "product_name": "Wireless Mouse", "category": "Electronics", "price": 29.99, "tags": ["pc", "accessory"]} +{"id": "2", "product_name": "Mechanical Keyboard", "category": "Electronics", "price": "89.99", "tags": ["pc", "gaming"]}' +``` + +--- + +### Deep Dive: 3. Advanced Filtering & Search +Typesense supports sophisticated boolean filtering (`&&`, `||`, and parenthesis grouping), array filtering, and scoped filtering for arrays of nested objects. + +#### Python Snippet (Faceted Search with Nested and Boolean Filters) +```python +search_parameters = { + 'q': 'mouse', + 'query_by': 'product_name,description', + # Filter: Category is Electronics AND (Price is <= 50 OR tags contain "accessory") + 'filter_by': 'category:=Electronics && (price:<=50.0 || tags:=[accessory])', + 'facet_by': 'category,tags', + 'sort_by': 'price:asc', + 'per_page': 10 +} + +results = client.collections['products'].documents.search(search_parameters) +print(results) +``` + +#### Node.js Equivalent +```javascript +const searchParameters = { + q: 'mouse', + query_by: 'product_name,description', + filter_by: 'category:=Electronics && (price:<=50.0 || tags:=[accessory])', + facet_by: 'category,tags', + sort_by: 'price:asc', + per_page: 10 +}; + +client.collections('products').documents().search(searchParameters); +``` + +#### CLI / Shell Equivalent +```bash +curl -H "X-TYPESENSE-API-KEY: xyz" \ + "http://localhost:8108/collections/products/documents/search\ +?q=mouse\ +&query_by=product_name,description\ +&filter_by=category:=Electronics%20%26%20(price:<=50.0%20||%20tags:=[accessory])\ +&facet_by=category,tags\ +&sort_by=price:asc\ +&per_page=10" +``` + +--- + +## 3. Real-World Use Cases & Templates + +### Showcase Projects & Templates +1. **Typesense Recipe Search**: [showcase-recipe-search Repo](https://github.com/typesense/showcase-recipe-search) + * Indexes over 2 million cooking recipes. Demonstrates instant typo-tolerant search-as-you-type, multi-facet filtering, and high throughput. +2. **Airport Geo Search**: [showcase-airports-geosearch Repo](https://github.com/typesense/showcase-airports-geosearch) + * Built with Next.js and Typesense. Demonstrates geosearch capabilities, filtering results within a specific radius (`location:(lat, lng, 100 km)`), and distance-based sorting. +3. **HackerNews Semantic Search**: [showcase-hn-comments-semantic-search Repo](https://github.com/typesense/showcase-hn-comments-semantic-search) + * Demonstrates Hybrid Search (combining keyword and vector-based semantic search) on millions of HackerNews comments. + +### Common Integration Patterns +* **Algolia InstantSearch Integration**: Typesense provides a highly optimized adapter called `typesense-instantsearch-adapter` ([GitHub Repo](https://github.com/typesense/typesense-instantsearch-adapter)). This allows developers to drop Typesense directly into frontend applications built with Algolia's InstantSearch.js (including React, Vue, and Angular variants) with minimal config modifications. +* **Documentation Site Scraping**: The `typesense-docsearch-scraper` ([GitHub Repo](https://github.com/typesense/typesense-docsearch-scraper)) crawls websites, extracts structured content based on CSS selectors, and indexes them into Typesense. This is the standard open-source alternative to Algolia DocSearch, powering searches for Docusaurus and other documentation frameworks. + +--- + +## 4. Developer Friction Points & Edge Cases + +### 1. In-Place Field Type Alteration +* **Description**: Attempting to change an existing field's data type (e.g., from `int32` to `float` or `string`) using the collection update schema API. +* **Symptom / Error**: + ```text + RequestMalformed: [Errno 400] Schema change is incompatible with the type of documents already stored in this collection. Existing data for field XXX cannot be coerced... + ``` +* **Underlying Cause**: Typesense supports in-place schema changes (such as adding or dropping fields), but it validates stored documents against the new types. If existing documents cannot be coerced into the new type, the update fails. +* **Resolution**: Developers must perform a zero-downtime migration: + 1. Create a new collection with the updated schema (or use the Clone Collection API). + 2. Reindex/import all documents into the new collection. + 3. Update a Collection Alias to point to the new collection. + 4. Drop the old collection. +* **References**: [GitHub Issue #96](https://github.com/typesense/typesense/issues/96) and [GitHub Issue #1211](https://github.com/typesense/typesense/issues/1211). + +### 2. Updating Auto-Embedding Models In-Place +* **Description**: Changing the model of an auto-embedding vector field (e.g., from `ts/e5-small` to `ts/all-MiniLM-L12-v2`) in a single collection update. +* **Symptom / Error**: + ```text + RequestMalformed: [Errno 400] Schema change is incompatible with the type of documents already stored in this collection. error: Field content_embedding contains an invalid embedding. + ``` +* **Underlying Cause**: When altering a vector field's model configuration, Typesense validates the dimensions of existing stored embeddings against the new model's expected dimensions (e.g., 384 vs 768 dimensions), causing immediate validation failure. +* **Resolution**: The modification must be done in two separate schema update requests: + 1. First API Call: Update the collection schema to drop the vector field. + 2. Second API Call: Update the schema to add the vector field back with the new model configuration. Typesense will then regenerate the embeddings for all existing documents in the background. +* **References**: [GitHub Issue #1450](https://github.com/typesense/typesense/issues/1450). + +### 3. Sibling Object Filtering in Arrays of Objects +* **Description**: Filtering on multiple fields inside an array of nested objects (e.g., matching recipes containing "cheese" with "concentration < 50"). +* **Symptom / Error**: Standard dot-notation filters like `ingredients.name:=cheese && ingredients.concentration:<50` return documents where "cheese" is in one array element, and another element has a concentration < 50, rather than matching both conditions on the *same* nested object. +* **Underlying Cause**: Typesense flattens nested arrays of objects into separate arrays of primitives, losing the sibling relationship between properties in individual objects. +* **Resolution**: Developers must use the special scoped nested array syntax: `ingredients.{name:=cheese && concentration:<50}`. This instructs Typesense to evaluate the boolean expression against each sibling object individually. +* **References**: [GitHub Issue #828](https://github.com/typesense/typesense/issues/828) and [GitHub Issue #2261](https://github.com/typesense/typesense/issues/2261). + +--- + +## 5. Evaluation Ideas (Self-Contained & Container-Ready) + +The following benchmark tasks are designed to be fully self-contained. They download the native Typesense Linux binary, run it in the background inside the agent's Docker container, and execute automated test scripts to verify correct implementation. + +### [Simple] Task 1: Basic Collection Setup & Document CRUD +* **Goal**: Write a script to download the Typesense standalone binary, launch it on port 8108, create a `books` collection with an explicit schema, and implement basic CRUD operations (create, retrieve, partial update, delete). + +### [Medium] Task 2: Dirty Data Import & Coercion Handling +* **Goal**: Create a `devices` collection with auto-schema detection (`.*` of type `auto`) and write an import script that successfully ingests a dirty dataset containing mixed types (e.g., stringified integers and nulls) using the `dirty_values: "coerce_or_reject"` parameter. + +### [Medium] Task 3: Sibling Object Filtering on Nested Arrays +* **Goal**: Define a `recipes` collection schema with nested fields enabled (`enable_nested_fields: true`) and an array of objects (`ingredients`), index sample recipes, and implement a search script that correctly uses the scoped nested array syntax (`ingredients.{name:=X && concentration: +
+
+ ); +} + +function ScoreCell({ value }: { value: number }) { + let colorClass = "bg-primary"; + let textClass = "text-muted-foreground"; + + if (value >= 90) { + colorClass = "bg-emerald-500"; + textClass = "text-emerald-500 font-bold"; + } else if (value >= 75) { + colorClass = "bg-blue-500"; + textClass = "text-blue-500 font-medium"; + } else if (value >= 60) { + colorClass = "bg-amber-500"; + textClass = "text-amber-500"; + } else { + colorClass = "bg-red-500"; + textClass = "text-red-500"; + } + + return ( +
+ {value}% + +
+ ); +} + +export default function LeaderboardTable({ data }: { data: LeaderboardEntry[] }) { + const router = useRouter(); + const [devMode, setDevMode] = useState(process.env.NODE_ENV === "development"); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevMode(isDev); + }, []); + + const filteredData = useMemo(() => { + let processedData = data; + const config = zealtConfig as { pending_models?: string[] }; + + if (!devMode && config.pending_models && config.pending_models.length > 0) { + processedData = processedData.filter((item) => + !config.pending_models!.includes(item.rawModel) + ); + } + + if (searchQuery) { + const query = searchQuery.toLowerCase(); + processedData = processedData.filter(item => + item.model.toLowerCase().includes(query) + ); + } + + return processedData; + }, [data, searchQuery, devMode]); + + return ( + <> + {/* Controls & Filters */} +
+

+ Model Performance +

+ +
+ + + View Tasks + + +
+ + setSearchQuery(e.target.value)} + className="pl-9 pr-4 py-2 bg-card border border-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary/20 w-full sm:w-64 transition-all" + /> +
+
+
+ + {/* Leaderboard Table */} +
+
+ + + + + + + + + + + {filteredData.length === 0 ? ( + + + + ) : ( + filteredData.map((row, index) => ( + router.push(`./tasks?model=${encodeURIComponent(row.model)}`)} + className="group hover:bg-secondary/30 transition-colors duration-200 cursor-pointer" + > + + + + + + )))} + +
ModelPassedAvg DurationSuccess Rate
+ No results found matching your search. +
+ #{index + 1} +
+ + {row.model} + {index === 0 && } + {row.isNew && ( + + NEW + + )} + +
+ 		+ {row.passedEvals} + + {row.avgLatency > 0 ? `${row.avgLatency.toFixed(1)}s` : '-'} + +
+ +
+
+
+
+ + ); +} diff --git a/site/app/(home)/page.tsx b/site/app/(home)/page.tsx new file mode 100644 index 00000000..d6171231 --- /dev/null +++ b/site/app/(home)/page.tsx @@ -0,0 +1,176 @@ +import { Suspense } from "react"; +import { Github, Terminal, ClipboardList, ListTree } from "lucide-react"; +import Link from "next/link"; +import tasksData from "@/zealt/tasks.json"; +import pendingTasksData from "@/zealt/pending-tasks.json"; +import zealtConfig from "@/zealt/config.json"; +import PendingReviewCard from "@/components/pending-review-card"; +import LeaderboardTable, { type LeaderboardEntry } from "./components/leaderboard-table"; + +type TaskTrial = { + agent: string; + model: string; + passed: boolean; + latency_sec: number | null; +}; + +type TaskValue = { + trials?: TaskTrial[]; +}; + +type PendingTasksValue = { + 'pending-tasks'?: number; +}; + +export default function Home() { + const totalTasks = Object.keys(tasksData as Record).length; + const hasTasks = totalTasks > 0; + const pendingSampleCases = Math.max( + 0, + Number((pendingTasksData as PendingTasksValue)['pending-tasks'] ?? 0), + ); + + // Process tasks.json to compute leaderboard stats directly on the server + const statsMap = new Map(); + + Object.values(tasksData as Record).forEach((taskValue) => { + let trials: TaskTrial[] = []; + if (Array.isArray(taskValue)) { + trials = taskValue as TaskTrial[]; + } else if (typeof taskValue === "object" && taskValue !== null) { + const task = taskValue as TaskValue; + trials = Array.isArray(task.trials) ? task.trials : []; + } + + trials.forEach((trial) => { + // Simplify model name + const modelName = trial.model.split('/').pop() || trial.model; + const agentName = trial.agent.charAt(0).toUpperCase() + trial.agent.slice(1); + + const key = `${modelName}-${agentName}`; + + if (!statsMap.has(key)) { + statsMap.set(key, { + passed: 0, + total: 0, + totalLatency: 0, + latencyCount: 0, + model: modelName, + rawModel: trial.model, + agent: agentName + }); + } + + const stats = statsMap.get(key); + if (!stats) { + return; + } + stats.total += 1; + if (trial.passed) { + stats.passed += 1; + } + if (trial.latency_sec) { + stats.totalLatency += trial.latency_sec; + stats.latencyCount += 1; + } + }); + }); + + const data: LeaderboardEntry[] = Array.from(statsMap.values()) + .map((stats, index) => { + const successRate = stats.total > 0 ? Math.round((stats.passed / stats.total) * 100) : 0; + const avgLatency = stats.latencyCount > 0 ? stats.totalLatency / stats.latencyCount : 0; + return { + id: String(index + 1), + model: stats.model, + rawModel: stats.rawModel, + agent: stats.agent, + passedEvals: stats.passed, + successRate: successRate, + avgLatency: avgLatency, + }; + }) + .sort((a, b) => b.successRate - a.successRate); + + // Re-assign IDs based on sorted order and adjust isNew + data.forEach((item, index) => { + item.id = String(index + 1); + item.isNew = index === 0; // Keeping the original visual effect for the top item + }); + + return ( +
+ {/* Background Gradient Effect */} +
+ +
+ {/* Header Section */} +
+
+ + Live Benchmarks +
+ +

+ {zealtConfig.title} +

+ +

+ {zealtConfig.description} +

+ +
+ + + View on GitHub + + {data.length > 0 && ( + <> +
+ + + Total tasks: {totalTasks} + + + )} +
+ + + Last run: {new Date().toLocaleDateString()} + +
+
+ + {!hasTasks ? ( + + ) : data.length === 0 ? ( +
+

No evaluation data yet

+
+ + + View Tasks + +
+
+ ) : ( + // Client Component for Interactive Table + Loading leaderboard...
}> + + + )} +
+ + ); +} diff --git a/site/app/globals.css b/site/app/globals.css new file mode 100644 index 00000000..437c041d --- /dev/null +++ b/site/app/globals.css @@ -0,0 +1,149 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); + --color-background: var(--background); + --color-foreground: var(--foreground); + --color-card: var(--card); + --color-card-foreground: var(--card-foreground); + --color-popover: var(--popover); + --color-popover-foreground: var(--popover-foreground); + --color-primary: var(--primary); + --color-primary-foreground: var(--primary-foreground); + --color-secondary: var(--secondary); + --color-secondary-foreground: var(--secondary-foreground); + --color-muted: var(--muted); + --color-muted-foreground: var(--muted-foreground); + --color-accent: var(--accent); + --color-accent-foreground: var(--accent-foreground); + --color-destructive: var(--destructive); + --color-border: var(--border); + --color-input: var(--input); + --color-ring: var(--ring); + --color-chart-1: var(--chart-1); + --color-chart-2: var(--chart-2); + --color-chart-3: var(--chart-3); + --color-chart-4: var(--chart-4); + --color-chart-5: var(--chart-5); + --color-sidebar: var(--sidebar); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.141 0.005 285.823); + --card: oklch(1 0 0); + --card-foreground: oklch(0.141 0.005 285.823); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.141 0.005 285.823); + --primary: oklch(0.21 0.006 285.885); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.967 0.001 286.375); + --secondary-foreground: oklch(0.21 0.006 285.885); + --muted: oklch(0.967 0.001 286.375); + --muted-foreground: oklch(0.552 0.016 285.938); + --accent: oklch(0.967 0.001 286.375); + --accent-foreground: oklch(0.21 0.006 285.885); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.92 0.004 286.32); + --input: oklch(0.92 0.004 286.32); + --ring: oklch(0.705 0.015 286.067); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.141 0.005 285.823); + --sidebar-primary: oklch(0.21 0.006 285.885); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.967 0.001 286.375); + --sidebar-accent-foreground: oklch(0.21 0.006 285.885); + --sidebar-border: oklch(0.92 0.004 286.32); + --sidebar-ring: oklch(0.705 0.015 286.067); +} + +.dark { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.21 0.006 285.885); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.92 0.004 286.32); + --primary-foreground: oklch(0.21 0.006 285.885); + --secondary: oklch(0.274 0.006 286.033); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --accent: oklch(0.274 0.006 286.033); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.552 0.016 285.938); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.21 0.006 285.885); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.274 0.006 286.033); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.552 0.016 285.938); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} + +@layer utilities { + .custom-scrollbar { + scrollbar-width: thin; + scrollbar-color: var(--border) transparent; + } + + .custom-scrollbar::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + .custom-scrollbar::-webkit-scrollbar-track { + background: transparent; + } + + .custom-scrollbar::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; + } + + .custom-scrollbar::-webkit-scrollbar-thumb:hover { + background: color-mix(in oklch, var(--foreground) 30%, transparent); + } +} \ No newline at end of file diff --git a/site/app/jobs/[jobName]/[trialName]/run/components/artifacts-panel.tsx b/site/app/jobs/[jobName]/[trialName]/run/components/artifacts-panel.tsx new file mode 100644 index 00000000..bf9202c1 --- /dev/null +++ b/site/app/jobs/[jobName]/[trialName]/run/components/artifacts-panel.tsx @@ -0,0 +1,265 @@ +"use client"; + +import { useMemo } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { ChevronRight, File, Folder } from "lucide-react"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + fetchLogText, + LogContentSkeleton, + LogErrorView, + getLogErrorMessage, +} from "./trajectory-page"; + +export type ArtifactNode = { + name: string; + type: "file" | "dir"; + path: string; + children?: ArtifactNode[]; +}; + +export type ArtifactNodeWithUrl = + | { name: string; type: "file"; path: string; url: string } + | { name: string; type: "dir"; path: string; children: ArtifactNodeWithUrl[] }; + +type ArtifactsPanelProps = { + artifactTree: ArtifactNodeWithUrl[]; +}; + +export function ArtifactsPanel({ artifactTree }: ArtifactsPanelProps) { + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + + const queryArtifact = searchParams.get("artifact"); + const queryPath = searchParams.get("path"); + + const activeArtifact = useMemo(() => { + if (queryArtifact) { + const found = artifactTree.find((node) => node.name === queryArtifact); + if (found) return found; + } + return artifactTree[0] ?? null; + }, [artifactTree, queryArtifact]); + + const updateParams = (next: { artifact?: string | null; path?: string | null }) => { + const params = new URLSearchParams(searchParams.toString()); + if (next.artifact === null) { + params.delete("artifact"); + } else if (next.artifact !== undefined) { + params.set("artifact", next.artifact); + } + if (next.path === null) { + params.delete("path"); + } else if (next.path !== undefined) { + params.set("path", next.path); + } + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + + const handleSelectArtifact = (name: string) => { + updateParams({ artifact: name, path: null }); + }; + + if (!activeArtifact) { + return ( +
+ No artifacts available. +
+ ); + } + + return ( +
+
+
+ {artifactTree.map((node) => ( + + ))} +
+
+ +
+ {activeArtifact.type === "file" ? ( + + ) : ( + updateParams({ path: path ?? null })} + /> + )} +
+
+ ); +} + +type DirectoryArtifactViewProps = { + artifact: Extract; + currentPath: string | null; + onPathChange: (path: string | null) => void; +}; + +function DirectoryArtifactView({ artifact, currentPath, onPathChange }: DirectoryArtifactViewProps) { + // Resolve the active path. If currentPath isn't valid for this artifact, fall back to the artifact root. + const resolved = useMemo(() => resolvePath(artifact, currentPath), [artifact, currentPath]); + const { folder, selectedFile, breadcrumbDirs } = resolved; + + return ( +
+
+ {breadcrumbDirs.map((dir, idx) => { + const isLast = idx === breadcrumbDirs.length - 1 && !selectedFile; + return ( + + + {(idx < breadcrumbDirs.length - 1 || selectedFile) && ( + + )} + + ); + })} + {selectedFile && ( + {selectedFile.name} + )} +
+ +
+
+ {folder.children.length === 0 && ( + Empty directory. + )} + {folder.children.map((child) => { + const isActive = child.type === "file" && selectedFile?.path === child.path; + return ( + + ); + })} +
+
+ +
+ {selectedFile ? ( + + ) : ( +
+ Select a file to view its contents. +
+ )} +
+
+ ); +} + +function FileViewer({ url }: { url: string }) { + const query = useQuery({ + queryKey: ["artifact-file", url], + queryFn: () => fetchLogText(url), + }); + + return ( + +
+ {query.isPending || query.isFetching ? ( + + ) : query.isError ? ( + void query.refetch()} + /> + ) : query.data ? ( + 
+            {query.data}
+
+ ) : ( +

Empty file.

+ )} +
+ + ); +} + +type ResolvedPath = { + folder: Extract; + selectedFile: Extract | null; + breadcrumbDirs: { name: string; path: string }[]; +}; + +function resolvePath( + root: Extract, + targetPath: string | null, +): ResolvedPath { + const breadcrumbDirs: { name: string; path: string }[] = [{ name: root.name, path: root.path }]; + let folder = root; + let selectedFile: Extract | null = null; + + if (!targetPath || !targetPath.startsWith(`${root.path}/`)) { + return { folder, selectedFile, breadcrumbDirs }; + } + + const remainder = targetPath.slice(root.path.length + 1); + const segments = remainder.split("/").filter(Boolean); + + let currentPath = root.path; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]; + currentPath = `${currentPath}/${segment}`; + const child = folder.children.find((c) => c.name === segment); + if (!child) { + // Path doesn't exist; bail and show what we resolved so far. + return { folder, selectedFile: null, breadcrumbDirs }; + } + if (child.type === "dir") { + folder = child; + breadcrumbDirs.push({ name: child.name, path: child.path }); + } else { + // File at the end of the path. + selectedFile = child; + // Don't push the file as a breadcrumb dir; the caller renders it separately. + break; + } + } + + return { folder, selectedFile, breadcrumbDirs }; +} diff --git a/site/app/jobs/[jobName]/[trialName]/run/components/trajectory-page.tsx b/site/app/jobs/[jobName]/[trialName]/run/components/trajectory-page.tsx new file mode 100644 index 00000000..0fab8fc8 --- /dev/null +++ b/site/app/jobs/[jobName]/[trialName]/run/components/trajectory-page.tsx @@ -0,0 +1,404 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useTheme } from "next-themes"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Button } from "@/components/ui/button"; +import { HttpError } from "@/lib/http-error"; +import { ArtifactsPanel, type ArtifactNodeWithUrl } from "./artifacts-panel"; + +export type TabConfig = { + value: string; + label: React.ReactNode; +}; + +type TrajectoryPageProps = { + trajectoryUrl: string; + browserVerificationUrls: { name: string; url: string }[]; + fallbackUrl: string; + stderrLogUrl: string | null; + verifierLogUrl: string | null; + tabsConfig: TabConfig[]; + artifactTree?: ArtifactNodeWithUrl[]; +}; + +export async function fetchLogText(url: string): Promise { + let response: Response; + try { + response = await fetch(url, { cache: "force-cache" }); + } catch (_e) { + throw new HttpError("Network request failed."); + } + + if (!response.ok) { + throw new HttpError(`Request failed with status ${response.status}.`, { status: response.status }); + } + + return response.text(); +} + +export function TrajectoryPage({ + trajectoryUrl, + browserVerificationUrls, + fallbackUrl, + stderrLogUrl, + verifierLogUrl, + tabsConfig, + artifactTree, +}: TrajectoryPageProps) { + const { resolvedTheme } = useTheme(); + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + const [mounted, setMounted] = useState(false); + const [iframeLoading, setIframeLoading] = useState(true); + const [devModeEnabled, setDevModeEnabled] = useState(process.env.NODE_ENV === "development"); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevModeEnabled(isDev); + }, []); + + const visibleTabsConfig = useMemo( + () => + devModeEnabled + ? tabsConfig + : tabsConfig.filter((t) => t.value !== "artifacts"), + [tabsConfig, devModeEnabled], + ); + const validTabs = visibleTabsConfig.map((t) => t.value); + const [activeTab, setActiveTab] = useState(() => { + const queryTab = searchParams.get("tab"); + return queryTab && validTabs.includes(queryTab) ? queryTab : validTabs[0]; + }); + + const handleTabChange = (value: string) => { + setActiveTab(value); + const params = new URLSearchParams(searchParams.toString()); + params.set("tab", value); + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + const [browserIframeLoading, setBrowserIframeLoading] = useState>({}); + const [activeBrowserVerificationTab, setActiveBrowserVerificationTab] = useState( + browserVerificationUrls[0]?.name || "", + ); + + const iframeTheme = mounted && resolvedTheme === "light" ? "light" : "dark"; + + const iframeUrl = useMemo(() => { + const url = new URL(trajectoryUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [trajectoryUrl, iframeTheme]); + + const activeBrowserVerificationBaseUrl = useMemo(() => { + const testCase = browserVerificationUrls.find((tc) => tc.name === activeBrowserVerificationTab); + return testCase ? testCase.url : null; + }, [browserVerificationUrls, activeBrowserVerificationTab]); + + const activeBrowserVerificationUrl = useMemo(() => { + if (!activeBrowserVerificationBaseUrl) { + return null; + } + + const url = new URL(activeBrowserVerificationBaseUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [activeBrowserVerificationBaseUrl, iframeTheme]); + + useEffect(() => { + setMounted(true); + }, []); + + useEffect(() => { + if (!mounted || !activeBrowserVerificationBaseUrl) { + return; + } + + setBrowserIframeLoading((prev) => ({ ...prev, [activeBrowserVerificationTab]: true })); + }, [activeBrowserVerificationBaseUrl, activeBrowserVerificationTab, mounted]); + + const stderrQuery = useQuery({ + queryKey: ["trajectory-stderr", stderrLogUrl], + enabled: Boolean(stderrLogUrl), + queryFn: async () => { + if (!stderrLogUrl) { + return null; + } + + return fetchLogText(stderrLogUrl); + }, + }); + + const verifierQuery = useQuery({ + queryKey: ["trajectory-verifier", verifierLogUrl], + enabled: Boolean(verifierLogUrl), + queryFn: async () => { + if (!verifierLogUrl) { + return null; + } + + return fetchLogText(verifierLogUrl); + }, + }); + + const handleIframeLoad = () => { + setIframeLoading(false); + }; + + const handleBrowserIframeLoad = (testCaseName: string) => { + setBrowserIframeLoading((prev) => ({ ...prev, [testCaseName]: false })); + }; + + const handleIframeError = () => { + window.location.replace(fallbackUrl); + }; + + const renderLogContent = ( + text: string | null | undefined, + isLoading: boolean, + isError: boolean, + error: unknown, + onRetry: () => void, + emptyMessage: string, + ) => { + if (isLoading) { + return ; + } + + if (isError) { + return ; + } + + if (!text) { + return

{emptyMessage}

; + } + + return ( + 
+        {text}
+
+ ); + }; + + return ( +
+
+ +
+ + {visibleTabsConfig.map((tab) => ( + + {tab.label} + + ))} + +
+ + +
+ +
+ {mounted && ( +