diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..828b276 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,69 @@ +name: Deploy Next.js site to Pages + +on: + push: + branches: ["main"] + pull_request: + types: [opened, reopened, synchronize, closed] + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --frozen-lockfile + working-directory: site + + - name: Setup Pages + id: setup_pages + uses: actions/configure-pages@v5 + + - name: Set base uri + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}/pr-preview/pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" + else + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}" >> "$GITHUB_ENV" + fi + + - name: Build with Next.js + run: bun run build + working-directory: site + + - name: Deploy preview + if: github.event_name == 'pull_request' + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site/out + + - name: Deploy production + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + clean-exclude: pr-preview/ + force: false + folder: site/out diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0cf2fa4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +__pycache__/ +*.py[cod] +*$py.class + +# testing +coverage + +# next.js +.next/ + +# The `out` directory should not be ignored by version control +out/ + +# production +build + +# misc +.DS_Store +*.pem +*~ +\#* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.zealt/config.json b/.zealt/config.json new file mode 100644 index 0000000..308d9d5 --- /dev/null +++ b/.zealt/config.json @@ -0,0 +1,5 @@ +{ + "title": "Godot Benchmark", + "description": "Performance results of AI coding models on Godot tasks, measuring success rate and execution time with high precision.", + "github_repo": "https://github.com/kweizh/godot-benchmark" +} \ No newline at end of file diff --git a/README.md b/README.md index dc8b6a0..a1e93fc 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# godot-benchmark \ No newline at end of file + +# Godot Benchmark + +This repository contains benchmarks for evaluating AI models on **Godot**. + +You can view the evaluation reports at [https://kweizh.github.io/godot-benchmark/](https://kweizh.github.io/godot-benchmark/). + +## Project Structure + +- `tasks/`: Contains the benchmark tasks, each with its own instructions. +- `jobs/`: Stores the results of benchmark runs. +- `site/`: A Next.js application to visualize benchmark results. + +## Getting Started + +This benchmark is evaluated using the [Harbor framework](https://github.com/harbor-framework/harbor) and the [Pochi agent](https://github.com/TabbyML/pochi). + +### Running Evaluation + +You can run the evaluation using the Harbor CLI. Here is an example: + +```bash +harbor run \ + --agent codex \ + --model "gpt-5.2-codex" \ + --env daytona \ + --path ./tasks \ + --n-attempts 1 \ + --max-retries 5 \ + --n-concurrent 5 \ + --retry-include RuntimeError \ + --retry-include DaytonaError \ + --retry-include AgentTimeoutError +``` + +### Evaluation Details + +Before starting the evaluation, you should set the necessary environment variables for your chosen agent. +For example, if using Pochi, you should export `POCHI_API_KEY`. + +Evaluation can be run locally with Docker (default), or using [Daytona.io](https://www.daytona.io/) by setting `--env daytona`. + +When running with Daytona, please note that Daytona blocks some network access for tier 1 and tier 2 users. If you meet any network issues, please refer to [Daytona network limits](https://www.daytona.io/docs/en/network-limits/). + +--- +Generated by [Zealt](https://github.com/TabbyML/zealt) diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..78d7bbb --- /dev/null +++ b/plan.md @@ -0,0 +1,108 @@ +# Godot Engine Evaluation Dataset Research + +### 1. Library Overview +* **Description**: Godot Engine is a free, open-source, cross-platform 2D and 3D game engine. It features a unique node-and-scene architecture, a dedicated Python-like scripting language (GDScript), and support for C# and C++ (via GDExtension). +* **Ecosystem Role**: A major competitor to Unity and Unreal Engine, favored for its lightweight nature, permissive MIT license, and excellent 2D capabilities. It is increasingly used for 3D games and non-game applications (tools, simulators). +* **Project Setup**: + 1. **Download**: Godot is a single executable. No installation required. + 2. **Initialize**: Create a new folder and a `project.godot` file (automatically done via the Project Manager). + 3. **CLI**: + * Open editor: `godot -e` + * Run project: `godot` + * Export: `godot --export-release "Linux/X11" path/to/export` + 4. **Structure**: Standard practice uses `res://` as the root. Common folders: `scenes/`, `scripts/`, `assets/`, `prefabs/`. + +### 2. Core Primitives & APIs + +* **Nodes & Scenes**: Everything is a Node. Nodes are organized into Scenes. Scenes can be instanced within other scenes. + * [Nodes and Scenes Docs](https://docs.godotengine.org/en/stable/getting_started/step_by_step/nodes_and_scenes.html) +* **GDScript**: A high-level, dynamically typed language optimized for Godot. + * [GDScript Basics](https://docs.godotengine.org/en/stable/tutorials/scripting/gdscript/gdscript_basics.html) + * **Snippet (Basic Player)**: + ```gdscript + extends CharacterBody2D + + @export var speed = 300.0 + @export var jump_velocity = -400.0 + + func _physics_process(delta): + # Add gravity + if not is_on_floor(): + velocity += get_gravity() * delta + + # Handle Jump + if Input.is_action_just_pressed("ui_accept") and is_on_floor(): + velocity.y = jump_velocity + + # Get input direction + var direction = Input.get_axis("ui_left", "ui_right") + if direction: + velocity.x = direction * speed + else: + velocity.x = move_toward(velocity.x, 0, speed) + + move_and_slide() + ``` +* **Signals**: The Observer pattern implementation. Used for decoupled communication. + * [Using Signals](https://docs.godotengine.org/en/stable/getting_started/step_by_step/signals.html) + * **Snippet (Connecting via Code)**: + ```gdscript + func _ready(): + var timer = get_node("Timer") + timer.timeout.connect(_on_timer_timeout) + + func _on_timer_timeout(): + print("Timer finished!") + ``` +* **Resources**: Data containers (e.g., Textures, Scripts, custom data). + * [Resources Docs](https://docs.godotengine.org/en/stable/tutorials/scripting/resources.html) + * **Snippet (Custom Resource)**: + ```gdscript + # item_data.gd + extends Resource + class_name ItemData + + @export var name: String + @export var icon: Texture2D + @export var damage: int + ``` +* **Networking**: High-level multiplayer API using RPCs and Synchronizers. + * [Multiplayer Docs](https://docs.godotengine.org/en/stable/tutorials/networking/high_level_multiplayer.html) + * **Snippet (RPC)**: + ```gdscript + @rpc("any_peer", "call_local") + func update_score(value): + score += value + ``` +* **GDExtension (C++)**: High-performance extension system without recompiling the engine. + * [GDExtension Docs](https://docs.godotengine.org/en/stable/tutorials/scripting/gdextension/gdextension_cpp_example.html) + * **Key Concept**: Requires a `.gdextension` config file to map shared libraries to platforms. + +### 3. Real-World Use Cases & Templates +* **SaaS/Tool UI**: Using `Control` nodes, `GridContainer`, and `Theme` for complex editors. +* **Multiplayer FPS/Platformer**: Utilizing `MultiplayerSynchronizer` for state and `MultiplayerSpawner` for dynamic entities. +* **Procedural Generation**: Using `TileMapLayer` and `FastNoiseLite` for infinite worlds. +* **Official Demos**: [Godot Demo Projects Repository](https://github.com/godotengine/godot-demo-projects). + +### 4. Developer Friction Points +* **Circular Dependencies**: GDScript can fail to load scripts if they reference each other in a loop (e.g., `A.gd` uses `B.gd` and vice versa). [Issue Discussion](https://github.com/godotengine/godot/issues/78040). +* **Tween API Changes**: Migration from Godot 3 `Tween` node to Godot 4 `create_tween()` method is a frequent source of confusion. +* **GDExtension Setup**: The boilerplate for C++ (SCons, godot-cpp, registration macros) is significantly steeper than GDScript. +* **NavigationServer**: Handling dynamic obstacles with `NavigationAgent` and `NavigationRegion` often requires complex setup of baking/avoidance layers. + +### 5. Evaluation Ideas +* **Basic**: Implement a "Coin Collector" logic where a player (CharacterBody2D) collects items (Area2D) and updates a UI label. +* **Intermediate**: Create a custom `Resource` for "Enemy Stats" and a system that loads these resources to spawn different enemy types. +* **Intermediate**: Build a nested UI menu that supports both mouse clicking and keyboard/gamepad focus navigation. +* **Advanced**: Implement a "Dissolve" shader effect using `VisualShader` or GDShader code that triggers when an enemy dies. +* **Advanced**: Set up a basic client-server lobby where players can join, and their positions are synced using `MultiplayerSynchronizer`. +* **Advanced**: Refactor a GDScript-based heavy calculation (e.g., pathfinding or mesh generation) into a GDExtension C++ class. + +### 6. Sources +1. [Godot Official Documentation](https://docs.godotengine.org/en/stable/) - Primary source for all API details. +2. [Godot 4.6 branch index](https://docs.godotengine.org/en/stable/index.html) - Documentation root. +3. [GDScript Basics](https://docs.godotengine.org/en/stable/tutorials/scripting/gdscript/gdscript_basics.html) - Language reference. +4. [GDExtension C++ Example](https://docs.godotengine.org/en/stable/tutorials/scripting/gdextension/gdextension_cpp_example.html) - C++ integration guide. +5. [High-level Multiplayer](https://docs.godotengine.org/en/stable/tutorials/networking/high_level_multiplayer.html) - Networking API. +6. [Godot GitHub Issues](https://github.com/godotengine/godot/issues) - Source for friction points and bugs. +7. [GDQuest Tutorials](https://www.gdquest.com/) - Best practices for signals and architecture. \ No newline at end of file diff --git a/scratchpad/pending-tasks/accessible_ui_navigation_implementation/instruction.md b/scratchpad/pending-tasks/accessible_ui_navigation_implementation/instruction.md new file mode 100644 index 0000000..5f7f3bb --- /dev/null +++ b/scratchpad/pending-tasks/accessible_ui_navigation_implementation/instruction.md @@ -0,0 +1,8 @@ +You are developing a complex settings menu for a SaaS tool built in Godot. The menu must be fully accessible via both mouse clicks and keyboard/gamepad focus navigation. + +You need to construct a UI script that manages a `GridContainer` populated with multiple `Button` nodes. The script must programmatically assign the focus neighbors (`focus_neighbor_left`, `focus_neighbor_right`, etc.) for every button in the grid so that directional input smoothly wraps around the edges of the grid (e.g., pressing right on the last item of a row focuses the first item of that row). + +**Constraints:** +- You MUST dynamically calculate and assign the focus properties via script based on the `GridContainer`'s columns and child count. +- Do NOT hardcode the paths or names of specific buttons. +- The script must handle edge cases, such as an incomplete bottom row in the grid. \ No newline at end of file diff --git a/scratchpad/pending-tasks/basic_2d_player_movement_and_signal_integration/instruction.md b/scratchpad/pending-tasks/basic_2d_player_movement_and_signal_integration/instruction.md new file mode 100644 index 0000000..a757cb8 --- /dev/null +++ b/scratchpad/pending-tasks/basic_2d_player_movement_and_signal_integration/instruction.md @@ -0,0 +1,8 @@ +You are building a simple platformer prototype where a player moves around and collects coins. + +You need to write a GDScript for a `CharacterBody2D` that handles basic left/right movement, jumping, and gravity. Additionally, you must implement a function to connect to an `Area2D`'s `body_entered` signal dynamically via code to increment a local score variable when the player touches a coin. + +**Constraints:** +- You MUST use Godot 4 signal syntax (e.g., `signal_name.connect(callable)`). +- You MUST use standard 2D physics methods like `move_and_slide()` and `is_on_floor()`. +- Do NOT use the editor UI to connect the signal; it must be done entirely in script. \ No newline at end of file diff --git a/scratchpad/pending-tasks/custom_resource_creation_and_loading/instruction.md b/scratchpad/pending-tasks/custom_resource_creation_and_loading/instruction.md new file mode 100644 index 0000000..dab5666 --- /dev/null +++ b/scratchpad/pending-tasks/custom_resource_creation_and_loading/instruction.md @@ -0,0 +1,8 @@ +You are designing a data-driven system for defining different enemy types without duplicating scene nodes. + +You need to create a custom GDScript `Resource` named `EnemyStats` that defines exported properties for an enemy's name (String), health (int), and speed (float). Following this, write a separate spawner script that exports an array of `EnemyStats` resources and iterates through them on `_ready()`, printing each enemy's name to the console. + +**Constraints:** +- The custom resource MUST use the `class_name EnemyStats` declaration. +- You MUST use the `@export` annotation to expose the variables to the inspector. +- Do NOT instantiate any physical nodes in the spawner script; only handle the resource data. \ No newline at end of file diff --git a/scratchpad/pending-tasks/high_level_multiplayer_synchronization/instruction.md b/scratchpad/pending-tasks/high_level_multiplayer_synchronization/instruction.md new file mode 100644 index 0000000..28ea8e6 --- /dev/null +++ b/scratchpad/pending-tasks/high_level_multiplayer_synchronization/instruction.md @@ -0,0 +1,8 @@ +You are developing a fast-paced multiplayer arena game and need to synchronize game state across multiple connected clients. + +You need to implement a GDScript for a player entity that configures a `MultiplayerSynchronizer` to automatically sync the player's `global_position` across the network. Additionally, implement an RPC method to broadcast a score update whenever a player scores a point, ensuring the update executes locally and on all peers. + +**Constraints:** +- The score update function MUST use the `@rpc("any_peer", "call_local")` annotation. +- Position syncing must rely EXCLUSIVELY on `MultiplayerSynchronizer` configuration; do not manually send position data via RPCs. +- The script must assume the network peer and multiplayer authority are already established. \ No newline at end of file diff --git a/scratchpad/pending-tasks/modernizing_tween_animations/instruction.md b/scratchpad/pending-tasks/modernizing_tween_animations/instruction.md new file mode 100644 index 0000000..055c8fe --- /dev/null +++ b/scratchpad/pending-tasks/modernizing_tween_animations/instruction.md @@ -0,0 +1,8 @@ +You are tasked with migrating a legacy Godot 3 script to Godot 4. The old script relies on an outdated `Tween` node to animate a UI panel's appearance. + +You need to rewrite the animation logic to use Godot 4's built-in `create_tween()` method. The script must animate the UI Control's `scale` property from `Vector2(0, 0)` to `Vector2(1, 1)` over a duration of 0.5 seconds, applying an ease-out transition. + +**Constraints:** +- Do NOT use or reference a `Tween` node in the scene tree. +- You MUST use `tween_property()` to execute the animation. +- The script must be fully compatible with Godot 4's SceneTreeTween API. \ No newline at end of file diff --git a/scratchpad/pending-tasks/resolving_gdscript_circular_dependencies/instruction.md b/scratchpad/pending-tasks/resolving_gdscript_circular_dependencies/instruction.md new file mode 100644 index 0000000..f036049 --- /dev/null +++ b/scratchpad/pending-tasks/resolving_gdscript_circular_dependencies/instruction.md @@ -0,0 +1,8 @@ +You are debugging a project that fails to load due to a cyclical reference error. `Player.gd` and `Weapon.gd` strongly type reference each other using `class_name` (e.g., the Player script has a variable typed as `Weapon`, and the Weapon script has a variable typed as `Player`). + +You need to refactor both scripts to successfully resolve the circular dependency while maintaining the ability for the `Weapon` to call a `take_damage()` method on the `Player`, and the `Player` to access the `Weapon`'s `damage` property. + +**Constraints:** +- Both files MUST remain written in GDScript. +- You cannot combine both classes into a single file. +- The resulting code must compile and run without throwing parse errors or cyclic dependency warnings. \ No newline at end of file diff --git a/site/app/(home)/components/leaderboard-table.tsx b/site/app/(home)/components/leaderboard-table.tsx new file mode 100644 index 0000000..d10b1f0 --- /dev/null +++ b/site/app/(home)/components/leaderboard-table.tsx @@ -0,0 +1,183 @@ +"use client"; + +import { useState, useMemo, useEffect } from "react"; +import { useRouter } from "next/navigation"; +import { Search, Trophy, ListTree } from "lucide-react"; +import { clsx, type ClassValue } from "clsx"; +import { twMerge } from "tailwind-merge"; +import Link from "next/link"; +import zealtConfig from "@/zealt/config.json"; + +function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} + +export interface LeaderboardEntry { + id: string; + model: string; + rawModel: string; + agent: string; + passedEvals: number; + successRate: number; + avgLatency: number; + isNew?: boolean; +} + +function ProgressBar({ value, colorClass }: { value: number; colorClass: string }) { + return ( +
+
+
+ ); +} + +function ScoreCell({ value }: { value: number }) { + let colorClass = "bg-primary"; + let textClass = "text-muted-foreground"; + + if (value >= 90) { + colorClass = "bg-emerald-500"; + textClass = "text-emerald-500 font-bold"; + } else if (value >= 75) { + colorClass = "bg-blue-500"; + textClass = "text-blue-500 font-medium"; + } else if (value >= 60) { + colorClass = "bg-amber-500"; + textClass = "text-amber-500"; + } else { + colorClass = "bg-red-500"; + textClass = "text-red-500"; + } + + return ( +
+ {value}% + +
+ ); +} + +export default function LeaderboardTable({ data }: { data: LeaderboardEntry[] }) { + const router = useRouter(); + const [devMode, setDevMode] = useState(process.env.NODE_ENV === "development"); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevMode(isDev); + }, []); + + const filteredData = useMemo(() => { + let processedData = data; + const config = zealtConfig as { pending_models?: string[] }; + + if (!devMode && config.pending_models && config.pending_models.length > 0) { + processedData = processedData.filter((item) => + !config.pending_models!.includes(item.rawModel) + ); + } + + if (searchQuery) { + const query = searchQuery.toLowerCase(); + processedData = processedData.filter(item => + item.model.toLowerCase().includes(query) + ); + } + + return processedData; + }, [data, searchQuery, devMode]); + + return ( + <> + {/* Controls & Filters */} +
+

+ Model Performance +

+ +
+ + + View Tasks + + +
+ + setSearchQuery(e.target.value)} + className="pl-9 pr-4 py-2 bg-card border border-border rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary/20 w-full sm:w-64 transition-all" + /> +
+
+
+ + {/* Leaderboard Table */} +
+
+ + + + + + + + + + + {filteredData.length === 0 ? ( + + + + ) : ( + filteredData.map((row, index) => ( + router.push(`./tasks?model=${encodeURIComponent(row.model)}`)} + className="group hover:bg-secondary/30 transition-colors duration-200 cursor-pointer" + > + + + + + + )))} + +
ModelPassedAvg DurationSuccess Rate
+ No results found matching your search. +
+ #{index + 1} +
+ + {row.model} + {index === 0 && } + {row.isNew && ( + + NEW + + )} + +
+ 		+ {row.passedEvals} + + {row.avgLatency > 0 ? `${row.avgLatency.toFixed(1)}s` : '-'} + +
+ +
+
+
+
+ + ); +} diff --git a/site/app/(home)/page.tsx b/site/app/(home)/page.tsx new file mode 100644 index 0000000..d617123 --- /dev/null +++ b/site/app/(home)/page.tsx @@ -0,0 +1,176 @@ +import { Suspense } from "react"; +import { Github, Terminal, ClipboardList, ListTree } from "lucide-react"; +import Link from "next/link"; +import tasksData from "@/zealt/tasks.json"; +import pendingTasksData from "@/zealt/pending-tasks.json"; +import zealtConfig from "@/zealt/config.json"; +import PendingReviewCard from "@/components/pending-review-card"; +import LeaderboardTable, { type LeaderboardEntry } from "./components/leaderboard-table"; + +type TaskTrial = { + agent: string; + model: string; + passed: boolean; + latency_sec: number | null; +}; + +type TaskValue = { + trials?: TaskTrial[]; +}; + +type PendingTasksValue = { + 'pending-tasks'?: number; +}; + +export default function Home() { + const totalTasks = Object.keys(tasksData as Record).length; + const hasTasks = totalTasks > 0; + const pendingSampleCases = Math.max( + 0, + Number((pendingTasksData as PendingTasksValue)['pending-tasks'] ?? 0), + ); + + // Process tasks.json to compute leaderboard stats directly on the server + const statsMap = new Map(); + + Object.values(tasksData as Record).forEach((taskValue) => { + let trials: TaskTrial[] = []; + if (Array.isArray(taskValue)) { + trials = taskValue as TaskTrial[]; + } else if (typeof taskValue === "object" && taskValue !== null) { + const task = taskValue as TaskValue; + trials = Array.isArray(task.trials) ? task.trials : []; + } + + trials.forEach((trial) => { + // Simplify model name + const modelName = trial.model.split('/').pop() || trial.model; + const agentName = trial.agent.charAt(0).toUpperCase() + trial.agent.slice(1); + + const key = `${modelName}-${agentName}`; + + if (!statsMap.has(key)) { + statsMap.set(key, { + passed: 0, + total: 0, + totalLatency: 0, + latencyCount: 0, + model: modelName, + rawModel: trial.model, + agent: agentName + }); + } + + const stats = statsMap.get(key); + if (!stats) { + return; + } + stats.total += 1; + if (trial.passed) { + stats.passed += 1; + } + if (trial.latency_sec) { + stats.totalLatency += trial.latency_sec; + stats.latencyCount += 1; + } + }); + }); + + const data: LeaderboardEntry[] = Array.from(statsMap.values()) + .map((stats, index) => { + const successRate = stats.total > 0 ? Math.round((stats.passed / stats.total) * 100) : 0; + const avgLatency = stats.latencyCount > 0 ? stats.totalLatency / stats.latencyCount : 0; + return { + id: String(index + 1), + model: stats.model, + rawModel: stats.rawModel, + agent: stats.agent, + passedEvals: stats.passed, + successRate: successRate, + avgLatency: avgLatency, + }; + }) + .sort((a, b) => b.successRate - a.successRate); + + // Re-assign IDs based on sorted order and adjust isNew + data.forEach((item, index) => { + item.id = String(index + 1); + item.isNew = index === 0; // Keeping the original visual effect for the top item + }); + + return ( +
+ {/* Background Gradient Effect */} +
+ +
+ {/* Header Section */} +
+
+ + Live Benchmarks +
+ +

+ {zealtConfig.title} +

+ +

+ {zealtConfig.description} +

+ +
+ + + View on GitHub + + {data.length > 0 && ( + <> +
+ + + Total tasks: {totalTasks} + + + )} +
+ + + Last run: {new Date().toLocaleDateString()} + +
+
+ + {!hasTasks ? ( + + ) : data.length === 0 ? ( +
+

No evaluation data yet

+
+ + + View Tasks + +
+
+ ) : ( + // Client Component for Interactive Table + Loading leaderboard...
}> + + + )} +
+
+ ); +} diff --git a/site/app/globals.css b/site/app/globals.css new file mode 100644 index 0000000..437c041 --- /dev/null +++ b/site/app/globals.css @@ -0,0 +1,149 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); + --color-background: var(--background); + --color-foreground: var(--foreground); + --color-card: var(--card); + --color-card-foreground: var(--card-foreground); + --color-popover: var(--popover); + --color-popover-foreground: var(--popover-foreground); + --color-primary: var(--primary); + --color-primary-foreground: var(--primary-foreground); + --color-secondary: var(--secondary); + --color-secondary-foreground: var(--secondary-foreground); + --color-muted: var(--muted); + --color-muted-foreground: var(--muted-foreground); + --color-accent: var(--accent); + --color-accent-foreground: var(--accent-foreground); + --color-destructive: var(--destructive); + --color-border: var(--border); + --color-input: var(--input); + --color-ring: var(--ring); + --color-chart-1: var(--chart-1); + --color-chart-2: var(--chart-2); + --color-chart-3: var(--chart-3); + --color-chart-4: var(--chart-4); + --color-chart-5: var(--chart-5); + --color-sidebar: var(--sidebar); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.141 0.005 285.823); + --card: oklch(1 0 0); + --card-foreground: oklch(0.141 0.005 285.823); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.141 0.005 285.823); + --primary: oklch(0.21 0.006 285.885); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.967 0.001 286.375); + --secondary-foreground: oklch(0.21 0.006 285.885); + --muted: oklch(0.967 0.001 286.375); + --muted-foreground: oklch(0.552 0.016 285.938); + --accent: oklch(0.967 0.001 286.375); + --accent-foreground: oklch(0.21 0.006 285.885); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.92 0.004 286.32); + --input: oklch(0.92 0.004 286.32); + --ring: oklch(0.705 0.015 286.067); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.141 0.005 285.823); + --sidebar-primary: oklch(0.21 0.006 285.885); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.967 0.001 286.375); + --sidebar-accent-foreground: oklch(0.21 0.006 285.885); + --sidebar-border: oklch(0.92 0.004 286.32); + --sidebar-ring: oklch(0.705 0.015 286.067); +} + +.dark { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.21 0.006 285.885); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.92 0.004 286.32); + --primary-foreground: oklch(0.21 0.006 285.885); + --secondary: oklch(0.274 0.006 286.033); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --accent: oklch(0.274 0.006 286.033); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.552 0.016 285.938); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.21 0.006 285.885); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.274 0.006 286.033); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.552 0.016 285.938); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} + +@layer utilities { + .custom-scrollbar { + scrollbar-width: thin; + scrollbar-color: var(--border) transparent; + } + + .custom-scrollbar::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + .custom-scrollbar::-webkit-scrollbar-track { + background: transparent; + } + + .custom-scrollbar::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; + } + + .custom-scrollbar::-webkit-scrollbar-thumb:hover { + background: color-mix(in oklch, var(--foreground) 30%, transparent); + } +} \ No newline at end of file diff --git a/site/app/layout.tsx b/site/app/layout.tsx new file mode 100644 index 0000000..3ee8c0a --- /dev/null +++ b/site/app/layout.tsx @@ -0,0 +1,37 @@ +import "./globals.css"; + +import type { Metadata } from 'next'; +import { Inter } from "next/font/google"; +import zealtConfig from "@/zealt/config.json"; +import { ThemeProvider } from "@/components/theme-provider"; +import { ThemeToggle } from "@/components/theme-toggle"; +import { AppQueryProvider } from "@/components/query-provider"; + +const inter = Inter({ subsets: ["latin"] }); + +export const metadata: Metadata = { + title: zealtConfig.title, + description: zealtConfig.description, +}; + + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + +
+ +
+ {children} + + + + + ); +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx new file mode 100644 index 0000000..bf9202c --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx @@ -0,0 +1,265 @@ +"use client"; + +import { useMemo } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { ChevronRight, File, Folder } from "lucide-react"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + fetchLogText, + LogContentSkeleton, + LogErrorView, + getLogErrorMessage, +} from "./trajectory-page"; + +export type ArtifactNode = { + name: string; + type: "file" | "dir"; + path: string; + children?: ArtifactNode[]; +}; + +export type ArtifactNodeWithUrl = + | { name: string; type: "file"; path: string; url: string } + | { name: string; type: "dir"; path: string; children: ArtifactNodeWithUrl[] }; + +type ArtifactsPanelProps = { + artifactTree: ArtifactNodeWithUrl[]; +}; + +export function ArtifactsPanel({ artifactTree }: ArtifactsPanelProps) { + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + + const queryArtifact = searchParams.get("artifact"); + const queryPath = searchParams.get("path"); + + const activeArtifact = useMemo(() => { + if (queryArtifact) { + const found = artifactTree.find((node) => node.name === queryArtifact); + if (found) return found; + } + return artifactTree[0] ?? null; + }, [artifactTree, queryArtifact]); + + const updateParams = (next: { artifact?: string | null; path?: string | null }) => { + const params = new URLSearchParams(searchParams.toString()); + if (next.artifact === null) { + params.delete("artifact"); + } else if (next.artifact !== undefined) { + params.set("artifact", next.artifact); + } + if (next.path === null) { + params.delete("path"); + } else if (next.path !== undefined) { + params.set("path", next.path); + } + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + + const handleSelectArtifact = (name: string) => { + updateParams({ artifact: name, path: null }); + }; + + if (!activeArtifact) { + return ( +
+ No artifacts available. +
+ ); + } + + return ( +
+
+
+ {artifactTree.map((node) => ( + + ))} +
+
+ +
+ {activeArtifact.type === "file" ? ( + + ) : ( + updateParams({ path: path ?? null })} + /> + )} +
+
+ ); +} + +type DirectoryArtifactViewProps = { + artifact: Extract; + currentPath: string | null; + onPathChange: (path: string | null) => void; +}; + +function DirectoryArtifactView({ artifact, currentPath, onPathChange }: DirectoryArtifactViewProps) { + // Resolve the active path. If currentPath isn't valid for this artifact, fall back to the artifact root. + const resolved = useMemo(() => resolvePath(artifact, currentPath), [artifact, currentPath]); + const { folder, selectedFile, breadcrumbDirs } = resolved; + + return ( +
+
+ {breadcrumbDirs.map((dir, idx) => { + const isLast = idx === breadcrumbDirs.length - 1 && !selectedFile; + return ( + + + {(idx < breadcrumbDirs.length - 1 || selectedFile) && ( + + )} + + ); + })} + {selectedFile && ( + {selectedFile.name} + )} +
+ +
+
+ {folder.children.length === 0 && ( + Empty directory. + )} + {folder.children.map((child) => { + const isActive = child.type === "file" && selectedFile?.path === child.path; + return ( + + ); + })} +
+
+ +
+ {selectedFile ? ( + + ) : ( +
+ Select a file to view its contents. +
+ )} +
+
+ ); +} + +function FileViewer({ url }: { url: string }) { + const query = useQuery({ + queryKey: ["artifact-file", url], + queryFn: () => fetchLogText(url), + }); + + return ( + +
+ {query.isPending || query.isFetching ? ( + + ) : query.isError ? ( + void query.refetch()} + /> + ) : query.data ? ( + 
+            {query.data}
+
+ ) : ( +

Empty file.

+ )} +
+ + ); +} + +type ResolvedPath = { + folder: Extract; + selectedFile: Extract | null; + breadcrumbDirs: { name: string; path: string }[]; +}; + +function resolvePath( + root: Extract, + targetPath: string | null, +): ResolvedPath { + const breadcrumbDirs: { name: string; path: string }[] = [{ name: root.name, path: root.path }]; + let folder = root; + let selectedFile: Extract | null = null; + + if (!targetPath || !targetPath.startsWith(`${root.path}/`)) { + return { folder, selectedFile, breadcrumbDirs }; + } + + const remainder = targetPath.slice(root.path.length + 1); + const segments = remainder.split("/").filter(Boolean); + + let currentPath = root.path; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]; + currentPath = `${currentPath}/${segment}`; + const child = folder.children.find((c) => c.name === segment); + if (!child) { + // Path doesn't exist; bail and show what we resolved so far. + return { folder, selectedFile: null, breadcrumbDirs }; + } + if (child.type === "dir") { + folder = child; + breadcrumbDirs.push({ name: child.name, path: child.path }); + } else { + // File at the end of the path. + selectedFile = child; + // Don't push the file as a breadcrumb dir; the caller renders it separately. + break; + } + } + + return { folder, selectedFile, breadcrumbDirs }; +} diff --git a/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx new file mode 100644 index 0000000..0fab8fc --- /dev/null +++ b/site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx @@ -0,0 +1,404 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useTheme } from "next-themes"; +import { useSearchParams, useRouter, usePathname } from "next/navigation"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Button } from "@/components/ui/button"; +import { HttpError } from "@/lib/http-error"; +import { ArtifactsPanel, type ArtifactNodeWithUrl } from "./artifacts-panel"; + +export type TabConfig = { + value: string; + label: React.ReactNode; +}; + +type TrajectoryPageProps = { + trajectoryUrl: string; + browserVerificationUrls: { name: string; url: string }[]; + fallbackUrl: string; + stderrLogUrl: string | null; + verifierLogUrl: string | null; + tabsConfig: TabConfig[]; + artifactTree?: ArtifactNodeWithUrl[]; +}; + +export async function fetchLogText(url: string): Promise { + let response: Response; + try { + response = await fetch(url, { cache: "force-cache" }); + } catch (_e) { + throw new HttpError("Network request failed."); + } + + if (!response.ok) { + throw new HttpError(`Request failed with status ${response.status}.`, { status: response.status }); + } + + return response.text(); +} + +export function TrajectoryPage({ + trajectoryUrl, + browserVerificationUrls, + fallbackUrl, + stderrLogUrl, + verifierLogUrl, + tabsConfig, + artifactTree, +}: TrajectoryPageProps) { + const { resolvedTheme } = useTheme(); + const searchParams = useSearchParams(); + const router = useRouter(); + const pathname = usePathname(); + const [mounted, setMounted] = useState(false); + const [iframeLoading, setIframeLoading] = useState(true); + const [devModeEnabled, setDevModeEnabled] = useState(process.env.NODE_ENV === "development"); + + useEffect(() => { + const isDev = process.env.NODE_ENV === "development" || + localStorage.getItem("devMode") === "true"; + setDevModeEnabled(isDev); + }, []); + + const visibleTabsConfig = useMemo( + () => + devModeEnabled + ? tabsConfig + : tabsConfig.filter((t) => t.value !== "artifacts"), + [tabsConfig, devModeEnabled], + ); + const validTabs = visibleTabsConfig.map((t) => t.value); + const [activeTab, setActiveTab] = useState(() => { + const queryTab = searchParams.get("tab"); + return queryTab && validTabs.includes(queryTab) ? queryTab : validTabs[0]; + }); + + const handleTabChange = (value: string) => { + setActiveTab(value); + const params = new URLSearchParams(searchParams.toString()); + params.set("tab", value); + router.replace(`${pathname}?${params.toString()}`, { scroll: false }); + }; + const [browserIframeLoading, setBrowserIframeLoading] = useState>({}); + const [activeBrowserVerificationTab, setActiveBrowserVerificationTab] = useState( + browserVerificationUrls[0]?.name || "", + ); + + const iframeTheme = mounted && resolvedTheme === "light" ? "light" : "dark"; + + const iframeUrl = useMemo(() => { + const url = new URL(trajectoryUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [trajectoryUrl, iframeTheme]); + + const activeBrowserVerificationBaseUrl = useMemo(() => { + const testCase = browserVerificationUrls.find((tc) => tc.name === activeBrowserVerificationTab); + return testCase ? testCase.url : null; + }, [browserVerificationUrls, activeBrowserVerificationTab]); + + const activeBrowserVerificationUrl = useMemo(() => { + if (!activeBrowserVerificationBaseUrl) { + return null; + } + + const url = new URL(activeBrowserVerificationBaseUrl); + const hashParams = new URLSearchParams(url.hash.slice(1)); + hashParams.set("theme", iframeTheme); + url.hash = hashParams.toString(); + return url.toString(); + }, [activeBrowserVerificationBaseUrl, iframeTheme]); + + useEffect(() => { + setMounted(true); + }, []); + + useEffect(() => { + if (!mounted || !activeBrowserVerificationBaseUrl) { + return; + } + + setBrowserIframeLoading((prev) => ({ ...prev, [activeBrowserVerificationTab]: true })); + }, [activeBrowserVerificationBaseUrl, activeBrowserVerificationTab, mounted]); + + const stderrQuery = useQuery({ + queryKey: ["trajectory-stderr", stderrLogUrl], + enabled: Boolean(stderrLogUrl), + queryFn: async () => { + if (!stderrLogUrl) { + return null; + } + + return fetchLogText(stderrLogUrl); + }, + }); + + const verifierQuery = useQuery({ + queryKey: ["trajectory-verifier", verifierLogUrl], + enabled: Boolean(verifierLogUrl), + queryFn: async () => { + if (!verifierLogUrl) { + return null; + } + + return fetchLogText(verifierLogUrl); + }, + }); + + const handleIframeLoad = () => { + setIframeLoading(false); + }; + + const handleBrowserIframeLoad = (testCaseName: string) => { + setBrowserIframeLoading((prev) => ({ ...prev, [testCaseName]: false })); + }; + + const handleIframeError = () => { + window.location.replace(fallbackUrl); + }; + + const renderLogContent = ( + text: string | null | undefined, + isLoading: boolean, + isError: boolean, + error: unknown, + onRetry: () => void, + emptyMessage: string, + ) => { + if (isLoading) { + return ; + } + + if (isError) { + return ; + } + + if (!text) { + return

{emptyMessage}

; + } + + return ( + 
+        {text}
+
+ ); + }; + + return ( +
+
+ +
+ + {visibleTabsConfig.map((tab) => ( + + {tab.label} + + ))} + +
+ + +
+ +
+ {mounted && ( +