From b26961e144b888bdb4441e356b178e02563a76e5 Mon Sep 17 00:00:00 2001 From: "zealt-staging[bot]" <264479255+zealt-staging[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 04:19:27 +0000 Subject: [PATCH] Initialize repository with benchmark template and tasks --- .github/workflows/deploy.yml | 69 ++ .gitignore | 49 + .zealt/config.json | 5 + README.md | 47 +- plan.md | 136 +++ .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../instruction.md | 8 + .../type_safe_navigation_menu/instruction.md | 7 + .../(home)/components/leaderboard-table.tsx | 183 +++ site/app/(home)/page.tsx | 176 +++ site/app/globals.css | 149 +++ site/app/layout.tsx | 37 + .../trajectory/components/artifacts-panel.tsx | 265 ++++ .../trajectory/components/trajectory-page.tsx | 404 ++++++ .../tasks/[name]/[jobId]/trajectory/page.tsx | 414 +++++++ site/app/tasks/components/back-to-top.tsx | 41 + site/app/tasks/components/multi-select.tsx | 104 ++ .../tasks/components/tasks-page-client.tsx | 724 +++++++++++ site/app/tasks/page.tsx | 142 +++ site/bun.lock | 1082 +++++++++++++++++ site/components.json | 23 + site/components/pending-review-card.tsx | 44 + site/components/query-provider.tsx | 21 + site/components/theme-provider.tsx | 27 + site/components/theme-toggle.tsx | 32 + site/components/ui/badge.tsx | 48 + site/components/ui/button.tsx | 64 + site/components/ui/checkbox.tsx | 32 + site/components/ui/command.tsx | 184 +++ site/components/ui/dialog.tsx | 158 +++ site/components/ui/drawer.tsx | 131 ++ site/components/ui/hover-card.tsx | 44 + site/components/ui/popover.tsx | 89 ++ site/components/ui/scroll-area.tsx | 59 + site/components/ui/select.tsx | 190 +++ site/components/ui/sheet.tsx | 153 +++ site/components/ui/skeleton.tsx | 16 + site/components/ui/tabs.tsx | 69 ++ site/lib/http-error.ts | 8 + site/lib/utils.ts | 6 + site/next.config.ts | 17 + site/package.json | 34 + site/postcss.config.mjs | 7 + site/scripts/check-trajectory.ts | 118 ++ site/scripts/compute-tasks.ts | 252 ++++ site/tsconfig.json | 41 + site/types/result.d.ts | 30 + site/zealt | 1 + 51 files changed, 5971 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 .zealt/config.json create mode 100644 plan.md create mode 100644 scratchpad/pending-tasks/headless_table_initialization_and_typing/instruction.md create mode 100644 scratchpad/pending-tasks/optimistic_updates_with_tanstack_query/instruction.md create mode 100644 scratchpad/pending-tasks/resolving_query_ssr_hydration_mismatch/instruction.md create mode 100644 scratchpad/pending-tasks/server_function_creation_in_tanstack_start/instruction.md create mode 100644 scratchpad/pending-tasks/synchronized_url_search_parameters/instruction.md create mode 100644 scratchpad/pending-tasks/type_safe_navigation_menu/instruction.md create mode 100644 site/app/(home)/components/leaderboard-table.tsx create mode 100644 site/app/(home)/page.tsx create mode 100644 site/app/globals.css create mode 100644 site/app/layout.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/artifacts-panel.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/components/trajectory-page.tsx create mode 100644 site/app/tasks/[name]/[jobId]/trajectory/page.tsx create mode 100644 site/app/tasks/components/back-to-top.tsx create mode 100644 site/app/tasks/components/multi-select.tsx create mode 100644 site/app/tasks/components/tasks-page-client.tsx create mode 100644 site/app/tasks/page.tsx create mode 100644 site/bun.lock create mode 100644 site/components.json create mode 100644 site/components/pending-review-card.tsx create mode 100644 site/components/query-provider.tsx create mode 100644 site/components/theme-provider.tsx create mode 100644 site/components/theme-toggle.tsx create mode 100644 site/components/ui/badge.tsx create mode 100644 site/components/ui/button.tsx create mode 100644 site/components/ui/checkbox.tsx create mode 100644 site/components/ui/command.tsx create mode 100644 site/components/ui/dialog.tsx create mode 100644 site/components/ui/drawer.tsx create mode 100644 site/components/ui/hover-card.tsx create mode 100644 site/components/ui/popover.tsx create mode 100644 site/components/ui/scroll-area.tsx create mode 100644 site/components/ui/select.tsx create mode 100644 site/components/ui/sheet.tsx create mode 100644 site/components/ui/skeleton.tsx create mode 100644 site/components/ui/tabs.tsx create mode 100644 site/lib/http-error.ts create mode 100644 site/lib/utils.ts create mode 100644 site/next.config.ts create mode 100644 site/package.json create mode 100644 site/postcss.config.mjs create mode 100644 site/scripts/check-trajectory.ts create mode 100644 site/scripts/compute-tasks.ts create mode 100644 site/tsconfig.json create mode 100644 site/types/result.d.ts create mode 120000 site/zealt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..828b276 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,69 @@ +name: Deploy Next.js site to Pages + +on: + push: + branches: ["main"] + pull_request: + types: [opened, reopened, synchronize, closed] + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install --frozen-lockfile + working-directory: site + + - name: Setup Pages + id: setup_pages + uses: actions/configure-pages@v5 + + - name: Set base uri + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}/pr-preview/pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" + else + echo "NEXT_PUBLIC_BASE_PATH=${{ steps.setup_pages.outputs.base_path }}" >> "$GITHUB_ENV" + fi + + - name: Build with Next.js + run: bun run build + working-directory: site + + - name: Deploy preview + if: github.event_name == 'pull_request' + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site/out + + - name: Deploy production + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + clean-exclude: pr-preview/ + force: false + folder: site/out diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0cf2fa4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +__pycache__/ +*.py[cod] +*$py.class + +# testing +coverage + +# next.js +.next/ + +# The `out` directory should not be ignored by version control +out/ + +# production +build + +# misc +.DS_Store +*.pem +*~ +\#* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.zealt/config.json b/.zealt/config.json new file mode 100644 index 0000000..dd30f00 --- /dev/null +++ b/.zealt/config.json @@ -0,0 +1,5 @@ +{ + "title": "TanStack Benchmark", + "description": "Performance results of AI coding models on TanStack tasks, measuring success rate and execution time with high precision.", + "github_repo": "https://github.com/kweizh/tanstack-benchmark" +} \ No newline at end of file diff --git a/README.md b/README.md index 1b872c5..dc6b88c 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# tanstack-benchmark \ No newline at end of file + +# TanStack Benchmark + +This repository contains benchmarks for evaluating AI models on **TanStack**. + +You can view the evaluation reports at [https://kweizh.github.io/tanstack-benchmark/](https://kweizh.github.io/tanstack-benchmark/). + +## Project Structure + +- `tasks/`: Contains the benchmark tasks, each with its own instructions. +- `jobs/`: Stores the results of benchmark runs. +- `site/`: A Next.js application to visualize benchmark results. + +## Getting Started + +This benchmark is evaluated using the [Harbor framework](https://github.com/harbor-framework/harbor) and the [Pochi agent](https://github.com/TabbyML/pochi). + +### Running Evaluation + +You can run the evaluation using the Harbor CLI. Here is an example: + +```bash +harbor run \ + --agent codex \ + --model "gpt-5.2-codex" \ + --env daytona \ + --path ./tasks \ + --n-attempts 1 \ + --max-retries 5 \ + --n-concurrent 5 \ + --retry-include RuntimeError \ + --retry-include DaytonaError \ + --retry-include AgentTimeoutError +``` + +### Evaluation Details + +Before starting the evaluation, you should set the necessary environment variables for your chosen agent. +For example, if using Pochi, you should export `POCHI_API_KEY`. + +Evaluation can be run locally with Docker (default), or using [Daytona.io](https://www.daytona.io/) by setting `--env daytona`. + +When running with Daytona, please note that Daytona blocks some network access for tier 1 and tier 2 users. If you meet any network issues, please refer to [Daytona network limits](https://www.daytona.io/docs/en/network-limits/). + +--- +Generated by [Zealt](https://github.com/TabbyML/zealt) diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..2501f3f --- /dev/null +++ b/plan.md @@ -0,0 +1,136 @@ +# TanStack Benchmark Research Report + +## 1. Library Overview + +**Description**: TanStack is a collection of high-quality, open-source headless libraries for web development. It focuses on the "hard parts" of application development: state management, routing, data grids, and forms. The ecosystem's flagship is **TanStack Start**, a full-stack React framework that integrates these libraries into a cohesive, type-safe development experience. + +**Ecosystem Role**: TanStack provides the foundational "engine" for modern web apps. Unlike opinionated UI kits, TanStack libraries are "headless," providing logic and state without markup, allowing developers to use any UI library (e.g., Tailwind, Shadcn UI) while maintaining strict type safety from the database to the browser. + +**Project Setup**: +The recommended way to initialize a full-stack project is via the TanStack CLI: +```bash +npx @tanstack/cli@latest create +``` +Standard project structure for TanStack Start: +- `app/routes/`: File-based routing directory. +- `app/routeTree.gen.ts`: Automatically generated type-safe route tree. +- `app/ssr.tsx` & `app/client.tsx`: Entry points for server and client. +- `app/router.tsx`: Shared router configuration. + +--- + +## 2. Core Primitives & APIs + +### TanStack Query (Server State) +- **Concept**: Manages asynchronous state (fetching, caching, synchronization). +- **Core APIs**: `useQuery`, `useMutation`, `queryOptions`. +- **Code Snippet**: +```typescript +const postsQuery = queryOptions({ + queryKey: ['posts'], + queryFn: () => fetch('/api/posts').then(r => r.json()), +}) + +function Posts() { + const { data } = useQuery(postsQuery) + return