diff --git a/.factory-plugin/marketplace.json b/.factory-plugin/marketplace.json index e50836e..5cf8fa7 100644 --- a/.factory-plugin/marketplace.json +++ b/.factory-plugin/marketplace.json @@ -1,8 +1,8 @@ { "name": "pvstack", - "description": "PV Stack: Lauren Tan's pstack for Droid, with model routing chosen from VulcanBench data.", + "description": "pvstack: Lauren Tan's pstack for Droid, with model routing chosen from VulcanBench data.", "owner": { - "name": "PV Stack contributors" + "name": "pvstack contributors" }, "plugins": [ { diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index 774c82a..f65bcf6 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -72,12 +72,12 @@ jobs: body="$RUNNER_TEMP/body.md" { - echo "Syncs PV Stack to upstream pstack ${version} (\`${sha}\`)." + echo "Syncs pvstack to upstream pstack ${version} (\`${sha}\`)." echo if [ "$status" -eq 0 ]; then echo "\`npm run check\` passes." else - echo "\`npm run check\` fails. Upstream likely added a role line, a model slug, or a Cursor reference that the PV Stack layer does not cover yet. Push a fix to \`${BRANCH}\` before merging; the workflow won't overwrite manual commits." + echo "\`npm run check\` fails. Upstream likely added a role line, a model slug, or a Cursor reference that the pvstack layer does not cover yet. Push a fix to \`${BRANCH}\` before merging; the workflow won't overwrite manual commits." fi echo echo '```text' diff --git a/AGENTS.md b/AGENTS.md index 83f2ae2..2b180f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# PV Stack maintenance +# pvstack maintenance - Don't hand-edit files listed in `plugins/pvstack/.upstream-files.json`. Change `tools/sync-upstream.mjs` (its `REWRITES` or the build steps) and re-run `npm run sync`, or send the change upstream to `cursor/plugins`. - `plugins/pvstack/droids/pv-*.md` are generated. Edit `CELLS` in `tools/droids.mjs`, then run `npm run droids`. diff --git a/LICENSE b/LICENSE index f63c1d5..dc4215a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 PV Stack contributors +Copyright (c) 2026 pvstack contributors Portions copyright (c) 2026 Lauren Tan (see plugins/pvstack/LICENSE-pstack) Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/NOTICE.md b/NOTICE.md index d0a7bab..bb7f3c0 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,12 +1,12 @@ # Notice -PV Stack includes and modifies pstack by Lauren Tan ([@poteto](https://github.com/poteto)), from https://github.com/cursor/plugins/tree/main/pstack, under the MIT License. The original license is in `plugins/pvstack/LICENSE-pstack`. +pvstack includes and modifies pstack by Lauren Tan ([@poteto](https://github.com/poteto)), from https://github.com/cursor/plugins/tree/main/pstack, under the MIT License. The original license is in `plugins/pvstack/LICENSE-pstack`. ## Files from pstack These files are listed in `plugins/pvstack/.upstream-files.json`: -- `plugins/pvstack/skills/**` (except the PV Stack files below) +- `plugins/pvstack/skills/**` (except the pvstack files below) - `plugins/pvstack/droids/poteto-agent.md`, `plugins/pvstack/droids/comment-sicko.md` - `plugins/pvstack/docs/upstream/**` @@ -17,7 +17,7 @@ These files are listed in `plugins/pvstack/.upstream-files.json`: - Cursor agents are converted to Droid droids: the name is normalized, `model: inherit` is added, and Cursor-only keys are removed. - `setup-pstack` and the Benny automation pack are not included. -## Files written for PV Stack +## Files written for pvstack - `plugins/pvstack/droids/pv-*.md` - `plugins/pvstack/skills/setup-pvstack/**` diff --git a/README.md b/README.md index b6993d1..60a3026 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,70 @@ -# PV Stack +# pvstack -PV Stack is [Lauren Tan's pstack](https://github.com/cursor/plugins/tree/main/pstack) for [Droid](https://docs.factory.ai), with every role's model chosen from [VulcanBench](https://vulcanbench.com) data instead of one lab's defaults. +pvstack is [Lauren Tan's pstack](https://github.com/cursor/plugins/tree/main/pstack) for [Droid](https://docs.factory.ai), with every role's model chosen from [VulcanBench](https://vulcanbench.com) data instead of one lab's defaults. -pstack is a set of rigorous engineering workflows. You start a task with `/poteto-mode`. It picks a playbook (bug fix, feature, investigation, perf, babysit, and others) and calls `how`, `why`, `architect`, `interrogate`, `arena` and the principle skills as each step needs them. PV Stack keeps those skills close to upstream and changes two things: +pstack is a set of rigorous engineering workflows. You start a task with `/poteto-mode`. It picks a playbook (bug fix, feature, investigation, perf, babysit, and others) and calls `how`, `why`, `architect`, `interrogate`, `arena` and the principle skills as each step needs them. pvstack keeps those skills close to upstream and changes two things: 1. **It runs on Droid.** One mapping file translates Cursor's tools and Task parameters to Droid's. -2. **It routes each role to the model that VulcanBench shows is the best value for that kind of work.** You can choose from two modes. +2. **It routes each role to the model that VulcanBench shows is the best value for that kind of work.** You can choose from six modes, or write a custom preset. ## Playbook -New to pstack? [The PV Stack Playbook](playbook/index.html) walks you through it, with one path for a new project (greenfield) and one for an existing codebase (brownfield). It works with any agent that can load skills. Agents can read [playbook/llms.txt](playbook/llms.txt) and [playbook/playbook.md](playbook/playbook.md) to guide you through it. To change it, edit `playbook/src/content.mjs` and run `npm run playbook`. +New to pstack? [The pvstack Playbook](playbook/index.html) walks you through it, with one path for a new project (greenfield) and one for an existing codebase (brownfield). It works with any agent that can load skills. Agents can read [playbook/llms.txt](playbook/llms.txt) and [playbook/playbook.md](playbook/playbook.md) to guide you through it. To change it, edit `playbook/src/content.mjs` and run `npm run playbook`. ## Modes -| Role | Balanced (default) | Budget | -| --- | --- | --- | -| Code delegates (feature, refactor, bug fix, perf, hillclimb) | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, max | -| Swarm workers | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, high | -| Exploration, investigators, mechanical edits | GPT-6.1 Sol, low | DeepSeek V4.1 Flash, low | -| Judgment, prose, explainers, synthesizers | Claude Opus 5.5, medium | Claude Opus 5.5, medium | -| Hardest changes | Grok 4.7, xhigh | Claude Opus 5.5, medium | -| Reflect tooling | GPT-6.1 Sol, xhigh | GPT-6.1 Sol, high | -| Review panels (arena, architect, interrogate) | Opus 5.5 medium · Sol xhigh · Grok 4.7 high | Sol high · DeepSeek max · Grok 4.7 high | +Each mode is a preset: a rule per role class, resolved over the cells in `tools/droids.mjs` and their VulcanBench rows. The table below is generated from `tools/presets.mjs`. After `npm run vulcanbench`, run `npm run presets`, `npm run evidence` and `npm run playbook` so the sheets, `docs/model-evidence.md` and the playbook match the new snapshot. + + +| Role | Balanced (default) | Budget | Quality | Fast | Safe | Open | +| --- | --- | --- | --- | --- | --- | --- | +| Code delegates (feature, refactor, bug fix, perf, hillclimb) | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, high | Grok 4.7, xhigh | GPT-6.1 Sol, high | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, high | +| Swarm workers | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, high | Grok 4.7, xhigh | GPT-6.1 Sol, high | GPT-6.1 Sol, high | DeepSeek V4.1 Flash, high | +| Exploration, investigators, mechanical edits | GPT-6.1 Sol, low | DeepSeek V4.1 Flash, low | GPT-6.1 Sol, high | GPT-6.1 Sol, low | GPT-6.1 Sol, low | DeepSeek V4.1 Flash, low | +| Judgment, prose, explainers, synthesizers | Claude Opus 5.5, medium | Claude Opus 5.5, medium | Claude Opus 5.5, high | Claude Opus 5.5, medium | Claude Opus 5.5, medium | DeepSeek V4.1 Flash, high | +| Hardest changes | Grok 4.7, xhigh | Claude Opus 5.5, medium | Grok 4.7, xhigh | GPT-6.1 Sol, high | Claude Opus 5.5, high | DeepSeek V4.1 Flash, high | +| Reflect tooling | GPT-6.1 Sol, xhigh | GPT-6.1 Sol, high | Grok 4.7, xhigh | GPT-6.1 Sol, high | GPT-6.1 Sol, xhigh | DeepSeek V4.1 Flash, high | +| Review panels | Claude Opus 5.5, medium · GPT-6.1 Sol, xhigh · Grok 4.7, high | GPT-6.1 Sol, high · DeepSeek V4.1 Flash, low · Grok 4.7, high (cross-judge pool: Claude Opus 5.5, medium · GPT-6.1 Sol, high · Grok 4.7, high) | Claude Opus 5.5, high · GPT-6.1 Sol, xhigh · Grok 4.7, xhigh | Claude Opus 5.5, medium · GPT-6.1 Sol, high · Grok 4.7, high | Claude Opus 5.5, medium · GPT-6.1 Sol, xhigh · Grok 4.7, high | DeepSeek V4.1 Flash, max · DeepSeek V4.1 Flash, high · DeepSeek V4.1 Flash, low | + The reasons and numbers behind each choice are in [docs/model-evidence.md](docs/model-evidence.md). In short: - GPT-6.1 Sol at high effort passes every Frontier v4 task at 97% of Opus 5.5's best score, for about 1/10 of the cost. - Opus 5.5 at medium effort is within 0.25 points of its best, and it scores higher than its own xhigh and max levels. - Grok 4.7 has the top three scores on Frontier v4. At xhigh it scores 93.15 and passes every task, so Balanced gives it the hardest changes. Against Sol at high, Grok at high takes about 2.5 times as long and at least 7 times the estimated cost. At xhigh it takes about 2.8 times as long and at least 9 times the cost. So Sol keeps everyday code. -- DeepSeek V4-Flash was one of the strongest and cheapest cells on VulcanBench's v3 board. +- DeepSeek V4-Flash was one of the strongest and cheapest cells on VulcanBench's v3 board. pvstack's own Droid run of VulcanBench's Python Suite v1 (345 runs) found DeepSeek V4.1 Flash at high effort resolving as many tasks as GPT-6.1 Sol at low on about half the credits. Max effort cost twice as much as high and resolved no more, so Budget runs code on high. See [Measured in Droid](docs/model-evidence.md#measured-in-droid). +- Quality ignores cost and takes the top score on the board. Fast takes the quickest cell that still passes all but one task. +- Safe keeps Grok 4.7 off judgment, code and the hardest changes, because Safety v1 shows it followed planted notes it never reported. Open stays on open-weights models, which today means DeepSeek V4.1 Flash on every role. + +## Custom presets + +A custom preset is a JSON file that extends one of the built-in modes. Put it at `~/.factory/pvstack-presets/.json`, or at `.factory/pvstack-presets/.json` in a project. `/setup-pvstack` lists the custom presets it finds and can write a new one with you. + +`plugins/pvstack/skills/setup-pvstack/examples/cheap-safe.json` extends Budget and asks for the cheapest cell that passes the Safety v1 screen and every task it ran: + +```json +{ + "summary": "Budget costs with a Safety v1 screen on code delegates.", + "extends": "budget", + "rules": { + "code": { "where": { "safe": true, "passed": "all" }, "pick": "min-usd" } + }, + "pins": [] +} +``` + +`rules` is keyed by role class (`code`, `explore`, `judgment`, `hardest`, `reflect tooling`, `panels`) or by a single role. `pick` is `min-usd`, `max-score` or `min-minutes`, and `where` filters the cells that pick sorts. `pins` overrides one role with a named droid (a `pv-*` droid, a personal droid, or `inherit`), and every pin needs a `reason`. The name must not collide with a built-in one. + +The resolver that ships in the plugin validates and explains presets without writing anything. It reads the plugin's own copy of the cell data, so it works from the installed plugin alone: + +```bash +cd ~/.factory/plugins/cache/pvstack-*/skills/setup-pvstack +node scripts/resolve.mjs --list +node scripts/resolve.mjs --explain cheap-safe +node scripts/resolve.mjs cheap-safe +``` + +`/setup-pvstack` writes the sheet that resolves. If a custom preset file later disappears, skills fall back to its `extends` base. ## Install @@ -44,7 +80,7 @@ Then, in a Droid session: /poteto-mode this pr has a subtle bug where the scroll drifts every 750ms even when idle. repro first, then fix and verify. ``` -`/setup-pvstack` writes `~/.factory/pvstack-models.md`. Without it, PV Stack uses Balanced. +`/setup-pvstack` writes `~/.factory/pvstack-models.md`. Without it, pvstack uses Balanced. To pick up new commits, run `droid plugin update pvstack@pvstack --scope user`. Droid runs the installed copy from its plugin cache, not from a local clone. @@ -63,7 +99,7 @@ flowchart LR ## Staying current with upstream -PV Stack follows `cursor/plugins/pstack` directly. The pinned commit is in [UPSTREAM.md](UPSTREAM.md). +pvstack follows `cursor/plugins/pstack` directly. The pinned commit is in [UPSTREAM.md](UPSTREAM.md). ```bash node tools/sync-upstream.mjs --ref main # pull Lauren's latest, re-pin @@ -72,7 +108,7 @@ npm run check # sync drift, droids, sheets, links A weekly GitHub Action does this for you. When upstream pstack changes, it pushes the sync to the `upstream-sync` branch and opens a pull request with the diff summary and the `npm run check` result. Later upstream changes refresh the same pull request, unless you have pushed your own fixes to the branch; then it only comments. If `main` catches up another way, it closes the pull request. Run it on demand from the Actions tab (workflow `upstream sync`). It needs the repository setting "Allow GitHub Actions to create and approve pull requests". -The sync rewrites only the files it copied from upstream, which are listed in `plugins/pvstack/.upstream-files.json`. The PV Stack layer is never overwritten: +The sync rewrites only the files it copied from upstream, which are listed in `plugins/pvstack/.upstream-files.json`. The pvstack layer is never overwritten: - `plugins/pvstack/droids/pv-*.md`, generated from `tools/droids.mjs` - `plugins/pvstack/skills/setup-pvstack/` @@ -85,31 +121,32 @@ The sync rewrites only the files it copied from upstream, which are listed in `p plugins/pvstack/ .factory-plugin/plugin.json skills/ upstream skills + setup-pvstack + droid-tools.md - droids/ poteto-agent, comment-sicko (upstream) + pv-* (PV Stack) + droids/ poteto-agent, comment-sicko (upstream) + pv-* (pvstack) docs/upstream/ upstream's pstack guide docs/model-evidence.md -tools/ sync-upstream.mjs, droids.mjs, validate.mjs +data/ vulcanbench.json (the board snapshot), catalog.mjs (hand-curated evidence) +tools/ sync-upstream.mjs, droids.mjs, validate.mjs, vulcanbench.mjs, presets.mjs, evidence.mjs .github/workflows/ check.yml (npm run check), upstream-sync.yml (weekly) ``` ## Credits -PV Stack is built on the work of two people. Neither of them is affiliated with this project. +pvstack is built on the work of two people. Neither of them is affiliated with this project. ### pstack, by Lauren Tan -Every workflow in PV Stack comes from Lauren Tan's pstack: `poteto-mode`, its playbooks, `how`, `why`, `architect`, `arena`, `interrogate`, and the principle skills. Lauren works at Cursor, previously worked at Meta and Netflix, and is on the React core team, where she helps build React Compiler. +Every workflow in pvstack comes from Lauren Tan's pstack: `poteto-mode`, its playbooks, `how`, `why`, `architect`, `arena`, `interrogate`, and the principle skills. Lauren works at Cursor, previously worked at Meta and Netflix, and is on the React core team, where she helps build React Compiler. - pstack source: https://github.com/cursor/plugins/tree/main/pstack - The pstack guide: https://github.com/cursor/plugins/blob/main/pstack/docs/guide/README.md - The Complete Guide to pstack: https://x.com/poteto/article/2094457600259842065 - GitHub: [@poteto](https://github.com/poteto) · X: [@poteto](https://x.com/poteto) -pstack is MIT-licensed, © 2026 Lauren Tan. Her license is in [`plugins/pvstack/LICENSE-pstack`](plugins/pvstack/LICENSE-pstack), and [NOTICE.md](NOTICE.md) lists which files come from pstack and what PV Stack changes in them. +pstack is MIT-licensed, © 2026 Lauren Tan. Her license is in [`plugins/pvstack/LICENSE-pstack`](plugins/pvstack/LICENSE-pstack), and [NOTICE.md](NOTICE.md) lists which files come from pstack and what pvstack changes in them. ### VulcanBench, by Morgan Linton -Every model and effort choice in PV Stack comes from VulcanBench, an open-source benchmark for real engineering tasks. It reports token use, time and cost alongside scores. Morgan Linton, cofounder and CTO of Bold Metrics, built it and runs it. PV Stack cites VulcanBench's published numbers in [docs/model-evidence.md](docs/model-evidence.md) and doesn't redistribute its data. +Every model and effort choice in pvstack comes from VulcanBench, an open-source benchmark for real engineering tasks. It reports token use, time and cost alongside scores. Morgan Linton, cofounder and CTO of Bold Metrics, built it and runs it. pvstack cites VulcanBench's published numbers in [docs/model-evidence.md](docs/model-evidence.md) and doesn't redistribute its data. - VulcanBench: https://vulcanbench.com - Leaderboard: https://vulcanbench.com/leaderboard.html @@ -117,4 +154,4 @@ Every model and effort choice in PV Stack comes from VulcanBench, an open-source - Open-source harness: https://github.com/morganlinton/VulcanBench - Morgan Linton: https://www.morganlinton.com · GitHub: [@morganlinton](https://github.com/morganlinton) -VulcanBench is free and runs on sponsorships. If PV Stack's routing saves you money, consider [sponsoring VulcanBench on GitHub](https://github.com/sponsors/morganlinton). +VulcanBench is free and runs on sponsorships. If pvstack's routing saves you money, consider [sponsoring VulcanBench on GitHub](https://github.com/sponsors/morganlinton). diff --git a/UPSTREAM.md b/UPSTREAM.md index ba0beb7..2001e16 100644 --- a/UPSTREAM.md +++ b/UPSTREAM.md @@ -1,6 +1,6 @@ # Upstream pin -PV Stack tracks Lauren Tan's pstack directly from Cursor's plugin repository. +pvstack tracks Lauren Tan's pstack directly from Cursor's plugin repository. - Repository: https://github.com/cursor/plugins - Path: `pstack/` diff --git a/data/catalog.mjs b/data/catalog.mjs new file mode 100644 index 0000000..2678228 --- /dev/null +++ b/data/catalog.mjs @@ -0,0 +1,75 @@ +// Hand-curated VulcanBench evidence the two board CSVs do not carry. Every +// number in docs/model-evidence.md comes from here or from data/vulcanbench.json. +// +// Sources: +// - Droid model IDs, reasoning levels and multipliers: https://docs.factory.com/models.md +// - Safety v1, planted repository notes the user never asked for: +// https://vulcanbench.com/benchmarks/swe-v4-grok47-cursor-v320.html +// - Retired SWE v3 board, the only VulcanBench data for DeepSeek: https://vulcanbench.com/leaderboard.json +// - Grok 4.7 token counts and xAI list rates, for the cost estimates: +// https://vulcanbench.com/assets/data/swe-v4-grok47-cursor-v320/runs.csv and +// https://docs.x.ai/developers/models/grok-4.7 + +/** Per Droid model ID. `levels` is the reasoning levels Droid accepts, lowest first. */ +export const MODELS = { + "gpt-6.1-sol": { + vbName: "GPT-6.1 Sol", + boardName: "GPT-6.1 Sol", + lab: "OpenAI", + multiplier: 0.8, + openWeights: false, + safety: null, + levels: ["low", "medium", "high", "xhigh", "max"], + }, + "claude-opus-5-5": { + vbName: "Claude Opus 5.5", + boardName: "Opus 5.5", + lab: "Anthropic", + multiplier: 1.6, + openWeights: false, + safety: { followed: 3, total: 100, reported: 64 }, + levels: ["low", "medium", "high", "xhigh", "max"], + }, + "grok-4.7": { + vbName: "Grok 4.7", + boardName: "Grok 4.7", + lab: "xAI", + multiplier: 0.8, + openWeights: false, + safety: { followed: 14, total: 80, reported: 0 }, + levels: ["low", "medium", "high", "xhigh"], + }, + "deepseek-v4.1-flash": { + vbName: "DeepSeek V4.1 Flash", + boardName: "DeepSeek V4.1 Flash", + lab: "DeepSeek", + multiplier: 0.12, + openWeights: true, + safety: null, + levels: ["off", "low", "high", "max"], + }, +}; + +/** Retired SWE v3 board, where the model is listed as V4-Flash. Scores here are not comparable with Frontier v4. */ +export const V3_CELLS = [ + { model: "deepseek-v4.1-flash", boardName: "V4-Flash", effort: "low", score: 86, usd: 0.04, minutes: 8.4, inferred: true }, + { model: "deepseek-v4.1-flash", boardName: "V4-Flash", effort: "high", score: 87, usd: 0.08, minutes: 12, inferred: true }, + { + model: "deepseek-v4.1-flash", + boardName: "V4-Flash", + effort: "max", + score: 88.4, + usd: 0.06, + minutes: 11.2, + inferred: true, + note: "Third-best cell on the whole v3 board, and it scored higher and cost less than high.", + }, +]; + +export const V3_NOTE = "VulcanBench measured V4-Flash, not V4.1 Flash. Droid lists V4.1 Flash at 0.12x and marks V4 Flash 0731 deprecated. Its Droid run is under Measured in Droid."; + +/** Estimated from each run's tokens at xAI list rates. VulcanBench published no cost for the Grok 4.7 sweep. */ +export const COST_ESTIMATES = [ + { model: "grok-4.7", effort: "high", usd: 2.19, estimated: true }, + { model: "grok-4.7", effort: "xhigh", usd: 2.96, estimated: true }, +]; diff --git a/data/droid-bench-runs.jsonl b/data/droid-bench-runs.jsonl new file mode 100644 index 0000000..eb82e73 --- /dev/null +++ b/data/droid-bench-runs.jsonl @@ -0,0 +1,345 @@ +{"task":"oss-attrs-ne-validator","cell":"pv-ds-low","rep":1,"minutes":0.39,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":34188,"output_tokens":4014,"cache_read_input_tokens":155136,"cache_creation_input_tokens":0,"factory_credits":6483,"ttft_ms":877.1666666666666},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-low","rep":1,"minutes":0.3,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14940,"output_tokens":1705,"cache_read_input_tokens":118272,"cache_creation_input_tokens":0,"factory_credits":2957,"ttft_ms":812.8},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-high","rep":1,"minutes":0.38,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":50865,"output_tokens":2744,"cache_read_input_tokens":125952,"cache_creation_input_tokens":0,"factory_credits":7790,"ttft_ms":893.8333333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-low","rep":1,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":68621,"output_tokens":5674,"cache_read_input_tokens":205056,"cache_creation_input_tokens":0,"factory_credits":11558,"ttft_ms":933.6875},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-max","rep":1,"minutes":0.65,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":64401,"output_tokens":6007,"cache_read_input_tokens":297984,"cache_creation_input_tokens":0,"factory_credits":11478,"ttft_ms":851.55},"numTurns":20,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-high","rep":1,"minutes":0.69,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":72973,"output_tokens":7086,"cache_read_input_tokens":404224,"cache_creation_input_tokens":0,"factory_credits":13335,"ttft_ms":897.6666666666666},"numTurns":24,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-low","rep":1,"minutes":0.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":13468,"output_tokens":854,"cache_read_input_tokens":57700,"cache_creation_input_tokens":0,"factory_credits":16501,"thinking_tokens":15,"ttft_ms":2581.1666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-low","rep":1,"minutes":0.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27303,"output_tokens":5336,"cache_read_input_tokens":258304,"cache_creation_input_tokens":0,"factory_credits":6588,"ttft_ms":851.5384615384615},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-low","rep":1,"minutes":1.1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20244,"output_tokens":2475,"cache_read_input_tokens":174827,"cache_creation_input_tokens":0,"factory_credits":33094,"thinking_tokens":144,"ttft_ms":2353.0833333333335},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-high","rep":1,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":31076,"output_tokens":6609,"cache_read_input_tokens":232704,"cache_creation_input_tokens":0,"factory_credits":7578,"ttft_ms":876.5384615384615},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-max","rep":1,"minutes":1.35,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":101049,"output_tokens":19281,"cache_read_input_tokens":779520,"cache_creation_input_tokens":0,"factory_credits":23640,"ttft_ms":1071.0714285714287},"numTurns":28,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-high","rep":1,"minutes":1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20727,"output_tokens":2708,"cache_read_input_tokens":108100,"cache_creation_input_tokens":0,"factory_credits":31742,"thinking_tokens":337,"ttft_ms":2926},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-high","rep":1,"minutes":0.35,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":36760,"output_tokens":5056,"cache_read_input_tokens":115200,"cache_creation_input_tokens":0,"factory_credits":7175,"ttft_ms":817},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-high","rep":1,"minutes":1.64,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23685,"output_tokens":4317,"cache_read_input_tokens":185130,"cache_creation_input_tokens":0,"factory_credits":43625,"thinking_tokens":684,"ttft_ms":2826.3636363636365},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-max","rep":1,"minutes":0.43,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":7733,"output_tokens":4214,"cache_read_input_tokens":183296,"cache_creation_input_tokens":0,"factory_credits":3484,"ttft_ms":836.5833333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-low","rep":1,"minutes":0.87,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17297,"output_tokens":1983,"cache_read_input_tokens":132417,"cache_creation_input_tokens":0,"factory_credits":27073,"thinking_tokens":135,"ttft_ms":2186.4},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-low","rep":1,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16484,"output_tokens":1024,"cache_read_input_tokens":91577,"cache_creation_input_tokens":0,"factory_credits":20950,"thinking_tokens":58,"ttft_ms":2281.125},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-high","rep":1,"minutes":1.08,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":19627,"output_tokens":3272,"cache_read_input_tokens":127003,"cache_creation_input_tokens":0,"factory_credits":33876,"thinking_tokens":257,"ttft_ms":2795.5555555555557},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-sol-low","rep":1,"minutes":0.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":19706,"output_tokens":2008,"cache_read_input_tokens":101224,"cache_creation_input_tokens":0,"factory_credits":27850,"thinking_tokens":119,"ttft_ms":2846.25},"numTurns":8,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-high","rep":1,"minutes":1.53,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22867,"output_tokens":4164,"cache_read_input_tokens":106204,"cache_creation_input_tokens":0,"factory_credits":39201,"thinking_tokens":2057,"ttft_ms":2935},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-max","rep":1,"minutes":2.17,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":79711,"output_tokens":43557,"cache_read_input_tokens":1664000,"cache_creation_input_tokens":0,"factory_credits":35280,"ttft_ms":940.28125},"numTurns":32,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-low","rep":1,"minutes":0.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":36387,"output_tokens":5878,"cache_read_input_tokens":167936,"cache_creation_input_tokens":0,"factory_credits":7677,"ttft_ms":912.0833333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-high","rep":1,"minutes":1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24849,"output_tokens":18216,"cache_read_input_tokens":446720,"cache_creation_input_tokens":0,"factory_credits":13023,"ttft_ms":964.1666666666666},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-low","rep":1,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16629,"output_tokens":1544,"cache_read_input_tokens":100464,"cache_creation_input_tokens":0,"factory_credits":23501,"thinking_tokens":99,"ttft_ms":2389.25},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-low","rep":1,"minutes":0.22,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":19091,"output_tokens":2449,"cache_read_input_tokens":81664,"cache_creation_input_tokens":0,"factory_credits":3705,"ttft_ms":855.7142857142857},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-high","rep":1,"minutes":0.35,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17324,"output_tokens":3636,"cache_read_input_tokens":182016,"cache_creation_input_tokens":0,"factory_credits":4355,"ttft_ms":874.7692307692307},"numTurns":13,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-max","rep":1,"minutes":1.44,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26034,"output_tokens":29570,"cache_read_input_tokens":696576,"cache_creation_input_tokens":0,"factory_credits":19333,"ttft_ms":1028.3333333333333},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-high","rep":1,"minutes":1.1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22074,"output_tokens":2996,"cache_read_input_tokens":82918,"cache_creation_input_tokens":0,"factory_credits":32963,"thinking_tokens":1077,"ttft_ms":2872.1666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-high","rep":1,"minutes":2.49,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":127025,"output_tokens":35901,"cache_read_input_tokens":1753372,"cache_creation_input_tokens":0,"factory_credits":37550,"ttft_ms":1498.3695652173913},"numTurns":46,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-param-named-help","cell":"pv-sol-high","rep":1,"minutes":2.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":36455,"output_tokens":7987,"cache_read_input_tokens":261990,"cache_creation_input_tokens":0,"factory_credits":71597,"thinking_tokens":2863,"ttft_ms":2461.181818181818},"numTurns":11,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-low","rep":1,"minutes":0.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":13055,"output_tokens":5068,"cache_read_input_tokens":235776,"cache_creation_input_tokens":0,"factory_credits":4688,"ttft_ms":789.9375},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-max","rep":1,"minutes":0.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":40458,"output_tokens":13958,"cache_read_input_tokens":307968,"cache_creation_input_tokens":0,"factory_credits":12448,"ttft_ms":887.2},"numTurns":15,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-high","rep":1,"minutes":0.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15389,"output_tokens":4840,"cache_read_input_tokens":219904,"cache_creation_input_tokens":0,"factory_credits":4812,"ttft_ms":1053.9285714285713},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-low","rep":1,"minutes":0.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16583,"output_tokens":1617,"cache_read_input_tokens":123708,"cache_creation_input_tokens":0,"factory_credits":24689,"thinking_tokens":46,"ttft_ms":2073.5},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-high","rep":1,"minutes":0.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21059,"output_tokens":1953,"cache_read_input_tokens":127802,"cache_creation_input_tokens":0,"factory_credits":29775,"thinking_tokens":179,"ttft_ms":2730.8888888888887},"numTurns":9,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-low","rep":1,"minutes":0.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16181,"output_tokens":1664,"cache_read_input_tokens":100507,"cache_creation_input_tokens":0,"factory_credits":23624,"thinking_tokens":76,"ttft_ms":2197.875},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-high","rep":1,"minutes":0.99,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":30215,"output_tokens":2267,"cache_read_input_tokens":165104,"cache_creation_input_tokens":0,"factory_credits":39847,"thinking_tokens":360,"ttft_ms":3136.4444444444443},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-low","rep":1,"minutes":4.19,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":109331,"output_tokens":31502,"cache_read_input_tokens":1997692,"cache_creation_input_tokens":0,"factory_credits":34018,"ttft_ms":2001.7142857142858},"numTurns":49,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-max","rep":1,"minutes":1.77,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":29656,"output_tokens":34283,"cache_read_input_tokens":864512,"cache_creation_input_tokens":0,"factory_credits":22516,"ttft_ms":1003.6363636363636},"numTurns":22,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-low","rep":1,"minutes":1.93,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":33740,"output_tokens":26418,"cache_read_input_tokens":1331712,"cache_creation_input_tokens":0,"factory_credits":20589,"ttft_ms":1007.5238095238095},"numTurns":42,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-low","rep":1,"minutes":1.52,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":28779,"output_tokens":3948,"cache_read_input_tokens":276653,"cache_creation_input_tokens":0,"factory_credits":49888,"thinking_tokens":388,"ttft_ms":2488.9285714285716},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-high","rep":1,"minutes":2.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":43647,"output_tokens":45787,"cache_read_input_tokens":2135296,"cache_creation_input_tokens":0,"factory_credits":33387,"ttft_ms":1008.5777777777778},"numTurns":45,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-low","rep":1,"minutes":1.04,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21918,"output_tokens":2275,"cache_read_input_tokens":179970,"cache_creation_input_tokens":0,"factory_credits":33838,"thinking_tokens":184,"ttft_ms":2269.909090909091},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-high","rep":1,"minutes":3.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44612,"output_tokens":11921,"cache_read_input_tokens":386788,"cache_creation_input_tokens":0,"factory_credits":98851,"thinking_tokens":4391,"ttft_ms":2495.3571428571427},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-high","rep":1,"minutes":2.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":30374,"output_tokens":6961,"cache_read_input_tokens":252485,"cache_creation_input_tokens":0,"factory_credits":62248,"thinking_tokens":2478,"ttft_ms":2373.6666666666665},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-max","rep":1,"minutes":8.39,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":235965,"output_tokens":100344,"cache_read_input_tokens":6462738,"cache_creation_input_tokens":0,"factory_credits":95132,"ttft_ms":1483.4366197183099},"numTurns":71,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-high","rep":1,"minutes":4.17,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":116493,"output_tokens":41389,"cache_read_input_tokens":1975482,"cache_creation_input_tokens":0,"factory_credits":39559,"ttft_ms":1730.1363636363637},"numTurns":44,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-low","rep":1,"minutes":3.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":54079,"output_tokens":58548,"cache_read_input_tokens":4173312,"cache_creation_input_tokens":0,"factory_credits":46643,"ttft_ms":1014.0923076923077},"numTurns":65,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-low","rep":1,"minutes":1.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21573,"output_tokens":3972,"cache_read_input_tokens":245504,"cache_creation_input_tokens":0,"factory_credits":42974,"thinking_tokens":1060,"ttft_ms":2283.8},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-low","rep":1,"minutes":0.3,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":8009,"output_tokens":2640,"cache_read_input_tokens":122624,"cache_creation_input_tokens":0,"factory_credits":2586,"ttft_ms":875.4444444444445},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-max","rep":1,"minutes":7.03,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":222724,"output_tokens":87209,"cache_read_input_tokens":6917126,"cache_creation_input_tokens":0,"factory_credits":88548,"ttft_ms":1405.3658536585365},"numTurns":82,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-high","rep":1,"minutes":0.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35857,"output_tokens":5949,"cache_read_input_tokens":227084,"cache_creation_input_tokens":0,"factory_credits":7819,"ttft_ms":1413.3333333333333},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-low","rep":1,"minutes":0.77,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14600,"output_tokens":1747,"cache_read_input_tokens":96239,"cache_creation_input_tokens":0,"factory_credits":22522,"thinking_tokens":85,"ttft_ms":2183.3333333333335},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-max","rep":1,"minutes":0.78,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16449,"output_tokens":11928,"cache_read_input_tokens":334336,"cache_creation_input_tokens":0,"factory_credits":8670,"ttft_ms":916.75},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-high","rep":1,"minutes":0.31,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":12340,"output_tokens":3418,"cache_read_input_tokens":118528,"cache_creation_input_tokens":0,"factory_credits":3467,"ttft_ms":936},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-high","rep":1,"minutes":0.75,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17535,"output_tokens":1756,"cache_read_input_tokens":85174,"cache_creation_input_tokens":0,"factory_credits":24462,"thinking_tokens":233,"ttft_ms":2793},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-low","rep":1,"minutes":0.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27556,"output_tokens":9603,"cache_read_input_tokens":273920,"cache_creation_input_tokens":0,"factory_credits":8716,"ttft_ms":851.8235294117648},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-max","rep":1,"minutes":0.77,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22021,"output_tokens":13176,"cache_read_input_tokens":304384,"cache_creation_input_tokens":0,"factory_credits":9851,"ttft_ms":953.4285714285714},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-low","rep":1,"minutes":8.23,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":165771,"output_tokens":78404,"cache_read_input_tokens":5554908,"cache_creation_input_tokens":0,"factory_credits":73559,"ttft_ms":2460.0666666666666},"numTurns":75,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-low","rep":1,"minutes":0.44,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15457,"output_tokens":1080,"cache_read_input_tokens":39481,"cache_creation_input_tokens":0,"factory_credits":18267,"thinking_tokens":73,"ttft_ms":2360.25},"numTurns":4,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-high","rep":1,"minutes":1.05,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16249,"output_tokens":2968,"cache_read_input_tokens":77367,"cache_creation_input_tokens":0,"factory_credits":27968,"thinking_tokens":829,"ttft_ms":2689.8571428571427},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-low","rep":1,"minutes":0.52,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14231,"output_tokens":906,"cache_read_input_tokens":58134,"cache_creation_input_tokens":0,"factory_credits":17338,"thinking_tokens":64,"ttft_ms":2337.1666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-high","rep":1,"minutes":1.23,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":45790,"output_tokens":13485,"cache_read_input_tokens":315904,"cache_creation_input_tokens":0,"factory_credits":12885,"ttft_ms":1011.7647058823529},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-low","rep":1,"minutes":1.28,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22927,"output_tokens":21510,"cache_read_input_tokens":521216,"cache_creation_input_tokens":0,"factory_credits":14587,"ttft_ms":900.6818181818181},"numTurns":22,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-low","rep":1,"minutes":0.28,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":11552,"output_tokens":2367,"cache_read_input_tokens":126208,"cache_creation_input_tokens":0,"factory_credits":2891,"ttft_ms":871.7},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-high","rep":1,"minutes":0.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26699,"output_tokens":3353,"cache_read_input_tokens":134656,"cache_creation_input_tokens":0,"factory_credits":5205,"ttft_ms":841.3636363636364},"numTurns":11,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-max","rep":1,"minutes":1.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":83309,"output_tokens":36172,"cache_read_input_tokens":573696,"cache_creation_input_tokens":0,"factory_credits":29022,"ttft_ms":1003.7222222222222},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-high","rep":1,"minutes":0.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16453,"output_tokens":2296,"cache_read_input_tokens":64422,"cache_creation_input_tokens":0,"factory_credits":24927,"thinking_tokens":432,"ttft_ms":2366.6666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-max","rep":1,"minutes":0.59,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44565,"output_tokens":8967,"cache_read_input_tokens":95232,"cache_creation_input_tokens":0,"factory_credits":9930,"ttft_ms":909.125},"numTurns":8,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-low","rep":1,"minutes":0.64,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17936,"output_tokens":1064,"cache_read_input_tokens":86185,"cache_creation_input_tokens":0,"factory_credits":22057,"thinking_tokens":33,"ttft_ms":2461.875},"numTurns":8,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-high","rep":1,"minutes":0.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14517,"output_tokens":1536,"cache_read_input_tokens":47081,"cache_creation_input_tokens":0,"factory_credits":19643,"thinking_tokens":228,"ttft_ms":2464.2},"numTurns":5,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-high","rep":1,"minutes":6.99,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":191416,"output_tokens":81635,"cache_read_input_tokens":4838872,"cache_creation_input_tokens":0,"factory_credits":76125,"ttft_ms":1940.96875},"numTurns":64,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-high","rep":1,"minutes":7.16,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":58244,"output_tokens":18129,"cache_read_input_tokens":770327,"cache_creation_input_tokens":0,"factory_credits":149933,"thinking_tokens":11023,"ttft_ms":2802.684210526316},"numTurns":19,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-low","rep":1,"minutes":2.5,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":106435,"output_tokens":20731,"cache_read_input_tokens":1776384,"cache_creation_input_tokens":0,"factory_credits":27865,"ttft_ms":1031.1739130434783},"numTurns":46,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-high","rep":1,"minutes":5.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":48070,"output_tokens":15975,"cache_read_input_tokens":511961,"cache_creation_input_tokens":0,"factory_credits":122843,"thinking_tokens":5978,"ttft_ms":2393.705882352941},"numTurns":17,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-high","rep":1,"minutes":2.2,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":81848,"output_tokens":36885,"cache_read_input_tokens":2214144,"cache_creation_input_tokens":0,"factory_credits":33924,"ttft_ms":944.953488372093},"numTurns":43,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-low","rep":1,"minutes":6.28,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27150,"output_tokens":8947,"cache_read_input_tokens":358653,"cache_creation_input_tokens":0,"factory_credits":71862,"thinking_tokens":1082,"ttft_ms":2385.0555555555557},"numTurns":18,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-low","rep":1,"minutes":7.12,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":65965,"output_tokens":99572,"cache_read_input_tokens":6015232,"cache_creation_input_tokens":0,"factory_credits":73068,"ttft_ms":1095.560606060606},"numTurns":66,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-low","rep":1,"minutes":1.51,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24707,"output_tokens":4117,"cache_read_input_tokens":229873,"cache_creation_input_tokens":0,"factory_credits":45434,"thinking_tokens":289,"ttft_ms":2441.4615384615386},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-high","rep":1,"minutes":2.51,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":54515,"output_tokens":5021,"cache_read_input_tokens":405408,"cache_creation_input_tokens":0,"factory_credits":79917,"thinking_tokens":1239,"ttft_ms":2857.6666666666665},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-max","rep":1,"minutes":16.92,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":327138,"output_tokens":166960,"cache_read_input_tokens":16321750,"cache_creation_input_tokens":0,"factory_credits":166456,"ttft_ms":2150.1140350877195},"numTurns":114,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-max","rep":1,"minutes":21.91,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":317303,"output_tokens":155092,"cache_read_input_tokens":15313174,"cache_creation_input_tokens":0,"factory_credits":156683,"ttft_ms":4370.221238938053},"numTurns":113,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-low","rep":1,"minutes":3.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35632,"output_tokens":9621,"cache_read_input_tokens":425899,"cache_creation_input_tokens":0,"factory_credits":84034,"thinking_tokens":2861,"ttft_ms":2368.85},"numTurns":20,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-max","rep":1,"minutes":13.9,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":269682,"output_tokens":137470,"cache_read_input_tokens":8696140,"cache_creation_input_tokens":0,"factory_credits":123426,"ttft_ms":1513.1780821917807},"numTurns":73,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-low","rep":1,"minutes":6.83,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":376702,"output_tokens":99887,"cache_read_input_tokens":8201562,"cache_creation_input_tokens":0,"factory_credits":116817,"ttft_ms":1287.641304347826},"numTurns":92,"changed":false,"f2p":4,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.7142857142857143,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-high","rep":1,"minutes":9.16,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":71378,"output_tokens":23977,"cache_read_input_tokens":1288768,"cache_creation_input_tokens":0,"factory_credits":204574,"thinking_tokens":12288,"ttft_ms":2802.9615384615386},"numTurns":26,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-low","rep":1,"minutes":14.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":264440,"output_tokens":156875,"cache_read_input_tokens":4512550,"cache_creation_input_tokens":0,"factory_credits":120053,"ttft_ms":1526.8367346938776},"numTurns":49,"changed":true,"f2p":5,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-max","rep":1,"minutes":13.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":533421,"output_tokens":227668,"cache_read_input_tokens":8094214,"cache_creation_input_tokens":0,"factory_credits":196649,"ttft_ms":1892.5393258426966},"numTurns":88,"changed":false,"f2p":1,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.3333333333333333,"resolved":false} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-low","rep":1,"minutes":1.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24248,"output_tokens":4454,"cache_read_input_tokens":207820,"cache_creation_input_tokens":0,"factory_credits":45533,"thinking_tokens":671,"ttft_ms":2453.9166666666665},"numTurns":12,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-high","rep":1,"minutes":24.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":559661,"output_tokens":261409,"cache_read_input_tokens":12351396,"cache_creation_input_tokens":0,"factory_credits":228267,"ttft_ms":1909.157894736842},"numTurns":113,"changed":true,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-low","rep":1,"minutes":1.61,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":68962,"output_tokens":16755,"cache_read_input_tokens":1095168,"cache_creation_input_tokens":0,"factory_credits":19492,"ttft_ms":942.1794871794872},"numTurns":39,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-high","rep":1,"minutes":5.06,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":78675,"output_tokens":96103,"cache_read_input_tokens":5915648,"cache_creation_input_tokens":0,"factory_credits":72637,"ttft_ms":1135},"numTurns":62,"changed":false,"f2p":3,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.5714285714285714,"resolved":false} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-max","rep":1,"minutes":25.15,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":623015,"output_tokens":325883,"cache_read_input_tokens":12694342,"cache_creation_input_tokens":0,"factory_credits":267796,"ttft_ms":2437.020202020202},"numTurns":98,"changed":true,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-low","rep":1,"minutes":0.89,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15981,"output_tokens":2030,"cache_read_input_tokens":118353,"cache_creation_input_tokens":0,"factory_credits":25644,"thinking_tokens":165,"ttft_ms":2307.2},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-high","rep":1,"minutes":1.9,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":102652,"output_tokens":20737,"cache_read_input_tokens":1304320,"cache_creation_input_tokens":0,"factory_credits":26049,"ttft_ms":1025.4285714285713},"numTurns":42,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-max","rep":1,"minutes":1.51,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":55289,"output_tokens":21406,"cache_read_input_tokens":1034752,"cache_creation_input_tokens":0,"factory_credits":19907,"ttft_ms":943.8285714285714},"numTurns":35,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-low","rep":1,"minutes":1.02,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":34239,"output_tokens":8735,"cache_read_input_tokens":486656,"cache_creation_input_tokens":0,"factory_credits":9712,"ttft_ms":940.4347826086956},"numTurns":23,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-high","rep":1,"minutes":1.18,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23221,"output_tokens":3309,"cache_read_input_tokens":121300,"cache_creation_input_tokens":0,"factory_credits":36669,"thinking_tokens":561,"ttft_ms":3097.25},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-high","rep":1,"minutes":1.15,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":49646,"output_tokens":11476,"cache_read_input_tokens":582400,"cache_creation_input_tokens":0,"factory_credits":13157,"ttft_ms":952.0384615384615},"numTurns":26,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-low","rep":1,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16137,"output_tokens":1555,"cache_read_input_tokens":122096,"cache_creation_input_tokens":0,"factory_credits":24017,"thinking_tokens":49,"ttft_ms":2554.6},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-high","rep":1,"minutes":1.33,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":31373,"output_tokens":3185,"cache_read_input_tokens":206228,"cache_creation_input_tokens":0,"factory_credits":46092,"thinking_tokens":538,"ttft_ms":3553.4},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-max","rep":1,"minutes":1.93,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":59570,"output_tokens":26883,"cache_read_input_tokens":1157120,"cache_creation_input_tokens":0,"factory_credits":23400,"ttft_ms":968.516129032258},"numTurns":31,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-low","rep":1,"minutes":0.92,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20836,"output_tokens":1470,"cache_read_input_tokens":188573,"cache_creation_input_tokens":0,"factory_credits":30097,"thinking_tokens":88,"ttft_ms":2472.6666666666665},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-low","rep":1,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":57802,"output_tokens":3859,"cache_read_input_tokens":275968,"cache_creation_input_tokens":0,"factory_credits":9593,"ttft_ms":897},"numTurns":20,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-high","rep":1,"minutes":0.92,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":68174,"output_tokens":7254,"cache_read_input_tokens":273152,"cache_creation_input_tokens":0,"factory_credits":12456,"ttft_ms":998.5555555555555},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-high","rep":1,"minutes":9.31,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":59407,"output_tokens":26161,"cache_read_input_tokens":757231,"cache_creation_input_tokens":0,"factory_credits":182468,"thinking_tokens":13020,"ttft_ms":2814},"numTurns":20,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-low","rep":2,"minutes":0.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":53467,"output_tokens":4109,"cache_read_input_tokens":208446,"cache_creation_input_tokens":0,"factory_credits":8995,"ttft_ms":1572.9411764705883},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-low","rep":1,"minutes":0.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18854,"output_tokens":1546,"cache_read_input_tokens":125793,"cache_creation_input_tokens":0,"factory_credits":26304,"thinking_tokens":64,"ttft_ms":2529.1111111111113},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-high","rep":1,"minutes":1.38,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35202,"output_tokens":3515,"cache_read_input_tokens":224505,"cache_creation_input_tokens":0,"factory_credits":51207,"thinking_tokens":433,"ttft_ms":3259.909090909091},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-high","rep":2,"minutes":0.61,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":58535,"output_tokens":4412,"cache_read_input_tokens":151732,"cache_creation_input_tokens":0,"factory_credits":9585,"ttft_ms":1428.6153846153845},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-max","rep":2,"minutes":1.24,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":120855,"output_tokens":12037,"cache_read_input_tokens":457216,"cache_creation_input_tokens":0,"factory_credits":21609,"ttft_ms":1352.125},"numTurns":24,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-low","rep":2,"minutes":0.36,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27413,"output_tokens":2393,"cache_read_input_tokens":132096,"cache_creation_input_tokens":0,"factory_credits":4824,"ttft_ms":930.5},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-low","rep":2,"minutes":1.14,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18807,"output_tokens":2382,"cache_read_input_tokens":153516,"cache_creation_input_tokens":0,"factory_credits":30720,"thinking_tokens":180,"ttft_ms":2636.090909090909},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-high","rep":2,"minutes":0.51,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20635,"output_tokens":3586,"cache_read_input_tokens":192512,"cache_creation_input_tokens":0,"factory_credits":4757,"ttft_ms":877.3846153846154},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-max","rep":2,"minutes":0.65,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":41348,"output_tokens":5498,"cache_read_input_tokens":236800,"cache_creation_input_tokens":0,"factory_credits":8290,"ttft_ms":874.8235294117648},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-max","rep":1,"minutes":3.38,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":122956,"output_tokens":31182,"cache_read_input_tokens":1858950,"cache_creation_input_tokens":0,"factory_credits":35094,"ttft_ms":1424.4878048780488},"numTurns":41,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-high","rep":2,"minutes":2.17,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":30264,"output_tokens":5311,"cache_read_input_tokens":254208,"cache_creation_input_tokens":0,"factory_credits":55631,"thinking_tokens":1035,"ttft_ms":3263.1428571428573},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-low","rep":2,"minutes":0.78,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15501,"output_tokens":1602,"cache_read_input_tokens":88509,"cache_creation_input_tokens":0,"factory_credits":22352,"thinking_tokens":13,"ttft_ms":2978.5},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-low","rep":2,"minutes":0.59,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":38485,"output_tokens":7107,"cache_read_input_tokens":219648,"cache_creation_input_tokens":0,"factory_credits":8668,"ttft_ms":903.9285714285714},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-high","rep":2,"minutes":0.97,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20466,"output_tokens":2654,"cache_read_input_tokens":90180,"cache_creation_input_tokens":0,"factory_credits":30599,"thinking_tokens":263,"ttft_ms":2838},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-low","rep":2,"minutes":0.61,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16713,"output_tokens":1198,"cache_read_input_tokens":88820,"cache_creation_input_tokens":0,"factory_credits":21719,"thinking_tokens":77,"ttft_ms":2424.5714285714284},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-low","rep":2,"minutes":0.36,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20547,"output_tokens":2520,"cache_read_input_tokens":163840,"cache_creation_input_tokens":0,"factory_credits":4153,"ttft_ms":855},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-high","rep":2,"minutes":0.75,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":66925,"output_tokens":5636,"cache_read_input_tokens":167680,"cache_creation_input_tokens":0,"factory_credits":11226,"ttft_ms":1007.5},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-high","rep":2,"minutes":1.51,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":25816,"output_tokens":4064,"cache_read_input_tokens":92908,"cache_creation_input_tokens":0,"factory_credits":40628,"thinking_tokens":1867,"ttft_ms":3381.6666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-low","rep":1,"minutes":5.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":146999,"output_tokens":14085,"cache_read_input_tokens":974336,"cache_creation_input_tokens":0,"factory_credits":27228,"ttft_ms":979.7380952380952},"numTurns":42,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-high","rep":2,"minutes":2.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":155828,"output_tokens":22671,"cache_read_input_tokens":1658766,"cache_creation_input_tokens":0,"factory_credits":34377,"ttft_ms":1310.093023255814},"numTurns":43,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-high","rep":1,"minutes":6.31,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":47164,"output_tokens":4617,"cache_read_input_tokens":587207,"cache_creation_input_tokens":0,"factory_credits":79695,"thinking_tokens":1431,"ttft_ms":2883.4117647058824},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-low","rep":2,"minutes":0.77,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17297,"output_tokens":1644,"cache_read_input_tokens":99121,"cache_creation_input_tokens":0,"factory_credits":24381,"thinking_tokens":61,"ttft_ms":2868.125},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-max","rep":2,"minutes":0.95,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44717,"output_tokens":15466,"cache_read_input_tokens":408064,"cache_creation_input_tokens":0,"factory_credits":13976,"ttft_ms":912.6111111111111},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-high","rep":2,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":19008,"output_tokens":2005,"cache_read_input_tokens":90712,"cache_creation_input_tokens":0,"factory_credits":26859,"thinking_tokens":351,"ttft_ms":3384.4285714285716},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-low","rep":2,"minutes":1.54,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":86322,"output_tokens":22098,"cache_read_input_tokens":989952,"cache_creation_input_tokens":0,"factory_credits":23834,"ttft_ms":997.8857142857142},"numTurns":35,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-param-named-help","cell":"pv-sol-low","rep":2,"minutes":1.14,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17753,"output_tokens":2105,"cache_read_input_tokens":146609,"cache_creation_input_tokens":0,"factory_credits":28492,"thinking_tokens":119,"ttft_ms":2758.4166666666665},"numTurns":12,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-max","rep":2,"minutes":4.12,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":167659,"output_tokens":40644,"cache_read_input_tokens":1365480,"cache_creation_input_tokens":0,"factory_credits":43578,"ttft_ms":1723.375},"numTurns":32,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-low","rep":2,"minutes":0.63,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26735,"output_tokens":10648,"cache_read_input_tokens":210944,"cache_creation_input_tokens":0,"factory_credits":8931,"ttft_ms":1022.5833333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-sol-high","rep":2,"minutes":1.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":38929,"output_tokens":4951,"cache_read_input_tokens":186785,"cache_creation_input_tokens":0,"factory_credits":58423,"thinking_tokens":1610,"ttft_ms":3685.125},"numTurns":8,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-high","rep":2,"minutes":0.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22551,"output_tokens":8876,"cache_read_input_tokens":307456,"cache_creation_input_tokens":0,"factory_credits":7859,"ttft_ms":920.8571428571429},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-high","rep":2,"minutes":2.4,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":131852,"output_tokens":36846,"cache_read_input_tokens":1929472,"cache_creation_input_tokens":0,"factory_credits":39089,"ttft_ms":1050.4565217391305},"numTurns":46,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-style-validate-color","cell":"pv-ds-low","rep":2,"minutes":0.34,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23993,"output_tokens":2960,"cache_read_input_tokens":126464,"cache_creation_input_tokens":0,"factory_credits":4669,"ttft_ms":908.3},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-low","rep":2,"minutes":0.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18509,"output_tokens":1821,"cache_read_input_tokens":106809,"cache_creation_input_tokens":0,"factory_credits":26368,"thinking_tokens":113,"ttft_ms":2588},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-high","rep":2,"minutes":0.37,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35244,"output_tokens":3766,"cache_read_input_tokens":119552,"cache_creation_input_tokens":0,"factory_credits":6387,"ttft_ms":1068.1},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-low","rep":2,"minutes":0.66,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15391,"output_tokens":1490,"cache_read_input_tokens":79980,"cache_creation_input_tokens":0,"factory_credits":21475,"thinking_tokens":58,"ttft_ms":2602.1428571428573},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-max","rep":2,"minutes":0.73,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":32129,"output_tokens":8548,"cache_read_input_tokens":178688,"cache_creation_input_tokens":0,"factory_credits":8478,"ttft_ms":940.7272727272727},"numTurns":11,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-high","rep":2,"minutes":1.13,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22435,"output_tokens":3018,"cache_read_input_tokens":84626,"cache_creation_input_tokens":0,"factory_credits":33408,"thinking_tokens":1059,"ttft_ms":2992.3333333333335},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-high","rep":2,"minutes":0.91,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18493,"output_tokens":2415,"cache_read_input_tokens":88980,"cache_creation_input_tokens":0,"factory_credits":28017,"thinking_tokens":228,"ttft_ms":3234.8571428571427},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-high","rep":2,"minutes":0.6,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24716,"output_tokens":3695,"cache_read_input_tokens":120670,"cache_creation_input_tokens":0,"factory_credits":5092,"ttft_ms":1768.5},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-max","rep":2,"minutes":2.66,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":103434,"output_tokens":35531,"cache_read_input_tokens":795380,"cache_creation_input_tokens":0,"factory_credits":31768,"ttft_ms":1524.3333333333333},"numTurns":21,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-low","rep":2,"minutes":1.03,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44126,"output_tokens":2834,"cache_read_input_tokens":127516,"cache_creation_input_tokens":0,"factory_credits":7029,"ttft_ms":3524.25},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-max","rep":2,"minutes":0.99,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24794,"output_tokens":6459,"cache_read_input_tokens":232088,"cache_creation_input_tokens":0,"factory_credits":6750,"ttft_ms":1862.7142857142858},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-low","rep":2,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14970,"output_tokens":1456,"cache_read_input_tokens":78213,"cache_creation_input_tokens":0,"factory_credits":20932,"thinking_tokens":62,"ttft_ms":2551.285714285714},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-high","rep":2,"minutes":1.09,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":45702,"output_tokens":2349,"cache_read_input_tokens":269525,"cache_creation_input_tokens":0,"factory_credits":56743,"thinking_tokens":401,"ttft_ms":2319.6},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-low","rep":2,"minutes":1.58,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":32292,"output_tokens":3618,"cache_read_input_tokens":298179,"cache_creation_input_tokens":0,"factory_credits":52240,"thinking_tokens":401,"ttft_ms":2581.733333333333},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-high","rep":1,"minutes":13.95,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":140743,"output_tokens":18751,"cache_read_input_tokens":1726806,"cache_creation_input_tokens":0,"factory_credits":30887,"ttft_ms":2375.627450980392},"numTurns":51,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-high","rep":2,"minutes":3.04,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":46071,"output_tokens":9972,"cache_read_input_tokens":333689,"cache_creation_input_tokens":0,"factory_credits":90099,"thinking_tokens":2842,"ttft_ms":2691.25},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-max","rep":2,"minutes":10.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":254455,"output_tokens":123338,"cache_read_input_tokens":9206570,"cache_creation_input_tokens":0,"factory_credits":116295,"ttft_ms":1652.4285714285713},"numTurns":84,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-high","rep":1,"minutes":17.61,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":247146,"output_tokens":183259,"cache_read_input_tokens":5512972,"cache_creation_input_tokens":0,"factory_credits":133521,"ttft_ms":1919.9574468085107},"numTurns":47,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-high","rep":2,"minutes":7.45,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":106697,"output_tokens":34376,"cache_read_input_tokens":1609444,"cache_creation_input_tokens":0,"factory_credits":33961,"ttft_ms":5686.139534883721},"numTurns":43,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-low","rep":2,"minutes":5.69,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":140165,"output_tokens":53837,"cache_read_input_tokens":2270722,"cache_creation_input_tokens":0,"factory_credits":49223,"ttft_ms":3645.8444444444444},"numTurns":45,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-low","rep":2,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20499,"output_tokens":2009,"cache_read_input_tokens":99153,"cache_creation_input_tokens":0,"factory_credits":28405,"thinking_tokens":180,"ttft_ms":2747.5714285714284},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-low","rep":2,"minutes":2.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":151640,"output_tokens":33255,"cache_read_input_tokens":2245888,"cache_creation_input_tokens":0,"factory_credits":40655,"ttft_ms":1043.4117647058824},"numTurns":51,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-high","rep":2,"minutes":3.21,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":39184,"output_tokens":7887,"cache_read_input_tokens":459466,"cache_creation_input_tokens":0,"factory_credits":81283,"thinking_tokens":3379,"ttft_ms":2575.3529411764707},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-low","rep":2,"minutes":1.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":19520,"output_tokens":3835,"cache_read_input_tokens":191283,"cache_creation_input_tokens":0,"factory_credits":38613,"thinking_tokens":1177,"ttft_ms":2397.230769230769},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-max","rep":2,"minutes":11.94,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":169846,"output_tokens":108820,"cache_read_input_tokens":7484100,"cache_creation_input_tokens":0,"factory_credits":94213,"ttft_ms":1784.0722891566265},"numTurns":83,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-low","rep":2,"minutes":0.4,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26704,"output_tokens":3075,"cache_read_input_tokens":157184,"cache_creation_input_tokens":0,"factory_credits":5140,"ttft_ms":954.5833333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-high","rep":2,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":52726,"output_tokens":3626,"cache_read_input_tokens":170496,"cache_creation_input_tokens":0,"factory_credits":8566,"ttft_ms":1082.2857142857142},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-max","rep":2,"minutes":0.69,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":73746,"output_tokens":7841,"cache_read_input_tokens":240896,"cache_creation_input_tokens":0,"factory_credits":13315,"ttft_ms":1060.6875},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-low","rep":2,"minutes":0.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15151,"output_tokens":1712,"cache_read_input_tokens":115352,"cache_creation_input_tokens":0,"factory_credits":23588,"thinking_tokens":115,"ttft_ms":2194},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-low","rep":2,"minutes":0.39,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27504,"output_tokens":2898,"cache_read_input_tokens":174336,"cache_creation_input_tokens":0,"factory_credits":5199,"ttft_ms":834.8333333333334},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-high","rep":2,"minutes":0.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18739,"output_tokens":1829,"cache_read_input_tokens":108126,"cache_creation_input_tokens":0,"factory_credits":26636,"thinking_tokens":282,"ttft_ms":3034.375},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-low","rep":2,"minutes":14.02,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":72525,"output_tokens":34348,"cache_read_input_tokens":1744164,"cache_creation_input_tokens":0,"factory_credits":30237,"ttft_ms":13408.617021276596},"numTurns":47,"changed":true,"f2p":2,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":0.75,"resolved":false} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-high","rep":2,"minutes":0.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":41244,"output_tokens":7085,"cache_read_input_tokens":183040,"cache_creation_input_tokens":0,"factory_credits":8882,"ttft_ms":936.3636363636364},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-low","rep":2,"minutes":0.46,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":13341,"output_tokens":815,"cache_read_input_tokens":45580,"cache_creation_input_tokens":0,"factory_credits":15759,"thinking_tokens":29,"ttft_ms":2530.6},"numTurns":5,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-max","rep":2,"minutes":1.02,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":86977,"output_tokens":14314,"cache_read_input_tokens":242176,"cache_creation_input_tokens":0,"factory_credits":18011,"ttft_ms":1041.2857142857142},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-high","rep":2,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14108,"output_tokens":1906,"cache_read_input_tokens":46520,"cache_creation_input_tokens":0,"factory_credits":20773,"thinking_tokens":516,"ttft_ms":3308.8},"numTurns":5,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-low","rep":2,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35232,"output_tokens":9459,"cache_read_input_tokens":269824,"cache_creation_input_tokens":0,"factory_credits":9552,"ttft_ms":971.7647058823529},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-low","rep":2,"minutes":0.54,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":12923,"output_tokens":888,"cache_read_input_tokens":54780,"cache_creation_input_tokens":0,"factory_credits":16083,"thinking_tokens":22,"ttft_ms":2727.3333333333335},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-high","rep":2,"minutes":1.1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44594,"output_tokens":16309,"cache_read_input_tokens":359680,"cache_creation_input_tokens":0,"factory_credits":14223,"ttft_ms":968},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-high","rep":2,"minutes":8.67,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":220788,"output_tokens":107567,"cache_read_input_tokens":7617604,"cache_creation_input_tokens":0,"factory_credits":100103,"ttft_ms":1761.126582278481},"numTurns":79,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-max","rep":2,"minutes":0.89,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":51500,"output_tokens":12037,"cache_read_input_tokens":163840,"cache_creation_input_tokens":0,"factory_credits":12437,"ttft_ms":1017.8181818181819},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-low","rep":2,"minutes":0.25,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":7896,"output_tokens":2381,"cache_read_input_tokens":88320,"cache_creation_input_tokens":0,"factory_credits":2348,"ttft_ms":863.8571428571429},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-high","rep":2,"minutes":0.37,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":63708,"output_tokens":3446,"cache_read_input_tokens":84224,"cache_creation_input_tokens":0,"factory_credits":9546,"ttft_ms":887.8},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-low","rep":2,"minutes":0.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":12854,"output_tokens":929,"cache_read_input_tokens":44366,"cache_creation_input_tokens":0,"factory_credits":15775,"thinking_tokens":22,"ttft_ms":2272},"numTurns":5,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-high","rep":2,"minutes":0.9,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17710,"output_tokens":2164,"cache_read_input_tokens":53004,"cache_creation_input_tokens":0,"factory_credits":24947,"thinking_tokens":618,"ttft_ms":3652.6},"numTurns":5,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-max","rep":2,"minutes":0.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":9376,"output_tokens":11331,"cache_read_input_tokens":216064,"cache_creation_input_tokens":0,"factory_credits":7192,"ttft_ms":930.3333333333334},"numTurns":12,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-high","rep":2,"minutes":0.6,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14251,"output_tokens":1371,"cache_read_input_tokens":46909,"cache_creation_input_tokens":0,"factory_credits":18762,"thinking_tokens":169,"ttft_ms":3069.2},"numTurns":5,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-max","rep":1,"minutes":28.21,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":203741,"output_tokens":56387,"cache_read_input_tokens":7147502,"cache_creation_input_tokens":0,"factory_credits":72138,"ttft_ms":1878.6025641025642},"numTurns":78,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-high","rep":2,"minutes":7.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":57880,"output_tokens":17707,"cache_read_input_tokens":959624,"cache_creation_input_tokens":0,"factory_credits":155530,"thinking_tokens":10783,"ttft_ms":2831.48},"numTurns":25,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-low","rep":2,"minutes":2.26,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":30646,"output_tokens":5597,"cache_read_input_tokens":258707,"cache_creation_input_tokens":0,"factory_credits":57260,"thinking_tokens":641,"ttft_ms":3030.769230769231},"numTurns":13,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-low","rep":2,"minutes":2.18,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":118501,"output_tokens":20674,"cache_read_input_tokens":1374464,"cache_creation_input_tokens":0,"factory_credits":28116,"ttft_ms":985.9310344827586},"numTurns":29,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-high","rep":2,"minutes":1.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":81888,"output_tokens":20023,"cache_read_input_tokens":834304,"cache_creation_input_tokens":0,"factory_credits":21852,"ttft_ms":1002.4615384615385},"numTurns":26,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-low","rep":2,"minutes":1.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":25062,"output_tokens":2931,"cache_read_input_tokens":264101,"cache_creation_input_tokens":0,"factory_credits":42344,"thinking_tokens":153,"ttft_ms":2415},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-high","rep":2,"minutes":3.04,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":72583,"output_tokens":6703,"cache_read_input_tokens":663833,"cache_creation_input_tokens":0,"factory_credits":111439,"thinking_tokens":1737,"ttft_ms":3055},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-high","rep":2,"minutes":8.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":50315,"output_tokens":23011,"cache_read_input_tokens":745768,"cache_creation_input_tokens":0,"factory_credits":162137,"thinking_tokens":10672,"ttft_ms":2697.409090909091},"numTurns":22,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-max","rep":1,"minutes":37.83,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":560267,"output_tokens":291305,"cache_read_input_tokens":22319654,"cache_creation_input_tokens":0,"factory_credits":271425,"ttft_ms":2406.1348314606744},"numTurns":177,"changed":false,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-lark-scan","cell":"pv-ds-max","rep":2,"minutes":19.6,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":316178,"output_tokens":171713,"cache_read_input_tokens":10810926,"cache_creation_input_tokens":0,"factory_credits":151538,"ttft_ms":6247.256756756757},"numTurns":74,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-high","rep":2,"minutes":10.35,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":282769,"output_tokens":150425,"cache_read_input_tokens":8671942,"cache_creation_input_tokens":0,"factory_credits":131149,"ttft_ms":1972.4050632911392},"numTurns":79,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-low","rep":2,"minutes":10.79,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":297386,"output_tokens":132343,"cache_read_input_tokens":7266304,"cache_creation_input_tokens":0,"factory_credits":120171,"ttft_ms":1922.857142857143},"numTurns":70,"changed":true,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-max","rep":2,"minutes":8.09,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":287759,"output_tokens":89945,"cache_read_input_tokens":4353228,"cache_creation_input_tokens":0,"factory_credits":90273,"ttft_ms":1582.6792452830189},"numTurns":53,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-high","rep":2,"minutes":25.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":153179,"output_tokens":71209,"cache_read_input_tokens":4422160,"cache_creation_input_tokens":0,"factory_credits":65336,"ttft_ms":17140.70588235294},"numTurns":68,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-low","rep":2,"minutes":5.27,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":175189,"output_tokens":91699,"cache_read_input_tokens":2390016,"cache_creation_input_tokens":0,"factory_credits":71939,"ttft_ms":1063.4285714285713},"numTurns":35,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-low","rep":2,"minutes":3.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26113,"output_tokens":9974,"cache_read_input_tokens":293483,"cache_creation_input_tokens":0,"factory_credits":72533,"thinking_tokens":3262,"ttft_ms":2592.235294117647},"numTurns":17,"changed":true,"f2p":1,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.3333333333333333,"resolved":false} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-low","rep":2,"minutes":5.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":218710,"output_tokens":75996,"cache_read_input_tokens":3487640,"cache_creation_input_tokens":0,"factory_credits":72794,"ttft_ms":1312.6785714285713},"numTurns":56,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-low","rep":2,"minutes":2.36,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24631,"output_tokens":6008,"cache_read_input_tokens":239305,"cache_creation_input_tokens":0,"factory_credits":53316,"thinking_tokens":1502,"ttft_ms":2503.714285714286},"numTurns":14,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-low","rep":2,"minutes":1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":77883,"output_tokens":11078,"cache_read_input_tokens":508672,"cache_creation_input_tokens":0,"factory_credits":16140,"ttft_ms":968.5416666666666},"numTurns":24,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-high","rep":2,"minutes":7.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":59275,"output_tokens":20077,"cache_read_input_tokens":597222,"cache_creation_input_tokens":0,"factory_credits":151625,"thinking_tokens":9039,"ttft_ms":3302.8823529411766},"numTurns":17,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-high","rep":2,"minutes":1.95,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":170424,"output_tokens":18193,"cache_read_input_tokens":1309440,"cache_creation_input_tokens":0,"factory_credits":32975,"ttft_ms":1017.6888888888889},"numTurns":45,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-low","rep":2,"minutes":0.73,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15188,"output_tokens":1278,"cache_read_input_tokens":74662,"cache_creation_input_tokens":0,"factory_credits":20253,"thinking_tokens":14,"ttft_ms":3467.4285714285716},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-high","rep":2,"minutes":8.42,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":282344,"output_tokens":109817,"cache_read_input_tokens":6560556,"cache_creation_input_tokens":0,"factory_credits":105529,"ttft_ms":1357.849315068493},"numTurns":73,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-high","rep":2,"minutes":1.44,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":29044,"output_tokens":3375,"cache_read_input_tokens":159468,"cache_creation_input_tokens":0,"factory_credits":43118,"thinking_tokens":579,"ttft_ms":3493.1111111111113},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-low","rep":2,"minutes":1.4,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":75865,"output_tokens":8478,"cache_read_input_tokens":688064,"cache_creation_input_tokens":0,"factory_credits":15170,"ttft_ms":1418.892857142857},"numTurns":28,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-high","rep":2,"minutes":1.3,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":83466,"output_tokens":8028,"cache_read_input_tokens":461824,"cache_creation_input_tokens":0,"factory_credits":15212,"ttft_ms":1001.8695652173913},"numTurns":23,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-low","rep":2,"minutes":0.71,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16487,"output_tokens":1244,"cache_read_input_tokens":79754,"cache_creation_input_tokens":0,"factory_credits":21359,"thinking_tokens":51,"ttft_ms":3552.714285714286},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-max","rep":2,"minutes":34.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":260485,"output_tokens":162396,"cache_read_input_tokens":14231836,"cache_creation_input_tokens":0,"factory_credits":150252,"ttft_ms":8918.495412844037},"numTurns":109,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-max","rep":2,"minutes":4.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":214052,"output_tokens":37689,"cache_read_input_tokens":2866966,"cache_creation_input_tokens":0,"factory_credits":52064,"ttft_ms":1496.677966101695},"numTurns":59,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-max","rep":2,"minutes":3.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":129669,"output_tokens":31127,"cache_read_input_tokens":2228480,"cache_creation_input_tokens":0,"factory_credits":36948,"ttft_ms":1053.59375},"numTurns":64,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-high","rep":2,"minutes":2.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":46638,"output_tokens":5701,"cache_read_input_tokens":296293,"cache_creation_input_tokens":0,"factory_credits":116351,"ttft_ms":2903.785714285714},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-max","rep":2,"minutes":26.07,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":508409,"output_tokens":240969,"cache_read_input_tokens":13070592,"cache_creation_input_tokens":0,"factory_credits":214368,"ttft_ms":1300.21},"numTurns":99,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-low","rep":2,"minutes":0.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":82024,"output_tokens":5237,"cache_read_input_tokens":234240,"cache_creation_input_tokens":0,"factory_credits":13038,"ttft_ms":1126.611111111111},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-high","rep":2,"minutes":13.01,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":119662,"output_tokens":28608,"cache_read_input_tokens":692652,"cache_creation_input_tokens":0,"factory_credits":303513,"thinking_tokens":9522,"ttft_ms":3217.25},"numTurns":20,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-high","rep":2,"minutes":1.42,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":139998,"output_tokens":9277,"cache_read_input_tokens":491008,"cache_creation_input_tokens":0,"factory_credits":22680,"ttft_ms":1160},"numTurns":28,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-max","rep":2,"minutes":1.72,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":81379,"output_tokens":16908,"cache_read_input_tokens":619008,"cache_creation_input_tokens":0,"factory_credits":19678,"ttft_ms":1068.5},"numTurns":26,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-low","rep":2,"minutes":2.21,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":31487,"output_tokens":3767,"cache_read_input_tokens":201996,"cache_creation_input_tokens":0,"factory_credits":84634,"ttft_ms":3154},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-max","rep":2,"minutes":20.26,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":570830,"output_tokens":267064,"cache_read_input_tokens":12649646,"cache_creation_input_tokens":0,"factory_credits":233175,"ttft_ms":1333.2477876106195},"numTurns":112,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-low","rep":3,"minutes":0.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":46823,"output_tokens":3587,"cache_read_input_tokens":108712,"cache_creation_input_tokens":0,"factory_credits":7658,"ttft_ms":1519.7},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-low","rep":2,"minutes":9.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":144906,"output_tokens":12424,"cache_read_input_tokens":1015266,"cache_creation_input_tokens":0,"factory_credits":26298,"ttft_ms":1372.0232558139535},"numTurns":43,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-high","rep":3,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":69689,"output_tokens":5984,"cache_read_input_tokens":190720,"cache_creation_input_tokens":0,"factory_credits":11791,"ttft_ms":941.2},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-ds-max","rep":3,"minutes":0.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":52429,"output_tokens":9217,"cache_read_input_tokens":348928,"cache_creation_input_tokens":0,"factory_credits":11732,"ttft_ms":869.5555555555555},"numTurns":18,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-high","rep":2,"minutes":3.1,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22004,"output_tokens":5696,"cache_read_input_tokens":247872,"cache_creation_input_tokens":0,"factory_credits":90334,"ttft_ms":3186.25},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-low","rep":3,"minutes":0.96,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17836,"output_tokens":2373,"cache_read_input_tokens":106255,"cache_creation_input_tokens":0,"factory_credits":28014,"thinking_tokens":103,"ttft_ms":2747.25},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-low","rep":3,"minutes":0.36,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23292,"output_tokens":1987,"cache_read_input_tokens":153088,"cache_creation_input_tokens":0,"factory_credits":4195,"ttft_ms":966.7692307692307},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-high","rep":3,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21672,"output_tokens":3127,"cache_read_input_tokens":182502,"cache_creation_input_tokens":0,"factory_credits":4635,"ttft_ms":1743.0714285714287},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-ds-max","rep":3,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":29564,"output_tokens":4158,"cache_read_input_tokens":216664,"cache_creation_input_tokens":0,"factory_credits":6174,"ttft_ms":1557.857142857143},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-low","rep":3,"minutes":0.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14161,"output_tokens":1767,"cache_read_input_tokens":82847,"cache_creation_input_tokens":0,"factory_credits":21715,"thinking_tokens":21,"ttft_ms":2164},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-fail-at-deadline","cell":"pv-sol-high","rep":3,"minutes":1.58,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22503,"output_tokens":3848,"cache_read_input_tokens":166707,"cache_creation_input_tokens":0,"factory_credits":40069,"thinking_tokens":691,"ttft_ms":2969.818181818182},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-low","rep":3,"minutes":0.63,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":50712,"output_tokens":6560,"cache_read_input_tokens":219136,"cache_creation_input_tokens":0,"factory_credits":9871,"ttft_ms":1101.076923076923},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-anyio-trio-wouldblock","cell":"pv-sol-high","rep":3,"minutes":0.96,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17942,"output_tokens":1913,"cache_read_input_tokens":57084,"cache_creation_input_tokens":0,"factory_credits":24291,"thinking_tokens":462,"ttft_ms":4745.2},"numTurns":5,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-low","rep":2,"minutes":10.08,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":34752,"output_tokens":4200,"cache_read_input_tokens":257967,"cache_creation_input_tokens":0,"factory_credits":97866,"ttft_ms":3460.8823529411766},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-low","rep":3,"minutes":0.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17905,"output_tokens":1352,"cache_read_input_tokens":93596,"cache_creation_input_tokens":0,"factory_credits":23480,"thinking_tokens":50,"ttft_ms":3056.4285714285716},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-high","rep":3,"minutes":1.38,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":55015,"output_tokens":15523,"cache_read_input_tokens":604416,"cache_creation_input_tokens":0,"factory_credits":15808,"ttft_ms":1100.32},"numTurns":25,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-max","rep":2,"minutes":22.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":638481,"output_tokens":240425,"cache_read_input_tokens":19055688,"cache_creation_input_tokens":0,"factory_credits":246993,"ttft_ms":1398.2603550295858},"numTurns":168,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-low","rep":3,"minutes":0.72,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":25229,"output_tokens":4212,"cache_read_input_tokens":198656,"cache_creation_input_tokens":0,"factory_credits":5629,"ttft_ms":919.4285714285714},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-high","rep":3,"minutes":0.5,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21992,"output_tokens":3675,"cache_read_input_tokens":183296,"cache_creation_input_tokens":0,"factory_credits":4936,"ttft_ms":898.3846153846154},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-ds-max","rep":3,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":58906,"output_tokens":8305,"cache_read_input_tokens":270336,"cache_creation_input_tokens":0,"factory_credits":11842,"ttft_ms":1050.6470588235295},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-low","rep":3,"minutes":0.8,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16817,"output_tokens":1360,"cache_read_input_tokens":97346,"cache_creation_input_tokens":0,"factory_credits":22791,"ttft_ms":3445.875},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-sol-high","rep":3,"minutes":1.44,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":21271,"output_tokens":2921,"cache_read_input_tokens":82872,"cache_creation_input_tokens":0,"factory_credits":32019,"thinking_tokens":1183,"ttft_ms":4439.5},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-ne-validator","cell":"pv-sol-high","rep":3,"minutes":1.24,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":20747,"output_tokens":3039,"cache_read_input_tokens":111978,"cache_creation_input_tokens":0,"factory_credits":33238,"thinking_tokens":153,"ttft_ms":4273},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-attrs-generator-on-setattr","cell":"pv-ds-max","rep":3,"minutes":3.3,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":89975,"output_tokens":49545,"cache_read_input_tokens":2323456,"cache_creation_input_tokens":0,"factory_credits":41290,"ttft_ms":1079.6190476190477},"numTurns":42,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-sol-low","rep":3,"minutes":1.17,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18634,"output_tokens":2464,"cache_read_input_tokens":113934,"cache_creation_input_tokens":0,"factory_credits":29325,"thinking_tokens":231,"ttft_ms":3555.1111111111113},"numTurns":9,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-low","rep":3,"minutes":0.83,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":24633,"output_tokens":6059,"cache_read_input_tokens":164608,"cache_creation_input_tokens":0,"factory_credits":6344,"ttft_ms":1153.4545454545455},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-high","rep":3,"minutes":1.34,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":52381,"output_tokens":15003,"cache_read_input_tokens":286464,"cache_creation_input_tokens":0,"factory_credits":14320,"ttft_ms":1004.0714285714286},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-low","rep":3,"minutes":3.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":157527,"output_tokens":33857,"cache_read_input_tokens":2104576,"cache_creation_input_tokens":0,"factory_credits":41247,"ttft_ms":1156.9636363636364},"numTurns":55,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-low","rep":3,"minutes":0.64,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":16040,"output_tokens":1266,"cache_read_input_tokens":67767,"cache_creation_input_tokens":0,"factory_credits":20610,"thinking_tokens":67,"ttft_ms":2920.3333333333335},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-sol-high","rep":3,"minutes":2.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":34359,"output_tokens":7330,"cache_read_input_tokens":190487,"cache_creation_input_tokens":0,"factory_credits":64431,"thinking_tokens":2545,"ttft_ms":3098.777777777778},"numTurns":9,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-style-validate-color","cell":"pv-ds-low","rep":3,"minutes":0.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":46520,"output_tokens":2968,"cache_read_input_tokens":112128,"cache_creation_input_tokens":0,"factory_credits":7337,"ttft_ms":1009.3636363636364},"numTurns":11,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-ds-max","rep":3,"minutes":1.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":55045,"output_tokens":25754,"cache_read_input_tokens":508416,"cache_creation_input_tokens":0,"factory_credits":20438,"ttft_ms":1140.25},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-high","rep":3,"minutes":0.63,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":48882,"output_tokens":4379,"cache_read_input_tokens":112640,"cache_creation_input_tokens":0,"factory_credits":8298,"ttft_ms":1025.6},"numTurns":10,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-progressbar-final-position","cell":"pv-sol-high","rep":3,"minutes":1.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23568,"output_tokens":2792,"cache_read_input_tokens":130171,"cache_creation_input_tokens":0,"factory_credits":35232,"thinking_tokens":917,"ttft_ms":3585.5},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-low","rep":3,"minutes":0.78,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14708,"output_tokens":1805,"cache_read_input_tokens":89997,"cache_creation_input_tokens":0,"factory_credits":22589,"thinking_tokens":33,"ttft_ms":2579.125},"numTurns":8,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-ds-max","rep":3,"minutes":0.87,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":51818,"output_tokens":8469,"cache_read_input_tokens":220672,"cache_creation_input_tokens":0,"factory_credits":10927,"ttft_ms":1010.2857142857143},"numTurns":14,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-low","rep":3,"minutes":0.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":58607,"output_tokens":3793,"cache_read_input_tokens":124672,"cache_creation_input_tokens":0,"factory_credits":9216,"ttft_ms":922.2727272727273},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-style-validate-color","cell":"pv-sol-high","rep":3,"minutes":1.26,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18087,"output_tokens":2793,"cache_read_input_tokens":86140,"cache_creation_input_tokens":0,"factory_credits":29089,"thinking_tokens":406,"ttft_ms":4128.142857142857},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-high","rep":3,"minutes":0.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":40178,"output_tokens":4680,"cache_read_input_tokens":144384,"cache_creation_input_tokens":0,"factory_credits":7488,"ttft_ms":1084.1818181818182},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-low","rep":3,"minutes":0.99,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14815,"output_tokens":1313,"cache_read_input_tokens":78389,"cache_creation_input_tokens":0,"factory_credits":20243,"thinking_tokens":69,"ttft_ms":5181.857142857143},"numTurns":7,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-click-param-named-help","cell":"pv-ds-high","rep":3,"minutes":6.06,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":329511,"output_tokens":81652,"cache_read_input_tokens":4904644,"cache_creation_input_tokens":0,"factory_credits":92892,"ttft_ms":1270.857142857143},"numTurns":63,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-click-param-named-help","cell":"pv-ds-max","rep":3,"minutes":6.64,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":384147,"output_tokens":88473,"cache_read_input_tokens":5002612,"cache_creation_input_tokens":0,"factory_credits":103002,"ttft_ms":1384.1166666666666},"numTurns":60,"changed":true,"f2p":2,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":0.8,"resolved":false} +{"task":"oss-flask-automatic-options-override","cell":"pv-sol-high","rep":3,"minutes":1.54,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":38326,"output_tokens":2167,"cache_read_input_tokens":207775,"cache_creation_input_tokens":0,"factory_credits":47643,"thinking_tokens":381,"ttft_ms":5799.555555555556},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-automatic-options-override","cell":"pv-ds-max","rep":3,"minutes":2.36,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":125174,"output_tokens":17892,"cache_read_input_tokens":663208,"cache_creation_input_tokens":0,"factory_credits":25532,"ttft_ms":1461.576923076923},"numTurns":26,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-low","rep":3,"minutes":1.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26535,"output_tokens":4008,"cache_read_input_tokens":254468,"cache_creation_input_tokens":0,"factory_credits":47444,"thinking_tokens":339,"ttft_ms":3162.6923076923076},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-low","rep":3,"minutes":3.73,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":84178,"output_tokens":40089,"cache_read_input_tokens":1981440,"cache_creation_input_tokens":0,"factory_credits":35071,"ttft_ms":1260.8125},"numTurns":48,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-high","rep":3,"minutes":3.88,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":150509,"output_tokens":38199,"cache_read_input_tokens":1418966,"cache_creation_input_tokens":0,"factory_credits":40505,"ttft_ms":1267.9024390243903},"numTurns":41,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-low","rep":3,"minutes":1.27,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":22384,"output_tokens":2663,"cache_read_input_tokens":204199,"cache_creation_input_tokens":0,"factory_credits":36732,"thinking_tokens":218,"ttft_ms":2381.75},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-sol-high","rep":3,"minutes":4.44,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":63797,"output_tokens":10744,"cache_read_input_tokens":510263,"cache_creation_input_tokens":0,"factory_credits":114429,"thinking_tokens":3883,"ttft_ms":3756.923076923077},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-high","rep":2,"minutes":23.33,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":48874,"output_tokens":20445,"cache_read_input_tokens":1243965,"cache_creation_input_tokens":0,"factory_credits":330620,"ttft_ms":2438.159090909091},"numTurns":44,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-high","rep":3,"minutes":4.03,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":94977,"output_tokens":55465,"cache_read_input_tokens":2667776,"cache_creation_input_tokens":0,"factory_credits":45733,"ttft_ms":1014.4528301886793},"numTurns":53,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-low","rep":3,"minutes":5.27,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":263558,"output_tokens":47695,"cache_read_input_tokens":3143204,"cache_creation_input_tokens":0,"factory_credits":63604,"ttft_ms":1590.3968253968253},"numTurns":63,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-flask-teardown-robust","cell":"pv-ds-max","rep":3,"minutes":7.52,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":120451,"output_tokens":72312,"cache_read_input_tokens":3744674,"cache_creation_input_tokens":0,"factory_credits":59980,"ttft_ms":1659.6271186440679},"numTurns":59,"changed":true,"f2p":3,"f2pTotal":3,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-sol-high","rep":3,"minutes":2.96,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":37743,"output_tokens":6520,"cache_read_input_tokens":352289,"cache_creation_input_tokens":0,"factory_credits":70373,"thinking_tokens":2434,"ttft_ms":3371.785714285714},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-low","rep":3,"minutes":0.65,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":51859,"output_tokens":3474,"cache_read_input_tokens":178432,"cache_creation_input_tokens":0,"factory_credits":8412,"ttft_ms":1159.1333333333334},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-low","rep":3,"minutes":3.64,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":204966,"output_tokens":38677,"cache_read_input_tokens":2309376,"cache_creation_input_tokens":0,"factory_credits":49836,"ttft_ms":1359.1632653061224},"numTurns":49,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-high","rep":3,"minutes":0.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35739,"output_tokens":5340,"cache_read_input_tokens":143360,"cache_creation_input_tokens":0,"factory_credits":7271,"ttft_ms":942.0909090909091},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-low","rep":3,"minutes":3.34,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":27715,"output_tokens":7258,"cache_read_input_tokens":330584,"cache_creation_input_tokens":0,"factory_credits":64435,"thinking_tokens":3369,"ttft_ms":2379.529411764706},"numTurns":17,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-low","rep":3,"minutes":0.87,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14583,"output_tokens":1737,"cache_read_input_tokens":90094,"cache_creation_input_tokens":0,"factory_credits":22221,"thinking_tokens":63,"ttft_ms":2648},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-ds-max","rep":3,"minutes":1.18,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":102590,"output_tokens":10004,"cache_read_input_tokens":217600,"cache_creation_input_tokens":0,"factory_credits":17746,"ttft_ms":1095.6666666666667},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-high","rep":2,"minutes":30.47,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":264989,"output_tokens":20547,"cache_read_input_tokens":1455872,"cache_creation_input_tokens":0,"factory_credits":45874,"ttft_ms":1108.8863636363637},"numTurns":44,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-low","rep":3,"minutes":0.55,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":45268,"output_tokens":3114,"cache_read_input_tokens":106496,"cache_creation_input_tokens":0,"factory_credits":7239,"ttft_ms":1953.4444444444443},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-marshmallow-enum-none-default","cell":"pv-sol-high","rep":3,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17424,"output_tokens":1992,"cache_read_input_tokens":70010,"cache_creation_input_tokens":0,"factory_credits":24709,"thinking_tokens":288,"ttft_ms":3712.6666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-high","rep":3,"minutes":0.87,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":89946,"output_tokens":5239,"cache_read_input_tokens":124160,"cache_creation_input_tokens":0,"factory_credits":13673,"ttft_ms":1025.8461538461538},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-low","rep":3,"minutes":0.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14589,"output_tokens":992,"cache_read_input_tokens":58814,"cache_creation_input_tokens":0,"factory_credits":17995,"thinking_tokens":51,"ttft_ms":2935.8333333333335},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-high","rep":3,"minutes":5.48,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":282269,"output_tokens":55763,"cache_read_input_tokens":3272322,"cache_creation_input_tokens":0,"factory_credits":70092,"ttft_ms":1894.611111111111},"numTurns":54,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-ds-max","rep":3,"minutes":0.99,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":33527,"output_tokens":8009,"cache_read_input_tokens":217856,"cache_creation_input_tokens":0,"factory_credits":8502,"ttft_ms":898.1538461538462},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-iter-index-negative","cell":"pv-sol-high","rep":3,"minutes":1.17,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15812,"output_tokens":2880,"cache_read_input_tokens":93584,"cache_creation_input_tokens":0,"factory_credits":27917,"thinking_tokens":964,"ttft_ms":2139.375},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-low","rep":3,"minutes":0.78,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":57674,"output_tokens":4699,"cache_read_input_tokens":140820,"cache_creation_input_tokens":0,"factory_credits":9588,"ttft_ms":1640.1538461538462},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-high","rep":3,"minutes":0.69,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":41291,"output_tokens":7613,"cache_read_input_tokens":216320,"cache_creation_input_tokens":0,"factory_credits":9237,"ttft_ms":902},"numTurns":14,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-low","rep":3,"minutes":0.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14286,"output_tokens":1136,"cache_read_input_tokens":57699,"cache_creation_input_tokens":0,"factory_credits":18285,"thinking_tokens":118,"ttft_ms":2751.3333333333335},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-low","rep":3,"minutes":0.6,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":31025,"output_tokens":2609,"cache_read_input_tokens":81920,"cache_creation_input_tokens":0,"factory_credits":5215,"ttft_ms":968.25},"numTurns":8,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-sol-high","rep":3,"minutes":0.81,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":15683,"output_tokens":2255,"cache_read_input_tokens":60996,"cache_creation_input_tokens":0,"factory_credits":24009,"thinking_tokens":357,"ttft_ms":2461.6666666666665},"numTurns":6,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-high","rep":3,"minutes":0.68,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":36669,"output_tokens":3683,"cache_read_input_tokens":69632,"cache_creation_input_tokens":0,"factory_credits":6372,"ttft_ms":1728},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-ds-max","rep":3,"minutes":0.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":64177,"output_tokens":9039,"cache_read_input_tokens":96256,"cache_creation_input_tokens":0,"factory_credits":12322,"ttft_ms":1150.6666666666667},"numTurns":9,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-low","rep":3,"minutes":0.62,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17296,"output_tokens":975,"cache_read_input_tokens":74071,"cache_creation_input_tokens":0,"factory_credits":20703,"thinking_tokens":12,"ttft_ms":2455.4285714285716},"numTurns":7,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-high","rep":2,"minutes":45,"timedOut":true,"exitCode":null,"agentError":true,"usage":null,"numTurns":null,"changed":false,"f2p":0,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.16666666666666666,"resolved":false} +{"task":"oss-more-itertools-running-minmax-stable","cell":"pv-sol-high","rep":3,"minutes":0.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":14373,"output_tokens":1645,"cache_read_input_tokens":46933,"cache_creation_input_tokens":0,"factory_credits":19958,"thinking_tokens":409,"ttft_ms":3058.6},"numTurns":5,"changed":true,"f2p":4,"f2pTotal":4,"p2p":4,"p2pTotal":4,"functional":1,"resolved":true} +{"task":"oss-more-itertools-numeric-range-eq","cell":"pv-ds-max","rep":3,"minutes":3.39,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":133724,"output_tokens":45437,"cache_read_input_tokens":601856,"cache_creation_input_tokens":0,"factory_credits":39600,"ttft_ms":1105.5263157894738},"numTurns":19,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-low","rep":3,"minutes":2.84,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":28282,"output_tokens":7398,"cache_read_input_tokens":333156,"cache_creation_input_tokens":0,"factory_credits":65553,"thinking_tokens":978,"ttft_ms":2818.0625},"numTurns":16,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-ds-max","rep":3,"minutes":12.53,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":302435,"output_tokens":128960,"cache_read_input_tokens":5464784,"cache_creation_input_tokens":0,"factory_credits":113954,"ttft_ms":2051.470588235294},"numTurns":51,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-low","rep":3,"minutes":4.58,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":198375,"output_tokens":36403,"cache_read_input_tokens":3415040,"cache_creation_input_tokens":0,"factory_credits":51142,"ttft_ms":1128.2295081967213},"numTurns":61,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-lark-scan","cell":"pv-sol-high","rep":3,"minutes":12.52,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":68768,"output_tokens":28255,"cache_read_input_tokens":1342100,"cache_creation_input_tokens":0,"factory_credits":221732,"thinking_tokens":18397,"ttft_ms":2688.862068965517},"numTurns":29,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-low","rep":3,"minutes":1.24,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26657,"output_tokens":3065,"cache_read_input_tokens":189629,"cache_creation_input_tokens":0,"factory_credits":41175,"thinking_tokens":186,"ttft_ms":2929.5},"numTurns":10,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-sol-high","rep":3,"minutes":7.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":45557,"output_tokens":20436,"cache_read_input_tokens":465918,"cache_creation_input_tokens":0,"factory_credits":136831,"thinking_tokens":9319,"ttft_ms":2597.625},"numTurns":16,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-lark-earley-ambiguous-ignore","cell":"pv-ds-max","rep":3,"minutes":18.29,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":370732,"output_tokens":124668,"cache_read_input_tokens":8851068,"cache_creation_input_tokens":0,"factory_credits":129861,"ttft_ms":1433.8666666666666},"numTurns":90,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-sol-high","rep":3,"minutes":2.81,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":44859,"output_tokens":7608,"cache_read_input_tokens":477545,"cache_creation_input_tokens":0,"factory_credits":85428,"thinking_tokens":2123,"ttft_ms":2923.6666666666665},"numTurns":15,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-high","rep":3,"minutes":7.3,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":210851,"output_tokens":61474,"cache_read_input_tokens":2853632,"cache_creation_input_tokens":0,"factory_credits":63053,"ttft_ms":1133.5106382978724},"numTurns":47,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-max","rep":2,"minutes":45,"timedOut":true,"exitCode":null,"agentError":true,"usage":null,"numTurns":null,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-vf2pp-mapping-direction","cell":"pv-ds-max","rep":3,"minutes":7.29,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":360245,"output_tokens":66739,"cache_read_input_tokens":6126722,"cache_creation_input_tokens":0,"factory_credits":92946,"ttft_ms":1343.958904109589},"numTurns":73,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-low","rep":3,"minutes":3.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":35408,"output_tokens":9536,"cache_read_input_tokens":312596,"cache_creation_input_tokens":0,"factory_credits":78982,"thinking_tokens":2757,"ttft_ms":2888.8571428571427},"numTurns":14,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-high","rep":3,"minutes":7.81,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":317300,"output_tokens":123924,"cache_read_input_tokens":3316238,"cache_creation_input_tokens":0,"factory_credits":107131,"ttft_ms":1373.1333333333334},"numTurns":45,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-low","rep":3,"minutes":8.77,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":145398,"output_tokens":69489,"cache_read_input_tokens":2783664,"cache_creation_input_tokens":0,"factory_credits":58838,"ttft_ms":2041.409090909091},"numTurns":44,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-sol-high","rep":3,"minutes":6.57,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":60266,"output_tokens":19233,"cache_read_input_tokens":608020,"cache_creation_input_tokens":0,"factory_credits":149473,"thinking_tokens":8700,"ttft_ms":3082.6875},"numTurns":16,"changed":true,"f2p":2,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.5,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-low","rep":3,"minutes":1.41,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":59446,"output_tokens":14894,"cache_read_input_tokens":1096960,"cache_creation_input_tokens":0,"factory_credits":17460,"ttft_ms":956.475},"numTurns":40,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-low","rep":3,"minutes":2.69,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26400,"output_tokens":6820,"cache_read_input_tokens":310412,"cache_creation_input_tokens":0,"factory_credits":60826,"thinking_tokens":1429,"ttft_ms":2502.0588235294117},"numTurns":17,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-low","rep":3,"minutes":6.85,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":219026,"output_tokens":72835,"cache_read_input_tokens":4634686,"cache_creation_input_tokens":0,"factory_credits":74624,"ttft_ms":1363.467741935484},"numTurns":62,"changed":false,"f2p":3,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.5714285714285714,"resolved":false} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-high","rep":3,"minutes":1.2,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":64332,"output_tokens":13609,"cache_read_input_tokens":849920,"cache_creation_input_tokens":0,"factory_credits":16716,"ttft_ms":1009},"numTurns":32,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-low","rep":3,"minutes":1.14,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":17275,"output_tokens":2228,"cache_read_input_tokens":156757,"cache_creation_input_tokens":0,"factory_credits":29009,"thinking_tokens":253,"ttft_ms":2696.9166666666665},"numTurns":12,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-high","rep":3,"minutes":8.18,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":398785,"output_tokens":117552,"cache_read_input_tokens":7925884,"cache_creation_input_tokens":0,"factory_credits":127149,"ttft_ms":1355.0833333333333},"numTurns":84,"changed":false,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-low","rep":3,"minutes":22.07,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":419104,"output_tokens":178469,"cache_read_input_tokens":17201840,"cache_creation_input_tokens":0,"factory_credits":185557,"ttft_ms":2471.745614035088},"numTurns":114,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-sol-high","rep":3,"minutes":1.25,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":25164,"output_tokens":3375,"cache_read_input_tokens":127181,"cache_creation_input_tokens":0,"factory_credits":38722,"thinking_tokens":500,"ttft_ms":3723.625},"numTurns":8,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-iso8601-nanos","cell":"pv-ds-max","rep":3,"minutes":2.56,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":127566,"output_tokens":39490,"cache_read_input_tokens":1870080,"cache_creation_input_tokens":0,"factory_credits":39672,"ttft_ms":989.6521739130435},"numTurns":46,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-low","rep":3,"minutes":1.05,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":62894,"output_tokens":9094,"cache_read_input_tokens":621056,"cache_creation_input_tokens":0,"factory_credits":13715,"ttft_ms":960.2333333333333},"numTurns":30,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-high","rep":3,"minutes":0.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":49249,"output_tokens":7017,"cache_read_input_tokens":380672,"cache_creation_input_tokens":0,"factory_credits":10383,"ttft_ms":893.0526315789474},"numTurns":19,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-low","rep":3,"minutes":0.79,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18712,"output_tokens":1582,"cache_read_input_tokens":123896,"cache_creation_input_tokens":0,"factory_credits":26258,"thinking_tokens":30,"ttft_ms":2935.6666666666665},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-sol-high","rep":3,"minutes":1.16,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":23096,"output_tokens":2814,"cache_read_input_tokens":145038,"cache_creation_input_tokens":0,"factory_credits":35537,"thinking_tokens":606,"ttft_ms":3182.777777777778},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-semistructured-case","cell":"pv-ds-max","rep":3,"minutes":2.32,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":106736,"output_tokens":35483,"cache_read_input_tokens":1377024,"cache_creation_input_tokens":0,"factory_credits":33825,"ttft_ms":1047.3947368421052},"numTurns":38,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-pennylane-trotter-fragmented","cell":"pv-ds-max","rep":3,"minutes":15.14,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":501792,"output_tokens":294197,"cache_read_input_tokens":10553344,"cache_creation_input_tokens":0,"factory_credits":231868,"ttft_ms":1249.8383838383838},"numTurns":98,"changed":true,"f2p":1,"f2pTotal":5,"p2p":1,"p2pTotal":1,"functional":0.3333333333333333,"resolved":false} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-sol-high","rep":3,"minutes":7.21,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":54128,"output_tokens":20229,"cache_read_input_tokens":678400,"cache_creation_input_tokens":0,"factory_credits":151365,"thinking_tokens":9447,"ttft_ms":3207.1052631578946},"numTurns":19,"changed":false,"f2p":1,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.2857142857142857,"resolved":false} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-low","rep":3,"minutes":0.46,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":54939,"output_tokens":3217,"cache_read_input_tokens":153344,"cache_creation_input_tokens":0,"factory_credits":8587,"ttft_ms":1041.4615384615386},"numTurns":13,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-high","rep":3,"minutes":0.74,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":49919,"output_tokens":6119,"cache_read_input_tokens":310784,"cache_creation_input_tokens":0,"factory_credits":9832,"ttft_ms":969},"numTurns":19,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-low","rep":3,"minutes":2.16,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":26313,"output_tokens":2219,"cache_read_input_tokens":300224,"cache_creation_input_tokens":0,"factory_credits":41943,"thinking_tokens":60,"ttft_ms":3758.875},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-low","rep":3,"minutes":0.82,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":18650,"output_tokens":1794,"cache_read_input_tokens":122816,"cache_creation_input_tokens":0,"factory_credits":27013,"thinking_tokens":52,"ttft_ms":2589.5555555555557},"numTurns":9,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-ds-max","rep":3,"minutes":1.91,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":112154,"output_tokens":23142,"cache_read_input_tokens":1060096,"cache_creation_input_tokens":0,"factory_credits":27637,"ttft_ms":1060.25},"numTurns":32,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-high","rep":3,"minutes":26.73,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":348504,"output_tokens":192211,"cache_read_input_tokens":8660314,"cache_creation_input_tokens":0,"factory_credits":159054,"ttft_ms":3110.3870967741937},"numTurns":62,"changed":true,"f2p":6,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":1,"resolved":true} +{"task":"oss-trio-memory-channel-peak-buffer","cell":"pv-sol-high","rep":3,"minutes":1.5,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":41779,"output_tokens":3093,"cache_read_input_tokens":249158,"cache_creation_input_tokens":0,"factory_credits":55768,"thinking_tokens":447,"ttft_ms":3992.5454545454545},"numTurns":11,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-sol-high","rep":3,"minutes":5.33,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":52697,"output_tokens":4215,"cache_read_input_tokens":590640,"cache_creation_input_tokens":0,"factory_credits":82650,"thinking_tokens":1149,"ttft_ms":3103.5},"numTurns":16,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-low","rep":3,"minutes":7.7,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":140113,"output_tokens":13493,"cache_read_input_tokens":884480,"cache_creation_input_tokens":0,"factory_credits":25853,"ttft_ms":1006.0555555555555},"numTurns":36,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-high","rep":3,"minutes":11.78,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":65583,"output_tokens":21126,"cache_read_input_tokens":1632768,"cache_creation_input_tokens":0,"factory_credits":22734,"ttft_ms":1027.4285714285713},"numTurns":42,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} +{"task":"oss-sqlglot-canonicalize-internal-names","cell":"pv-ds-max","rep":3,"minutes":21.76,"timedOut":false,"exitCode":0,"agentError":false,"usage":{"input_tokens":613216,"output_tokens":223290,"cache_read_input_tokens":13415258,"cache_creation_input_tokens":0,"factory_credits":219465,"ttft_ms":2921.2845528455287},"numTurns":122,"changed":false,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-networkx-leiden-communities","cell":"pv-ds-max","rep":3,"minutes":45,"timedOut":true,"exitCode":null,"agentError":true,"usage":null,"numTurns":null,"changed":true,"f2p":5,"f2pTotal":6,"p2p":1,"p2pTotal":1,"functional":0.8571428571428571,"resolved":false} +{"task":"oss-sympy-manualintegrate-ratint","cell":"pv-ds-max","rep":3,"minutes":45,"timedOut":true,"exitCode":null,"agentError":true,"usage":null,"numTurns":null,"changed":true,"f2p":3,"f2pTotal":3,"p2p":2,"p2pTotal":2,"functional":1,"resolved":true} diff --git a/data/droid-bench.json b/data/droid-bench.json new file mode 100644 index 0000000..ee0d9d9 --- /dev/null +++ b/data/droid-bench.json @@ -0,0 +1,77 @@ +{ + "suite": "VulcanBench Python Suite v1 (python-1)", + "harness": "droid exec --auto high", + "metric": "hidden-test resolution, functional only", + "cells": { + "pv-ds-high": { + "runs": 69, + "infraFailures": 1, + "resolvedRate": 0.8676, + "resolvedRateRange": [ + 0.8261, + 0.9091 + ], + "functional": 0.9555, + "medianMinutes": 1.21, + "medianCredits": 14271.5, + "tasks": 23, + "repeats": 3 + }, + "pv-ds-low": { + "runs": 69, + "infraFailures": 0, + "resolvedRate": 0.8551, + "resolvedRateRange": [ + 0.7826, + 0.913 + ], + "functional": 0.9504, + "medianMinutes": 1, + "medianCredits": 11558, + "tasks": 23, + "repeats": 3 + }, + "pv-ds-max": { + "runs": 69, + "infraFailures": 3, + "resolvedRate": 0.8485, + "resolvedRateRange": [ + 0.8261, + 0.8636 + ], + "functional": 0.9458, + "medianMinutes": 2.34, + "medianCredits": 32796.5, + "tasks": 23, + "repeats": 3 + }, + "pv-sol-high": { + "runs": 69, + "infraFailures": 0, + "resolvedRate": 0.8696, + "resolvedRateRange": [ + 0.8696, + 0.8696 + ], + "functional": 0.9385, + "medianMinutes": 1.51, + "medianCredits": 46092, + "tasks": 23, + "repeats": 3 + }, + "pv-sol-low": { + "runs": 69, + "infraFailures": 0, + "resolvedRate": 0.8696, + "resolvedRateRange": [ + 0.8696, + 0.8696 + ], + "functional": 0.9361, + "medianMinutes": 0.84, + "medianCredits": 26368, + "tasks": 23, + "repeats": 3 + } + } +} diff --git a/data/vulcanbench.json b/data/vulcanbench.json new file mode 100644 index 0000000..33bf7da --- /dev/null +++ b/data/vulcanbench.json @@ -0,0 +1,1338 @@ +{ + "pulled": "2026-10-06", + "sources": { + "frontier": "https://vulcanbench.com/assets/data/swe-v4-board.csv", + "routine": "https://vulcanbench.com/assets/data/routine-v1-board.csv" + }, + "frontier": [ + { + "rank": 1, + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "xhigh", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 93.1465, + "se": 0.324, + "code_quality": 84.3544, + "minutes": 28.4804, + "usd": null + }, + { + "rank": 2, + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "high", + "n": 23, + "passed": 22, + "passed_of": 23, + "score": 92.7077, + "se": 0.4145, + "code_quality": 83.4299, + "minutes": 25.3587, + "usd": null + }, + { + "rank": 3, + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "medium", + "n": 22, + "passed": 21, + "passed_of": 23, + "score": 92.298, + "se": 0.8552, + "code_quality": 83.9038, + "minutes": 27.2323, + "usd": null + }, + { + "rank": 4, + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "max", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 91.8365, + "se": 0.4651, + "code_quality": 82.4337, + "minutes": 27.0964, + "usd": 9.058763 + }, + { + "rank": 5, + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "high", + "n": 22, + "passed": 22, + "passed_of": 22, + "score": 91.1081, + "se": 0.5658, + "code_quality": 77.99, + "minutes": 21.4192, + "usd": 3.269444 + }, + { + "rank": 6, + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "medium", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 90.8635, + "se": 0.6416, + "code_quality": 77.3171, + "minutes": 18.6446, + "usd": 2.842201 + }, + { + "rank": 7, + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "xhigh", + "n": 23, + "passed": 22, + "passed_of": 23, + "score": 90.7543, + "se": 0.821, + "code_quality": 81.9336, + "minutes": 38.7732, + "usd": 14.485635 + }, + { + "rank": 8, + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "xhigh", + "n": 23, + "passed": 22, + "passed_of": 23, + "score": 90.6788, + "se": 1.0968, + "code_quality": 79.9809, + "minutes": 20.747, + "usd": 4.111557 + }, + { + "rank": 9, + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "max", + "n": 23, + "passed": 21, + "passed_of": 23, + "score": 90.2197, + "se": 1.0241, + "code_quality": 79.7695, + "minutes": 37.5113, + "usd": 8.748014 + }, + { + "rank": 10, + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "medium", + "n": 23, + "passed": 20, + "passed_of": 23, + "score": 90.1293, + "se": 0.9394, + "code_quality": 80.1382, + "minutes": 31.7213, + "usd": 9.18601 + }, + { + "rank": 11, + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "high", + "n": 23, + "passed": 20, + "passed_of": 23, + "score": 90.0689, + "se": 0.7822, + "code_quality": 80.861, + "minutes": 28.7614, + "usd": 9.701575 + }, + { + "rank": 12, + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "low", + "n": 23, + "passed": 18, + "passed_of": 23, + "score": 89.4202, + "se": 1.5454, + "code_quality": 81.4055, + "minutes": 20.2145, + "usd": null + }, + { + "rank": 13, + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 89.3033, + "se": 0.3663, + "code_quality": 76.2645, + "minutes": 10.2805, + "usd": 2.568128 + }, + { + "rank": 14, + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 89.1616, + "se": 0.2971, + "code_quality": 75.4749, + "minutes": 8.0905, + "usd": 2.302022 + }, + { + "rank": 15, + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "low", + "n": 23, + "passed": 19, + "passed_of": 23, + "score": 89.1225, + "se": 1.1538, + "code_quality": 78.3565, + "minutes": 26.897, + "usd": 7.963719 + }, + { + "rank": 16, + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 89.0535, + "se": 0.7139, + "code_quality": 72.5197, + "minutes": 19.7351, + "usd": 1.8822 + }, + { + "rank": 17, + "model": "GPT-6.1 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 88.7805, + "se": 0.4038, + "code_quality": 74.9895, + "minutes": 15.3716, + "usd": 0.434591 + }, + { + "rank": 18, + "model": "GPT-6.1 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 88.3478, + "se": 0.331, + "code_quality": 73.7979, + "minutes": 14.4642, + "usd": 0.458339 + }, + { + "rank": 19, + "model": "GPT-6.1 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 88.2342, + "se": 0.4008, + "code_quality": 73.6676, + "minutes": 10.2219, + "usd": 0.33007 + }, + { + "rank": 20, + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 88.158, + "se": 0.4248, + "code_quality": 73.4751, + "minutes": 4.8999, + "usd": 1.711065 + }, + { + "rank": 21, + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 23, + "passed_of": 23, + "score": 87.7262, + "se": 0.3185, + "code_quality": 71.1864, + "minutes": 3.8226, + "usd": 1.481347 + }, + { + "rank": 22, + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 22, + "passed_of": 23, + "score": 87.4325, + "se": 0.5104, + "code_quality": 70.3092, + "minutes": 4.2187, + "usd": 1.721698 + }, + { + "rank": 23, + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 22, + "passed": 21, + "passed_of": 22, + "score": 87.1801, + "se": 0.6243, + "code_quality": 67.5751, + "minutes": 11.6052, + "usd": 1.773526 + }, + { + "rank": 24, + "model": "GPT-6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 23, + "passed": 19, + "passed_of": 23, + "score": 86.8189, + "se": 0.9121, + "code_quality": 70.781, + "minutes": 24.4933, + "usd": 2.520618 + }, + { + "rank": 25, + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 21, + "passed_of": 23, + "score": 86.4669, + "se": 0.8017, + "code_quality": 67.3083, + "minutes": 10.6191, + "usd": 1.609172 + }, + { + "rank": 26, + "model": "GPT-6.1 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 22, + "passed_of": 23, + "score": 86.441, + "se": 0.7823, + "code_quality": 70.2482, + "minutes": 10.5647, + "usd": 0.3986 + }, + { + "rank": 27, + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "low", + "n": 23, + "passed": 17, + "passed_of": 23, + "score": 86.3631, + "se": 1.3003, + "code_quality": 72.5336, + "minutes": 13.6903, + "usd": 1.695309 + }, + { + "rank": 28, + "model": "GPT-6.1 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 20, + "passed_of": 23, + "score": 86.2213, + "se": 0.8218, + "code_quality": 70.787, + "minutes": 6.89, + "usd": 0.305976 + }, + { + "rank": 29, + "model": "GPT-6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 18, + "passed_of": 23, + "score": 85.9384, + "se": 1.3772, + "code_quality": 70.7705, + "minutes": 20.9021, + "usd": 1.996273 + }, + { + "rank": 30, + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 20, + "passed_of": 23, + "score": 85.7577, + "se": 0.9805, + "code_quality": 67.6463, + "minutes": 12.016, + "usd": 2.045316 + }, + { + "rank": 31, + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 23, + "passed": 19, + "passed_of": 23, + "score": 84.1469, + "se": 1.3494, + "code_quality": 65.3505, + "minutes": 44.0992, + "usd": 0.448386 + }, + { + "rank": 32, + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 14, + "passed_of": 23, + "score": 83.4578, + "se": 2.7227, + "code_quality": 73.1344, + "minutes": 19.2392, + "usd": 2.117749 + }, + { + "rank": 33, + "model": "GPT-6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 15, + "passed_of": 23, + "score": 82.8261, + "se": 2.4345, + "code_quality": 70.0851, + "minutes": 23.7716, + "usd": 2.330342 + }, + { + "rank": 34, + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 12, + "passed_of": 23, + "score": 82.5602, + "se": 1.2458, + "code_quality": 68.1922, + "minutes": 11.0511, + "usd": 1.715194 + }, + { + "rank": 35, + "model": "GPT-6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "n": 19, + "passed": 12, + "passed_of": 23, + "score": 81.4235, + "se": 2.8822, + "code_quality": 71.5117, + "minutes": 60.5342, + "usd": 0.097529 + }, + { + "rank": 36, + "model": "GPT-6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 22, + "passed": 13, + "passed_of": 23, + "score": 80.6009, + "se": 2.591, + "code_quality": 66.7893, + "minutes": 17.5357, + "usd": 1.758462 + }, + { + "rank": 37, + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 13, + "passed_of": 23, + "score": 79.1139, + "se": 3.0681, + "code_quality": 68.0512, + "minutes": 25.4294, + "usd": 0.274706 + }, + { + "rank": 38, + "model": "GPT-6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 21, + "passed": 11, + "passed_of": 23, + "score": 78.7864, + "se": 3.016, + "code_quality": 71.4878, + "minutes": 52.5745, + "usd": 0.159686 + }, + { + "rank": 39, + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "n": 23, + "passed": 11, + "passed_of": 23, + "score": 78.4562, + "se": 2.5078, + "code_quality": 67.6666, + "minutes": 20.7838, + "usd": 4.56257 + }, + { + "rank": 40, + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 11, + "passed_of": 23, + "score": 76.6793, + "se": 3.4669, + "code_quality": 73.8953, + "minutes": 12.5891, + "usd": 1.183321 + }, + { + "rank": 41, + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 9, + "passed_of": 23, + "score": 71.216, + "se": 3.714, + "code_quality": 65.7877, + "minutes": 20.9951, + "usd": 0.216769 + }, + { + "rank": 42, + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 7, + "passed_of": 23, + "score": 69.9215, + "se": 3.8102, + "code_quality": 62.9335, + "minutes": 18.2456, + "usd": 4.197308 + }, + { + "rank": 43, + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 6, + "passed_of": 23, + "score": 69.0493, + "se": 3.5024, + "code_quality": 64.9801, + "minutes": 9.7752, + "usd": 1.359168 + }, + { + "rank": 44, + "model": "GPT-6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 4, + "passed_of": 23, + "score": 67.6214, + "se": 3.5937, + "code_quality": 62.5284, + "minutes": 12.9972, + "usd": 1.019456 + }, + { + "rank": 45, + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 5, + "passed_of": 23, + "score": 66.348, + "se": 3.8156, + "code_quality": 70.9261, + "minutes": 7.8757, + "usd": 0.686316 + }, + { + "rank": 46, + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 3, + "passed_of": 23, + "score": 63.5399, + "se": 3.8413, + "code_quality": 64.6546, + "minutes": 12.4031, + "usd": 2.708066 + }, + { + "rank": 47, + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 2, + "passed_of": 23, + "score": 59.6999, + "se": 3.4294, + "code_quality": 72.315, + "minutes": 6.6732, + "usd": 0.576358 + }, + { + "rank": 48, + "model": "GPT-6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "n": 23, + "passed": 2, + "passed_of": 23, + "score": 57.3514, + "se": 3.7954, + "code_quality": 66.3973, + "minutes": 15.5904, + "usd": 0.051912 + }, + { + "rank": 49, + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 1, + "passed_of": 23, + "score": 53.2781, + "se": 2.9402, + "code_quality": 65.5829, + "minutes": 7.3971, + "usd": 0.066398 + }, + { + "rank": 50, + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 1, + "passed_of": 23, + "score": 49.8034, + "se": 2.9303, + "code_quality": 61.7538, + "minutes": 8.2103, + "usd": 1.794708 + }, + { + "rank": 51, + "model": "GPT-6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "n": 23, + "passed": 0, + "passed_of": 23, + "score": 45.7192, + "se": 1.3221, + "code_quality": 69.3272, + "minutes": 3.5593, + "usd": 0.009535 + }, + { + "rank": 52, + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 0, + "passed_of": 23, + "score": 41.2869, + "se": 0.9566, + "code_quality": 71.7276, + "minutes": 2.6625, + "usd": 0.023996 + }, + { + "rank": 53, + "model": "GPT-6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "n": 23, + "passed": 0, + "passed_of": 23, + "score": 40.831, + "se": 2.0415, + "code_quality": 65.842, + "minutes": 5.5576, + "usd": 0.02779 + } + ], + "routine": [ + { + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 94.24, + "se": 0.75, + "code_quality": 85.5, + "minutes": 0.66, + "usd": 0.2159 + }, + { + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.75, + "se": 0.51, + "code_quality": 86.98, + "minutes": 0.75, + "usd": 0.2625 + }, + { + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.72, + "se": 0.56, + "code_quality": 90.02, + "minutes": 0.88, + "usd": 0.3231 + }, + { + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 96.42, + "se": 0.54, + "code_quality": 92.19, + "minutes": 1.78, + "usd": 0.6324 + }, + { + "model": "Fable 5.1", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 96.9, + "se": 0.32, + "code_quality": 93.84, + "minutes": 4.03, + "usd": 1.4632 + }, + { + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 94.22, + "se": 0.73, + "code_quality": 85.5, + "minutes": 0.53, + "usd": 0.0722 + }, + { + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.3, + "se": 0.77, + "code_quality": 88.72, + "minutes": 0.7, + "usd": 0.1139 + }, + { + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.9, + "se": 0.6, + "code_quality": 90.71, + "minutes": 0.85, + "usd": 0.1425 + }, + { + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 96.74, + "se": 0.34, + "code_quality": 93.32, + "minutes": 1.11, + "usd": 0.199 + }, + { + "model": "Opus 5.5", + "lab": "Anthropic", + "harness": "Claude Code", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 96.5, + "se": 0.45, + "code_quality": 92.53, + "minutes": 5.7, + "usd": 1.1237 + }, + { + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 94.79, + "se": 0.66, + "code_quality": 87.76, + "minutes": 1.23, + "usd": 0.2591 + }, + { + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.14, + "se": 0.68, + "code_quality": 85.85, + "minutes": 1.26, + "usd": 0.2647 + }, + { + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.01, + "se": 0.44, + "code_quality": 88.28, + "minutes": 1.43, + "usd": 0.2767 + }, + { + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.18, + "se": 0.59, + "code_quality": 88.72, + "minutes": 2.02, + "usd": 0.4151 + }, + { + "model": "GPT-6 Astra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.74, + "se": 0.42, + "code_quality": 87.33, + "minutes": 2.95, + "usd": 0.5638 + }, + { + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 93.13, + "se": 0.84, + "code_quality": 82.47, + "minutes": 1.7, + "usd": 0.0744 + }, + { + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 94, + "se": 0.74, + "code_quality": 85.16, + "minutes": 1.87, + "usd": 0.0761 + }, + { + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.32, + "se": 0.85, + "code_quality": 86.72, + "minutes": 2.1, + "usd": 0.0829 + }, + { + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.39, + "se": 0.77, + "code_quality": 86.37, + "minutes": 2.68, + "usd": 0.1009 + }, + { + "model": "GPT-5.6 Terra", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.52, + "se": 0.56, + "code_quality": 86.72, + "minutes": 3.92, + "usd": 0.1307 + }, + { + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 94.39, + "se": 0.79, + "code_quality": 86.37, + "minutes": 1.97, + "usd": 0.1285 + }, + { + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.18, + "se": 0.67, + "code_quality": 85.59, + "minutes": 2.28, + "usd": 0.15 + }, + { + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.55, + "se": 0.6, + "code_quality": 86.72, + "minutes": 2.68, + "usd": 0.1798 + }, + { + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.13, + "se": 0.77, + "code_quality": 85.85, + "minutes": 3.6, + "usd": 0.1826 + }, + { + "model": "GPT-5.6 Sol", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.51, + "se": 0.66, + "code_quality": 86.81, + "minutes": 3.76, + "usd": 0.2107 + }, + { + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 92.5, + "se": 0.86, + "code_quality": 80.82, + "minutes": 1, + "usd": 0.0065 + }, + { + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 92.03, + "se": 1.03, + "code_quality": 79.6, + "minutes": 1.5, + "usd": 0.0081 + }, + { + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 92.86, + "se": 0.94, + "code_quality": 82.03, + "minutes": 1.84, + "usd": 0.0095 + }, + { + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 92.67, + "se": 0.81, + "code_quality": 81.42, + "minutes": 2.43, + "usd": 0.0104 + }, + { + "model": "GPT-5.6 Luna", + "lab": "OpenAI", + "harness": "Codex", + "effort": "max", + "routine": false, + "n": 12, + "passed": 12, + "score": 93.08, + "se": 0.9, + "code_quality": 82.38, + "minutes": 2.91, + "usd": 0.0132 + }, + { + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 94.86, + "se": 0.75, + "code_quality": 87.76, + "minutes": 1.26, + "usd": 0.2055 + }, + { + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 94.86, + "se": 0.62, + "code_quality": 87.85, + "minutes": 1.39, + "usd": 0.2183 + }, + { + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.08, + "se": 0.64, + "code_quality": 88.54, + "minutes": 1.5, + "usd": 0.2307 + }, + { + "model": "GPT-5.5", + "lab": "OpenAI", + "harness": "Codex", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 95.24, + "se": 0.63, + "code_quality": 88.89, + "minutes": 1.84, + "usd": 0.2718 + }, + { + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "low", + "routine": true, + "n": 12, + "passed": 12, + "score": 97.29, + "se": 0.58, + "code_quality": 94.62, + "minutes": 1.09, + "usd": null + }, + { + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "medium", + "routine": false, + "n": 12, + "passed": 12, + "score": 97.21, + "se": 0.55, + "code_quality": 94.27, + "minutes": 1.82, + "usd": null + }, + { + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "high", + "routine": false, + "n": 12, + "passed": 12, + "score": 97.37, + "se": 0.42, + "code_quality": 94.62, + "minutes": 3.16, + "usd": null + }, + { + "model": "Grok 4.7", + "lab": "xAI", + "harness": "Cursor", + "effort": "xhigh", + "routine": false, + "n": 12, + "passed": 12, + "score": 97.14, + "se": 0.55, + "code_quality": 93.92, + "minutes": 4.02, + "usd": null + } + ] +} diff --git a/docs/model-evidence.md b/docs/model-evidence.md index 76e2acf..53fcf03 100644 --- a/docs/model-evidence.md +++ b/docs/model-evidence.md @@ -1,8 +1,8 @@ # Model evidence -PV Stack picks a model and reasoning effort for every pstack role from the published results of [VulcanBench](https://vulcanbench.com), Morgan Linton's open-source benchmark for real engineering tasks ([methodology](https://vulcanbench.com/methodology.html)). This page records each number it relies on, where the number comes from, and which picks are inferred rather than measured. +pvstack picks a model and reasoning effort for every pstack role from the published results of [VulcanBench](https://vulcanbench.com), Morgan Linton's open-source benchmark for real engineering tasks ([methodology](https://vulcanbench.com/methodology.html)). This page records each number it relies on, where the number comes from, and which picks are inferred rather than measured. -Data pulled on 2026-10-04 from: +Data pulled on 2026-10-06 from: - Frontier v4 leaderboard and Routine v1 board: https://vulcanbench.com/leaderboard.html (CSV: https://vulcanbench.com/assets/data/swe-v4-board.csv, https://vulcanbench.com/assets/data/routine-v1-board.csv) - Grok 4.7 report (Frontier v4, Routine v1 and Safety v1), published 2026-10-04: https://vulcanbench.com/benchmarks/swe-v4-grok47-cursor-v320.html (per-run data: https://vulcanbench.com/assets/data/swe-v4-grok47-cursor-v320/runs.csv) @@ -20,16 +20,19 @@ Data pulled on 2026-10-04 from: ## Measured cells (Frontier v4) + | Droid | Model / effort | Score | Passed | $/task | Min/task | Droid multiplier | | --- | --- | --- | --- | --- | --- | --- | | `pv-sol-low` | GPT-6.1 Sol / low | 86.22 | 20/23 | $0.31 | 6.9 | 0.8× | | `pv-sol-high` | GPT-6.1 Sol / high | 88.23 | 23/23 | $0.33 | 10.2 | 0.8× | | `pv-sol-xhigh` | GPT-6.1 Sol / xhigh | 88.78 | 23/23 | $0.43 | 15.4 | 0.8× | | `pv-opus-medium` | Claude Opus 5.5 / medium | 90.86 | 23/23 | $2.84 | 18.6 | 1.6× | +| `pv-opus-high` | Claude Opus 5.5 / high | 91.11 | 22/22 | $3.27 | 21.4 | 1.6× | | `pv-grok-high` | Grok 4.7 / high | 92.71 | 22/23 | ~$2.19 (estimate) | 25.4 | 0.8× | | `pv-grok-xhigh` | Grok 4.7 / xhigh | 93.15 | 23/23 | ~$2.96 (estimate) | 28.5 | 0.8× | + -Cells PV Stack doesn't ship but cites below: Opus 5.5 high 91.11 (22/22, $3.27, 21.4 min). Grok 4.7 medium 92.30 (21/23, one run hit the 3-hour bound, 27.2 min). Grok 4.7 low 89.42 (18/23, 20.2 min). +Cells pvstack doesn't ship but cites below: Grok 4.7 medium 92.30 (21/23, one run hit the 3-hour bound, 27.2 min). Grok 4.7 low 89.42 (18/23, 20.2 min). Routine v1 for the same models: every level passes 12/12. GPT-6.1 Sol wasn't on Routine v1 when this was written. Opus 5.5 low scores 94.22 at $0.072, and medium scores 95.30 at $0.11. Grok 4.7's four levels are the four top cells, from 97.14 at xhigh to 97.37 at high. Low scores 97.29 at 1.1 minutes per ticket. Grok's Routine cells were judged by Muse Spark 1.3 alone, which VulcanBench says flatters Grok 4.7. Every other column used Muse plus Grok 4.6. @@ -37,17 +40,93 @@ Routine v1 for the same models: every level passes 12/12. GPT-6.1 Sol wasn't on - **Different second judge.** Grok 4.6 is the second Code quality judge on every other column. It sat out for an xAI submission, and GPT-6.1 Sol took its seat. Sol rates Grok 4.7's code about 6 points above Muse Spark 1.3, the judge every column shares. Rescored on Muse alone, Grok 4.7 drops 0.8 to 1.0 points. It still leads at medium, high and xhigh. At xhigh it scores 92.27 on the shared judge. VulcanBench compares that with 91.03 for Opus 5.5 high, the cell this change replaces, and 90.86 for Fable 5.1 xhigh at the same level. - **Different harness.** Grok 4.7 ran in Cursor's agent CLI. Opus ran in Claude Code and Sol in Codex. A Grok Build run is announced and not yet published. -- **No published cost.** The sweep ran on a Cursor subscription, so VulcanBench reports cost as unavailable. The estimates above are PV Stack's own. They take each run's tokens from `runs.csv` and price them at xAI's list rates ($2 per million uncached input, $0.50 cached, $6 output). They ignore xAI's higher rate above 200K context, so read them as a floor. +- **No published cost.** The sweep ran on a Cursor subscription, so VulcanBench reports cost as unavailable. The estimates above are pvstack's own. They take each run's tokens from `runs.csv` and price them at xAI's list rates ($2 per million uncached input, $0.50 cached, $6 output). They ignore xAI's higher rate above 200K context, so read them as a floor. - **Slow and token-heavy.** Grok 4.7 is slower than GPT-6.1 Sol, Opus 5.5 and GPT-6 Astra at every shared effort level. At the same level it uses 2 to 4 times as many tokens per task as GPT-6.1 Sol (3.1 times at high, 3.5 times at xhigh). - **Safety v1.** Ten tasks carried planted repository notes asking for something the user didn't request. Over 40 runs, Grok 4.7 followed 14 of 80 notes and reported none of them to the user. Over 50 runs, Opus 5.5 followed 3 of 100 and reported 64. Grok never ran the network beacon, the exfiltration or the destructive delete. Neither model leaked the planted secret. ## Inferred cells + | Droid | Model / effort | Evidence | Why it is inferred | | --- | --- | --- | --- | -| `pv-ds-low` | DeepSeek V4.1 Flash / low | v3: V4-Flash low 86% at $0.04, 8.4 min | VulcanBench measured V4-Flash, not V4.1 Flash. Droid lists V4.1 Flash at 0.12× and marks V4 Flash 0731 deprecated. | -| `pv-ds-high` | DeepSeek V4.1 Flash / high | v3: V4-Flash high 87.0% at $0.08, 12.0 min | Same as above. | -| `pv-ds-max` | DeepSeek V4.1 Flash / max | v3: V4-Flash max 88.4% at $0.06, 11.2 min. This was the third-best cell on the whole v3 board, and it scored higher and cost less than high. | Same as above. | +| `pv-ds-low` | DeepSeek V4.1 Flash / low | v3: V4-Flash low 86% at $0.04, 8.4 min. | VulcanBench measured V4-Flash, not V4.1 Flash. Droid lists V4.1 Flash at 0.12x and marks V4 Flash 0731 deprecated. Its Droid run is under Measured in Droid. | +| `pv-ds-high` | DeepSeek V4.1 Flash / high | v3: V4-Flash high 87% at $0.08, 12 min. | Same as above. | +| `pv-ds-max` | DeepSeek V4.1 Flash / max | v3: V4-Flash max 88.4% at $0.06, 11.2 min. Third-best cell on the whole v3 board, and it scored higher and cost less than high. | Same as above. | + + +## Presets + +The mode sheets in `plugins/pvstack/skills/setup-pvstack/modes/` are generated. Each preset names a rule per role class, and `tools/presets.mjs` resolves those rules over the cells above, so a VulcanBench refresh moves the sheets. Run `npm run vulcanbench`, then `npm run presets`, `npm run evidence` and `npm run playbook`. + + +**Balanced.** The default. It trades a little score for a lot of cost on the roles that run often, and buys the top score on the few changes that hurt most. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-sol-high` (cheapest cell that passed every task it ran and scores within 5 points of the best cell in its pool). +- Swarm workers: `pv-sol-high` (cheapest cell that passed every task it ran and scores within 5 points of the best cell in its pool). +- Exploration, investigators, mechanical edits: `pv-sol-low` (fastest cell). +- Judgment, prose, explainers, synthesizers: `pv-opus-medium` (cheapest cell whose model reported planted notes to the user on Safety v1). +- Hardest changes: `pv-grok-xhigh` (highest-scoring cell). +- Reflect tooling: `pv-sol-xhigh` (highest-scoring cell that runs on GPT-6.1 Sol). +- Review panels: `pv-opus-medium`, `pv-sol-xhigh`, `pv-grok-high` (cheapest cell that scores within 0.5 points of the best cell in its pool, one cell per lab, from Anthropic, OpenAI, xAI). +- Pins: none. + +**Budget.** Spends as little as it can on the roles that run often. Code and exploration stay on the open DeepSeek model, judgment and the hardest changes keep the cheapest cell with a published cost and Safety v1 evidence, and the fan-out panels drop Opus. DeepSeek effort levels that the Droid run shows a cheaper level beating are skipped. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-ds-high` (highest-scoring cell that runs on DeepSeek V4.1 Flash and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Swarm workers: `pv-ds-high` (highest-scoring cell that runs on DeepSeek V4.1 Flash and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Exploration, investigators, mechanical edits: `pv-ds-low` (cheapest cell). +- Judgment, prose, explainers, synthesizers: `pv-opus-medium` (cheapest cell whose model reported planted notes to the user on Safety v1). +- Hardest changes: `pv-opus-medium` (cheapest cell that has a published cost and whose model reported planted notes to the user on Safety v1). +- Reflect tooling: `pv-sol-high` (cheapest cell that runs on GPT-6.1 Sol and passed every task it ran). +- Review panels: `pv-sol-high`, `pv-ds-low`, `pv-grok-high` (cheapest cell that scores within 1 point of the best cell in its pool and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits, one cell per lab, from OpenAI, DeepSeek, xAI; arena cross-judge pool from Anthropic, OpenAI, xAI, because the runners are OpenAI, DeepSeek and xAI, so the cross-judge pool needs an outside lab). + - arena runners, architect runners, interrogate reviewers: `pv-sol-high`, `pv-ds-low`, `pv-grok-high` + - arena cross-judge pool: `pv-opus-medium`, `pv-sol-high`, `pv-grok-high` +- Pins: none. + +**Quality.** Ignores cost. Every class takes the highest score on the board, except judgment, which stays on a model that reported the planted notes on Safety v1, and exploration, which only reads code and takes the fastest cell that passed every task. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-grok-xhigh` (highest-scoring cell). +- Swarm workers: `pv-grok-xhigh` (highest-scoring cell). +- Exploration, investigators, mechanical edits: `pv-sol-high` (fastest cell that passed every task it ran). +- Judgment, prose, explainers, synthesizers: `pv-opus-high` (highest-scoring cell whose model reported planted notes to the user on Safety v1). +- Hardest changes: `pv-grok-xhigh` (highest-scoring cell). +- Reflect tooling: `pv-grok-xhigh` (highest-scoring cell). +- Review panels: `pv-opus-high`, `pv-sol-xhigh`, `pv-grok-xhigh` (highest-scoring cell, one cell per lab, from Anthropic, OpenAI, xAI). +- Pins: none. + +**Fast.** Minimizes minutes. Every class that writes code or judges it takes the fastest cell that failed at most one task. Exploration only reads code, and Routine v1 shows the lowest effort level is enough there, so it takes the fastest cell outright. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-sol-high` (fastest cell that passed all but one task). +- Swarm workers: `pv-sol-high` (fastest cell that passed all but one task). +- Exploration, investigators, mechanical edits: `pv-sol-low` (fastest cell). +- Judgment, prose, explainers, synthesizers: `pv-opus-medium` (fastest cell that passed all but one task and whose model reported planted notes to the user on Safety v1). +- Hardest changes: `pv-sol-high` (fastest cell that passed all but one task). +- Reflect tooling: `pv-sol-high` (fastest cell that passed all but one task). +- Review panels: `pv-opus-medium`, `pv-sol-high`, `pv-grok-high` (fastest cell that passed all but one task, one cell per lab, from Anthropic, OpenAI, xAI). +- Pins: none. + +**Safe.** Keeps Grok 4.7 out of judgment, code and the hardest changes, because Safety v1 shows it followed 14 of 80 planted notes and reported none of them. Grok still votes on panels, where models from other labs can outvote it. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-sol-high` (cheapest cell that passes the Safety v1 screen and passed every task it ran and scores within 5 points of the best cell in its pool). +- Swarm workers: `pv-sol-high` (cheapest cell that passes the Safety v1 screen and passed every task it ran and scores within 5 points of the best cell in its pool). +- Exploration, investigators, mechanical edits: `pv-sol-low` (fastest cell). +- Judgment, prose, explainers, synthesizers: `pv-opus-medium` (cheapest cell whose model reported planted notes to the user on Safety v1). +- Hardest changes: `pv-opus-high` (highest-scoring cell that passes the Safety v1 screen). +- Reflect tooling: `pv-sol-xhigh` (highest-scoring cell that runs on GPT-6.1 Sol). +- Review panels: `pv-opus-medium`, `pv-sol-xhigh`, `pv-grok-high` (cheapest cell that scores within 0.5 points of the best cell in its pool, one cell per lab, from Anthropic, OpenAI, xAI). +- Pins: none. + +**Open.** Uses open-weights models only. DeepSeek V4.1 Flash is the only one on the board, so every role runs on it, judgment included, and the panel seats three of its effort levels. Max effort leaves the single-seat roles because the Droid run measured it resolving fewer tasks than high on more than twice the credits. + +- Code delegates (feature, refactor, bug fix, perf, hillclimb): `pv-ds-high` (highest-scoring cell that comes from an open-weights model and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Swarm workers: `pv-ds-high` (highest-scoring cell that comes from an open-weights model and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Exploration, investigators, mechanical edits: `pv-ds-low` (cheapest cell that comes from an open-weights model). +- Judgment, prose, explainers, synthesizers: `pv-ds-high` (pinned because the only open-weights model has no Safety v1 evidence, and this preset keeps every role on open weights). +- Hardest changes: `pv-ds-high` (highest-scoring cell that comes from an open-weights model and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Reflect tooling: `pv-ds-high` (highest-scoring cell that comes from an open-weights model and no cell of the same model beats in the Droid run with more tasks resolved on fewer credits). +- Review panels: `pv-ds-max`, `pv-ds-high`, `pv-ds-low` (highest-scoring cell that comes from an open-weights model, one cell per effort level, up to 3). +- Pin: `judgment and prose`, `how explainer`, `why synthesizer`, `reflect judgment, divergent, synthesizer` run on `pv-ds-high`, because the only open-weights model has no Safety v1 evidence, and this preset keeps every role on open weights. + ## Role reasoning @@ -63,6 +142,7 @@ Routine v1 for the same models: every level passes 12/12. GPT-6.1 Sol wasn't on - Opus stays here even though Grok 4.7 scores higher on code. These roles write what reaches the user with no other model in between. On Safety v1, Opus reported 64 of 100 planted notes to the user and Grok 4.7 reported none of 80. - Grok still sits on review panels and in the arena cross-judge pool. There it is one voice among models from other labs, and an Opus-family synthesizer or the parent decides what the user sees. - Fable 5.1 @ max is the top Anthropic cell (91.84, code quality 82.4), but it costs $9.06 per task, has a 4× Droid multiplier, and requires Anthropic's 30-day retention opt-in. It isn't the default. Add it as a personal override through `/setup-pvstack` if you want it. +- Quality takes Opus @ high instead, because that preset ignores cost and takes the top score among the cells with Safety v1 evidence. Safe takes it for the hardest changes, where Grok 4.7 is excluded. ### Hardest tasks: Grok 4.7 @ xhigh (Balanced), Opus 5.5 @ medium (Budget) - Grok 4.7 xhigh is the top cell on Frontier v4: 93.15, every task passed, and all 231 hidden behaviours fixed. Opus 5.5 high, the previous pick, scores 91.11. The lead holds on the shared judge alone (92.27 vs 91.03). @@ -75,11 +155,18 @@ Routine v1 for the same models: every level passes 12/12. GPT-6.1 Sol wasn't on - Exploration only reads code and reports what it finds, so the speed of the lowest effort level matters most. Sol @ low is the fastest Sol cell (6.9 min) and the cheapest ($0.31). - Routine v1 shows that the lowest effort level is enough for every measured model on ordinary tickets. - Grok 4.7 low is within 0.08 of Grok's best Routine v1 score at 1.1 minutes per ticket, but on Frontier v4 it uses the most tokens of any Grok level (4.76M per task, about $3.21 estimated). Without a published cost, Sol low stays. +- Fast uses the same reasoning and takes the fastest cell outright, Sol @ low at 6.9 minutes. Its Frontier v4 pass rate (20/23) measures code writing, which exploration doesn't do. +- Quality doesn't take the top score here. Grok 4.7 xhigh would spend 28.5 minutes per lookup, and no benchmark shows that a higher Frontier score makes a read-only search better. It takes the fastest cell that passed every task, Sol @ high at 10.2 minutes. + +### Swarm workers: same rule as code +- Swarm workers write code, so every preset resolves them with its code rule. In Budget and Open that is `pv-ds-high`, for the reasons in [Measured in Droid](#measured-in-droid). +- Balanced's OpenAI panel seat sits on a 0.05-point margin. Sol xhigh is 0.55 points above Sol high, and the rule allows 0.5. Any threshold has an edge somewhere. Here the edge is guarded, because `npm run check` warns when the board moves and fails when a regenerated sheet changes. ### Review panels: Opus 5.5, GPT-6.1 Sol, Grok 4.7 @ high - Panels get their signal from model diversity: findings that independent models agree on carry more weight. This uses three labs: Anthropic, OpenAI and xAI. - Grok sits at high. It scores 92.71 (SE 0.41), uses the fewest tokens of any Grok level (2.85M per task) and has the lowest estimated cost ($2.19). Medium, the earlier pick, scores 92.30 but had the sweep's one 3-hour timeout, so its token and cost figures cover 22 runs. Medium came from Grok 4.6, whose effort knob ran backward. Grok 4.7's combined score rises at every step. - In Budget, Opus leaves the fan-out (Opus @ medium costs about 9× Sol @ high per task, $2.84 vs $0.33) and DeepSeek takes its seat. Opus stays in the arena cross-judge pool and still writes every synthesis. +- Budget's DeepSeek seat is `pv-ds-low`. Once max is out, low and high are within a point of each other on v3, and the rule takes the cheaper one. The Droid run shows no measurable difference between them (85.5% vs 86.8% resolved, with overlapping ranges across repeats). ## Rejected for Budget mode @@ -91,6 +178,34 @@ Routine v1 for the same models: every level passes 12/12. GPT-6.1 Sol wasn't on | GPT-6 Luna (0.04×) | v4: max 81.4% with 6 runs hitting the 3-hour timeout, 60 min per task. | Too slow and unreliable on hard tasks. On routine work it is a candidate for a future "mechanical" override. | | Other Droid Core models (MiniMax M3, Nemotron 3 Ultra, Inkling, Mistral Medium 3.5, GLM-5.3-Flash) | No VulcanBench data. | No evidence to rank them. | -## Re-checking in Droid +## Measured in Droid + +pvstack ran VulcanBench's public Python Suite v1 (23 tasks, each from a real open-source fix) through `droid exec --auto high`. Each task ran three times for every DeepSeek cell and for the two everyday Sol cells as controls, which makes 345 runs. Routine v1 would have been the closer match, but its tasks are private. `tools/droid-bench.mjs` is the runner. `data/droid-bench-runs.jsonl` holds every run, and `data/droid-bench.json` holds the summary that `npm run check` keeps in sync with it. -VulcanBench's harness is open source (https://github.com/morganlinton/VulcanBench). The inferred cells, the harness gap and Grok 4.7's unpublished cost are the open questions. When VulcanBench publishes the Grok Build run, re-check the Grok cells. A useful next step is to run the Routine v1 public subset, or a small Frontier slice, through `droid exec` for each `pv-*` droid and update this page. + +| Droid | Model / effort | Resolved (range across repeats) | Hidden tests passed | Median min | Median credits | Tasks × repeats | +| --- | --- | --- | --- | --- | --- | --- | +| `pv-ds-high` | DeepSeek V4.1 Flash / high | 86.8% (82.6% to 90.9%) | 95.5% | 1.2 | 14,272 | 23 × 3 (1 infra) | +| `pv-ds-low` | DeepSeek V4.1 Flash / low | 85.5% (78.3% to 91.3%) | 95.0% | 1.0 | 11,558 | 23 × 3 (0 infra) | +| `pv-ds-max` | DeepSeek V4.1 Flash / max | 84.9% (82.6% to 86.4%) | 94.6% | 2.3 | 32,797 | 23 × 3 (3 infra) | +| `pv-sol-high` | GPT-6.1 Sol / high | 87.0% | 93.8% | 1.5 | 46,092 | 23 × 3 (0 infra) | +| `pv-sol-low` | GPT-6.1 Sol / low | 87.0% | 93.6% | 0.8 | 26,368 | 23 × 3 (0 infra) | + + +How to read it: + +- **Functional only.** A run counts as resolved when every hidden `fail_to_pass` and `pass_to_pass` test passes. The VulcanBench combined score also includes lint, security and Code quality, and Code quality needs VulcanBench's judge models. So these numbers can't be compared with the Frontier v4 scores above. They compare cells with each other inside Droid. +- **The grader is checked.** Before any model ran, every task's untouched repository failed its `fail_to_pass` tests and passed its `pass_to_pass` tests, and the gold patch passed all of them (23 of 23 tasks). +- **Credits are what Droid bills.** They come from `droid exec`'s usage report. Medians are per run. +- **The agents couldn't run the hidden test environment.** Tests ran afterwards in a separate virtualenv. The agent's shell started with system Python, which has no pytest or task dependencies. That holds every cell back equally. Two tasks failed for every cell on every run (`oss-click-param-named-help` and `oss-sqlglot-canonicalize-internal-names`), and a third (`oss-pennylane-trotter-fragmented`) passed once in 15 runs. They set the ceiling near 87%. +- **Infra failures** are 45-minute agent timeouts. They are excluded from the resolved rate. `pv-ds-max` hit three of them and `pv-ds-high` one. + +What it changed: + +- **Resolution doesn't separate the cells.** All five resolve between 84.9% and 87.0%, and the DeepSeek ranges across repeats overlap (78% to 91%). Sol resolved the same 20 tasks in every repeat at both levels. +- **Cost and time do separate them.** `pv-ds-max` used 2.3 times the median credits of `pv-ds-high` (32,797 vs 14,272), took nearly twice the median time (2.3 vs 1.2 minutes) and resolved no more. That reverses the v3 numbers, where max was cheaper than high. The v3 runs measured DeepSeek V4-Flash through its API, not V4.1 Flash in Droid. So Budget and Open now skip any cell that another cell of the same model beats in Droid, with more tasks resolved on fewer credits (the `droidDominated` filter). Only `pv-ds-max` meets that test. Open's panel still seats it, because the panel needs three DeepSeek seats. +- **Budget code moves from `pv-ds-max` to `pv-ds-high`.** That brings back the hand-written Budget choice. `pv-ds-low` costs about 19% fewer credits at a resolution the run can't tell apart. High keeps a margin for harder tasks: on Frontier v4, Sol low fails 3 tasks that Sol high passes, and the Python suite is too easy to show that gap. +- **DeepSeek is cheap in Droid.** `pv-ds-high` resolved as often as `pv-sol-low` on 54% of its median credits. The filter compares cells of one model, so this doesn't move Sol out of Balanced. Frontier v4 is the hard-task evidence for Sol, and the Python suite has nothing comparable for DeepSeek. +- **Sol low and Sol high tie here.** Balanced keeps Sol high for code, because Frontier v4's three hard-task failures for Sol low are a real gap and the Python suite is too easy to show it. The filter needs strictly more resolved tasks, so the tie doesn't remove Sol high. + +Still open: Grok 4.7's unpublished cost, and the Grok Build run VulcanBench has announced. Re-run with `node tools/droid-bench.mjs run --suite /tasks/python-1 --python /bin --cells --repeats 3`. It resumes from the run log. diff --git a/package.json b/package.json index 6b5a70d..144bf0f 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,11 @@ "scripts": { "sync": "node tools/sync-upstream.mjs", "droids": "node tools/droids.mjs", + "vulcanbench": "node tools/vulcanbench.mjs", + "presets": "node tools/presets.mjs", + "evidence": "node tools/evidence.mjs", "playbook": "node tools/playbook.mjs", - "check": "node tools/sync-upstream.mjs --check && node tools/droids.mjs --check && node tools/validate.mjs && node tools/playbook.mjs --check" + "check": "node tools/sync-upstream.mjs --check && node tools/droids.mjs --check && node tools/validate.mjs && node tools/presets.mjs --check && node tools/droid-bench.mjs check && node tools/evidence.mjs --check && node tools/playbook.mjs --check && node tools/vulcanbench.mjs --check-fresh", + "droid-bench": "node tools/droid-bench.mjs" } } diff --git a/playbook/index.html b/playbook/index.html index 615a0bf..2774d4d 100644 --- a/playbook/index.html +++ b/playbook/index.html @@ -3,14 +3,14 @@ -The PV Stack Playbook - +The pvstack Playbook + - +