diff --git a/.github/ISSUE_TEMPLATE/qwen35_compatibility.yml b/.github/ISSUE_TEMPLATE/qwen35_compatibility.yml new file mode 100644 index 0000000..5c5cd03 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/qwen35_compatibility.yml @@ -0,0 +1,104 @@ +name: Qwen3.5 compatibility report +description: Record a reproducible working, partial, or failing Qwen3.5 configuration. +title: "[Qwen3.5] " +body: + - type: markdown + attributes: + value: | + This form maps real compatibility; it does not certify output quality or production readiness. **Never paste access tokens, authentication files, private prompts, proprietary model data, or other secrets.** Use a short public or synthetic prompt. + + - type: dropdown + id: outcome + attributes: + label: Outcome + options: + - Works for the reported path + - Partially works or gives incorrect behavior + - Fails with an exception + validations: + required: true + + - type: input + id: recurquant_version + attributes: + label: RecurQuant version or commit + placeholder: 0.2.0a1 or full commit hash + validations: + required: true + + - type: input + id: checkpoint + attributes: + label: Exact Qwen3.5 checkpoint and revision + description: Include both the model ID and immutable revision hash. + placeholder: Qwen/Qwen3.5-0.8B-Base @ dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68 + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Software and hardware + placeholder: | + OS: Windows 11 + Python: 3.11.x + PyTorch: 2.x (include CUDA suffix) + Transformers: 5.14.1 + Device: NVIDIA GPU model or CPU + CUDA runtime and driver: version or not applicable + render: shell + validations: + required: true + + - type: textarea + id: execution + attributes: + label: Execution configuration + description: Report values even when the run succeeds. + placeholder: | + Dtype: bfloat16 + Attention implementation: eager + Device map or placement: one CUDA device + Batch size: 1 + Path: direct forward, greedy generation, beam search, etc. + Prefill tokens: 32 + Decode or generated tokens: 32 + Cache: bits=4, group_size=128, rounding=nearest + Layer overrides: none + validations: + required: true + + - type: textarea + id: reproducer + attributes: + label: Minimal public reproducer + description: Use a public or synthetic prompt and include model loading, cache creation, and the failing or working call. + render: python + validations: + required: true + + - type: textarea + id: result + attributes: + label: Result + description: For a failure, include the complete traceback. For a working path, state exactly what completed; do not infer quality, speed, or memory improvements. + render: shell + validations: + required: true + + - type: textarea + id: storage_summary + attributes: + label: Cache storage summary + description: Paste `cache.storage_summary()` after at least one cache update when available. + render: shell + + - type: checkboxes + id: disclosure + attributes: + label: Submission check + options: + - label: I used a public or synthetic prompt and removed secrets, access tokens, authentication files, private prompts, and proprietary data. + required: true + - label: I reported the exact checkpoint revision and did not turn a successful run into a quality, speed, peak-memory, or production-readiness claim. + required: true diff --git a/.github/ISSUE_TEMPLATE/reproducible_bug.yml b/.github/ISSUE_TEMPLATE/reproducible_bug.yml new file mode 100644 index 0000000..41a8428 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/reproducible_bug.yml @@ -0,0 +1,105 @@ +name: Reproducible bug +description: Report a RecurQuant defect with enough detail to reproduce it. +title: "[Bug] " +body: + - type: markdown + attributes: + value: | + Thanks for reducing the problem. **Do not include access tokens, authentication files, private prompts, proprietary model data, or other secrets.** Replace private inputs with a minimal public example before submitting. + + - type: dropdown + id: area + attributes: + label: Affected area + options: + - Packed cache integration + - Quantization or byte accounting + - Qwen3.5 factory or generation + - CLI or packaging + - Evaluation tooling + - Other + validations: + required: true + + - type: input + id: recurquant_version + attributes: + label: RecurQuant version or commit + description: Use `python -c "import recurquant; print(recurquant.__version__)"` or provide the full commit hash. + placeholder: 0.2.0a1 or 0123456789abcdef... + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: Include every line and replace only values that truly do not apply. + placeholder: | + OS: Windows 11 + Python: 3.11.x + PyTorch: 2.x (include CUDA suffix) + Transformers: 5.14.1 + Device: NVIDIA GPU model or CPU + Dtype: bfloat16, float16, or float32 + Attention implementation: eager + Batch size: 1 + Execution mode: direct forward, greedy generation, beam search, etc. + validations: + required: true + + - type: textarea + id: model + attributes: + label: Model and cache configuration + description: Give the exact model ID, revision, cache bits, group size, rounding mode, and any layer overrides. Write `not applicable` for a model-independent bug. + placeholder: | + Model: Qwen/Qwen3.5-0.8B-Base + Revision: dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68 + Cache: bits=4, group_size=128, rounding=nearest + Layer overrides: none + validations: + required: true + + - type: textarea + id: reproducer + attributes: + label: Minimal reproducer + description: Provide one self-contained example using public or synthetic input. + render: python + validations: + required: true + + - type: textarea + id: observed + attributes: + label: Observed behavior and traceback + description: Paste the complete traceback or exact incorrect output. Remove secrets and private paths where possible. + render: shell + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: State the behavior you expected and why. + validations: + required: true + + - type: textarea + id: storage_summary + attributes: + label: Storage summary + description: For packing or byte-accounting bugs, paste `cache.storage_summary()`. Otherwise write `not applicable`. + render: shell + + - type: checkboxes + id: disclosure + attributes: + label: Submission check + options: + - label: I used a minimal public or synthetic input and removed secrets, access tokens, authentication files, private prompts, and proprietary data. + required: true + - label: I checked the current compatibility boundary and am not presenting an unsupported mode as validated. + required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d3ff01..3d685ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: CI on: push: + branches: + - main pull_request: permissions: @@ -13,15 +15,83 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v7.0.1 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7.0.0 with: python-version: "3.11" - uses: astral-sh/setup-uv@v9.0.0 with: python-version: "3.11" + version: "0.11.31" - name: Install run: uv pip install --system -e ".[dev]" + - name: Verify frozen MBPP confirmation + run: | + recurquant verify-confirmation \ + evidence/mbpp-v02-confirmation.json \ + evidence/mbpp-v02-confirmation-manifest.json \ + --expect-artifact-sha256 70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9 \ + --expect-artifact-evidence-sha256 2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0 - name: Lint run: ruff check . - name: Test run: pytest + - name: Check generated README assets + run: python scripts/generate_readme_assets.py --check + + package: + name: Build wheel and sdist + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-python@v7.0.0 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@v9.0.0 + with: + version: "0.11.31" + - name: Build distributions + run: uv build + - name: Check distribution metadata + run: uvx --from "twine==6.2.0" twine check --strict dist/* + - uses: actions/upload-artifact@v7.0.1 + with: + name: recurquant-dist + path: dist/* + if-no-files-found: error + retention-days: 7 + + wheel-install: + name: Wheel import / ${{ matrix.os }} / Python ${{ matrix.python-version }} + needs: package + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python-version: "3.11" + - os: ubuntu-latest + python-version: "3.13" + - os: windows-latest + python-version: "3.11" + steps: + - uses: actions/setup-python@v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v8.0.1 + with: + name: recurquant-dist + path: dist + - name: Install CPU PyTorch + run: python -m pip install "torch>=2.7" --index-url https://download.pytorch.org/whl/cpu + - name: Install the built wheel + run: >- + python -c "import pathlib, subprocess, sys; + wheel = next(pathlib.Path('dist').glob('*.whl')); + subprocess.check_call([sys.executable, '-m', 'pip', 'install', str(wheel)])" + - name: Validate import and console entry point + run: | + python -c "import importlib.metadata as m, recurquant; assert m.version('recurquant') == recurquant.__version__ == '0.2.0a1'; print(recurquant.__version__)" + recurquant demo --bits 4 --group-size 16 diff --git a/.gitleaks.toml b/.gitleaks.toml index 72a6e31..f598701 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -11,3 +11,11 @@ regexes = [ '''"token_ids_sha256":\s*"[0-9a-f]{64}"''', '''"tokenizer_revision":\s*"[0-9a-f]{40}"''', ] + +[[allowlists]] +description = "Prepared-manifest token ID digests" +targetRules = ["generic-api-key"] +regexTarget = "line" +regexes = [ + '''"(?:code_token_ids_sha256|prompt_token_ids_sha256|token_manifest_sha256)"\s*:\s*"[0-9a-f]{64}"''', +] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..02c6a4c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +This file records user-visible package changes. RecurQuant is pre-release +software, so compatibility can still change between development versions. + +## [0.2.0a1] - 2026-07-22 + +### Added + +- Physical INT4 nibble packing and INT8 payload storage for persistent Gated + DeltaNet recurrent states, with grouped FP16 or FP32 scales. +- `PackedRecurrentStateCache` for keeping recurrent states packed between layer + calls, with opt-in evidence recording and exact resident-byte accounting. +- A guarded Qwen3.5 cache factory, pinned Qwen3.5 quickstart, and compatibility + checks for the tested Transformers release and eager, single-device + inference path. +- A reusable frozen v0.2 mixed-policy cache helper and an installed + `recurquant qwen35` workflow shared with the source-tree quickstart. Uniform + INT4 remains available only as an explicitly named stress baseline. +- Machine-readable `recurquant qwen35 --json` output, a one-click Colab + notebook, and a hash-anchored reproduction and extension guide. +- Offline `recurquant verify-artifact` checks for strict JSON, whole-file + SHA256 anchors, and canonical evidence hashes. +- `recurquant verify-confirmation` independently checks the frozen MBPP + manifest, artifact, quality gates, and optional raw checkpoint arrays. It + distinguishes verified pass/fail outcomes from unanchored evidence. +- Frozen MBPP calibration and development evaluation workflows with pinned + dataset/model revisions, prepared token manifests, canonical evidence hashes, + equal-byte baselines, and resumable per-task checkpoints. +- Held-out confirmation evidence on all 500 frozen MBPP test tasks and 30,244 + teacher-forced reference-code tokens. The mixed layer-0 INT8 plus 17-layer + INT4 layout reduced task-macro excess NLL by 72.75% relative to uniform INT4 + while using exactly 2,564,096 resident recurrent-state bytes; every + preregistered quality gate passed. +- A compressed release attachment containing the retained raw confirmation + checkpoint for independent reconstruction of every reported gate. +- Unit coverage for packing parity, byte accounting, cache integration, public + data split discipline, and the supported Qwen3.5 factory path. + +### Changed + +- Scale-storage emulation now uses the declared physical FP16 or FP32 format; + superseding diagnostic results retain the earlier record instead of rewriting + it. +- The Transformers dependency is pinned to exactly `5.14.1` while the alpha + package depends on that release's internal linear-attention cache contract. +- FP16 scale storage is identified as the evaluated default. FP32 scale storage + remains supported for experiments but is not covered by full-model evidence. + +### Fixed + +- Beam-cache reordering now permutes packed INT4/INT8 payloads and scales + directly instead of dequantizing and requantizing the recurrent state. + +### Known limitations + +- The current Python path materializes one recurrent state while its layer + executes. It does not establish faster inference, lower whole-model memory, + or lower peak CUDA memory. +- Full-model evidence is currently limited to the pinned + `Qwen/Qwen3.5-0.8B-Base` revision and the environment described in + [docs/compatibility.md](docs/compatibility.md). +- The held-out result is limited to teacher-forced likelihood on the pinned + Qwen3.5 checkpoint and MBPP construction. It does not establish generated-code + correctness, novelty, cross-model generality, or a breakthrough. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..7c1b2dd --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,11 @@ +cff-version: 1.2.0 +message: "If you use RecurQuant, cite this software repository." +title: RecurQuant +type: software +authors: + - name: "Muhammad Labeeb Aryan" +version: 0.2.0a1 +date-released: 2026-07-22 +repository-code: "https://github.com/Labeeb2339/recurquant" +url: "https://github.com/Labeeb2339/recurquant" +license: Apache-2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..595b157 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,65 @@ +# Contributing to RecurQuant + +RecurQuant is an alpha research package with a deliberately narrow supported +surface. Contributions are most useful when they include a small reproducer, a +regression test, and an explicit statement of what the result does **not** show. + +## Set up and check a change + +The repository baseline is Python 3.11. On Windows, the same setup used by the +project is: + +```powershell +uv venv --python 3.11 .venv +uv pip install --python .venv\Scripts\python.exe -e ".[dev,eval]" +.venv\Scripts\python.exe -m pytest +.venv\Scripts\python.exe -m ruff check . +``` + +On Linux or macOS, replace `.venv\Scripts\python.exe` with +`.venv/bin/python` in the last three commands. + +The full-model experiments are separate from the unit suite and download public +model weights. Do not run them merely to validate a small code change. If a +change affects Qwen3.5 integration, also run the smallest relevant smoke path +documented in [compatibility and support](docs/compatibility.md), and report the +exact model revision, device, dtype, attention implementation, and Transformers +version. + +Before submitting a change: + +- add or update a focused test for changed behavior; +- run the complete unit suite and Ruff command above; +- keep generated evidence separate from hand-written interpretation; and +- do not include access tokens, authentication files, private prompts, + proprietary model data, or local machine secrets in code, logs, artifacts, or + issues. + +## Evidence and claim boundary + +Use the terminology in [the project claim boundary](research/CLAIM_BOUNDARY.md). +In particular: + +- report packed **resident recurrent-state bytes** separately from model + weights, attention KV caches, activations, allocator overhead, and the one + state materialized during a layer call; +- do not turn byte accounting into a speed, latency, peak-CUDA-memory, or + whole-model-memory claim; +- do not describe recurrent-state quantization itself as new, and do not call a + result a breakthrough from a diagnostic trace or a development split; +- attach the command, pinned revisions, environment, metric definition, and + machine-readable artifact for a new numerical claim; and +- preserve failed gates and negative results instead of silently replacing + them. + +The current supported package boundary is recorded in +[docs/compatibility.md](docs/compatibility.md). A new model, Transformers +release, execution backend, generation mode, or hardware target is unsupported +until it has a regression test and clearly scoped full-model evidence. + +## Keep pull requests reviewable + +Keep each change to one testable behavior. In the description, state the +problem, the exact verification commands, any new evidence file, and the claim +boundary affected. Documentation-only corrections should identify the source +artifact or code path that makes the replacement wording factual. diff --git a/README.md b/README.md index 4131767..5bb72fd 100644 --- a/README.md +++ b/README.md @@ -1,115 +1,259 @@ -# RecurQuant - -RecurQuant is a reproducible research harness for **persistent recurrent-state -quantization in Gated DeltaNet language models**. - -> **Current status:** the frozen diagnostic candidate passed its untouched -> confirmation trace. This is not yet a validated general method, realized -> memory reduction, speedup, or novelty result. - -The first calibration/development pilot found substantial layer heterogeneity. -At a 4.22-bit average payload, retaining only Gated DeltaNet layer 0 at INT8 and -using INT4 for the other 17 layers reduced worst-5% token KL by 79.8% on a -retrieval-style trace and 62.2% on a code-style trace relative to uniform INT4. -On the untouched multilingual trace, the same frozen plan reduced worst-5% -token KL by 77.8% and increased top-1 agreement from 25.0% to 59.4%. These are -short synthetic traces, not a benchmark or generalization result. - -## Research question - -Can sub-8-bit storage of Gated DeltaNet's fixed recurrent matrix state allocate -precision from query-weighted read sensitivity to preserve difficult -long-context behavior better than uniform quantization at the same modeled bit -budget? - -[Qwen3.5-0.8B-Base](https://huggingface.co/Qwen/Qwen3.5-0.8B-Base) is the first -target. Its language model repeats three Gated DeltaNet layers followed by one -full-attention layer, giving 18 persistent recurrent states and six ordinary KV -caches across 24 layers. - -## What this repository measures - -- Deterministic grouped INT8, INT6, and INT4 state round trips. -- Per-layer state size and numerical error. -- Paired token-level KL divergence and top-1 agreement against an FP32-state run. -- Tail error rather than only average perplexity. -- State-update magnitude for later sensitivity analysis. -- Query-weighted recurrent-read error, which measures the effect of state error - on the actual `q^T S` read. -- Modeled storage, with scale overhead included. - -The current simulator dequantizes states before the next model step. It can test -quality and error accumulation, but it **does not reduce PyTorch memory or prove -faster inference**. Those require a packed runtime kernel after the numerical -hypothesis passes. +

+ RecurQuant — packed recurrent state for Qwen3.5 +

-## Claim boundary +

+ CI status + Apache-2.0 license + Python 3.11 or newer + Transformers 5.14.1 + Open in Colab +

+ +

+ Quickstart · + Storage · + Evidence · + Compatibility · + Reproduce +

+ +RecurQuant is an alpha Python package that physically packs the persistent +recurrent matrix states used by Qwen3.5 Gated DeltaNet layers. Pass its cache to +ordinary eager Transformers model calls to keep those states as grouped INT4 or +INT8 payloads between calls. -Quantizing recurrent states is not new. Existing SSM work includes -[Quamba2](https://arxiv.org/abs/2503.22879), while newer systems use quantized -state checkpoints, stochastic rounding, and replay. Gated DeltaNet work also -uses update residuals to manage auxiliary memory. RecurQuant therefore does not -claim to be the first recurrent-cache quantizer, update-aware memory method, or -state-replay system. +Its frozen v0.2 layout passed a 500-task held-out MBPP teacher-forced fidelity +protocol: task-macro excess NLL was 72.75% lower than uniform INT4 while the +persistent recurrent state occupied exactly 2,564,096 resident bytes. -The narrower hypothesis under investigation is **precision allocation for the -persistent Gated DeltaNet matrix state**, conditioned on Gated DeltaNet dynamics -and compared at an equal bit budget. See -[the claim boundary](research/CLAIM_BOUNDARY.md) and -[pilot protocol](research/PILOT_PROTOCOL.md). The documented experiment trail -preserves the [failed signals and replacement](research/EXPERIMENT_001_SIGNAL_PIVOT.md) -and the [untouched confirmation](research/CONFIRMATION_001.md). +It currently targets +[`Qwen/Qwen3.5-0.8B-Base`](https://huggingface.co/Qwen/Qwen3.5-0.8B-Base). +RecurQuant does not quantize model weights or ordinary attention KV caches, and +its current Python path dequantizes one recurrent state while that layer runs. -[Gated DeltaNet-2](https://arxiv.org/abs/2605.22791) reinforces why erase, -write, and decay behavior should be analyzed separately. Its non-commercial -reference code is not a RecurQuant dependency, and the initial target remains -Apache-2.0 Qwen3.5. +I built and maintain RecurQuant as an open research project. — +[Muhammad Labeeb Aryan](https://github.com/Labeeb2339). Licensed under +[Apache-2.0](LICENSE). -## Local setup +## 60-second setup -Windows with an NVIDIA GPU: +The commands are short; the first run still needs to download the pinned model +and tokenizer. Python 3.11 and a CUDA GPU are recommended for the evaluated +path. + +Windows PowerShell: ```powershell -uv venv --python 3.11 .venv -uv pip install --python .venv\Scripts\python.exe torch --index-url https://download.pytorch.org/whl/cu128 -uv pip install --python .venv\Scripts\python.exe -e ".[dev]" -.venv\Scripts\python.exe -m pytest -.venv\Scripts\recurquant.exe demo --bits 4 --group-size 128 +git clone --branch v0.2.0a1 --depth 1 https://github.com/Labeeb2339/recurquant.git +cd recurquant +py -3.11 -m venv .venv +.\.venv\Scripts\python.exe -m pip install . +.\.venv\Scripts\recurquant.exe qwen35 --max-new-tokens 16 +``` + +macOS or Linux: + +```bash +git clone --branch v0.2.0a1 --depth 1 https://github.com/Labeeb2339/recurquant.git +cd recurquant +python3.11 -m venv .venv +.venv/bin/python -m pip install . +.venv/bin/recurquant qwen35 --max-new-tokens 16 ``` -The model experiment is intentionally separate from the unit-test suite because -it downloads approximately 1.75 GB of public model weights. +To verify the installation without downloading a model, run `recurquant demo` +with the platform-specific executable path above. It performs a deterministic +synthetic state round-trip and reports physical payload bytes, compression +ratio, and quantization error. -## Reproduce the frozen confirmation +The installed command and +[`examples/qwen35_quickstart.py`](examples/qwen35_quickstart.py) call the same +implementation. The default is the frozen v0.2 mixed policy: layer 0 at INT8 +and the remaining recurrent layers at INT4. Uniform INT4 is retained only as an +explicit stress baseline via `--policy uniform-int4-stress`. Add `--json` for +one machine-readable result containing the generated text, pinned model +provenance, selected policy, and raw storage counters. Read the +[compatibility contract](docs/compatibility.md) before using a different model, +Transformers version, device layout, or generation mode. -The script pins the model revision and records the environment, token digest, -state layout, metrics, and canonical evidence hash: +## Use it in Python -```powershell -.venv\Scripts\python.exe scripts\run_qwen35_smoke.py ` - --upgrade-layers 0 --low-bits 4 --high-bits 8 ` - --group-size 128 --rounding nearest ` - --prefill-tokens 32 --decode-tokens 32 ` - --prompt-profile multilingual ` - --output artifacts\multilingual-confirmation.json +This example uses the reusable frozen v0.2 helper, which keeps Gated DeltaNet +layer 0 at INT8 and the other 17 recurrent layers at INT4. The generic +`create_qwen35_packed_cache()` factory remains available for controlled policy +experiments. + +```python +import warnings + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from recurquant import create_qwen35_v02_mixed_cache + +MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" +REVISION = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +if device.type != "cuda": + dtype = torch.float32 +elif torch.cuda.is_bf16_supported(): + dtype = torch.bfloat16 +else: + warnings.warn( + "CUDA BF16 is unavailable; falling back to FP16. RecurQuant's public " + "full-model fidelity evidence has not been validated for FP16 weights.", + RuntimeWarning, + stacklevel=2, + ) + dtype = torch.float16 + +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, revision=REVISION) +model = AutoModelForCausalLM.from_pretrained( + MODEL_ID, + revision=REVISION, + dtype=dtype, + attn_implementation="eager", +).to(device) +model.eval() + +cache = create_qwen35_v02_mixed_cache(model) +inputs = tokenizer("Explain recurrent-state quantization simply.", return_tensors="pt") +inputs = inputs.to(device) +continuation = [] + +with torch.inference_mode(): + output = model(**inputs, past_key_values=cache, use_cache=True) + for step in range(32): + next_token = output.logits[:, -1, :].argmax(dim=-1, keepdim=True) + continuation.append(next_token) + reached_eos = ( + tokenizer.eos_token_id is not None + and bool((next_token == tokenizer.eos_token_id).all().item()) + ) + if reached_eos or step == 31: + break + output = model(input_ids=next_token, past_key_values=cache, use_cache=True) + +generated_ids = torch.cat(continuation, dim=1) +print(tokenizer.decode(generated_ids[0], skip_special_tokens=True)) +print(cache.storage_summary()) ``` -This reruns the already disclosed confirmation profile; it is a reproducibility -check, not a new held-out test. The recorded result and its limitations are in -[Confirmation 001](research/CONFIRMATION_001.md). +`create_qwen35_v02_mixed_cache()` and `create_qwen35_packed_cache()` reject +unsupported Transformers versions, non-eager attention, training mode, +multi-device placement, and incompatible Qwen configurations early. The +returned cache exposes exact live tensor byte accounting through +`storage_summary()`. + +## What is physically smaller + +For batch-one Qwen3.5-0.8B-Base recurrent states, the frozen mixed layout stores +2,564,096 resident bytes instead of 18,874,368 FP32-state bytes. Uniform INT4 +stores 2,433,024 bytes. These figures include packed payloads, FP16 group +scales, and padding. + +![Horizontal bars showing 18,874,368 bytes for FP32 recurrent states, 2,433,024 for uniform INT4, and 2,564,096 for the mixed layer-0 INT8 layout.](assets/recurrent-state-storage.svg) + +This is recurrent-state storage only. It is not a whole-model, peak-CUDA-memory, +latency, or throughput result. + +## Held-out confirmation + +The frozen layer-0 mixed layout passed every preregistered v0.2 quality and +integrity gate on the untouched MBPP test split. Task-macro excess negative +log-likelihood above the FP32-state reference fell from `2.949743` for uniform +INT4 to `0.803713`: a **72.75% reduction**. + +![Horizontal bars showing held-out MBPP task-macro excess NLL of 2.9497 for uniform INT4 and 0.8037 for the mixed layer-0 INT8 layout.](assets/mbpp-confirmation-fidelity.svg) + +The confirmation covers 500 paired tasks and 30,244 teacher-forced reference- +code tokens. The paired mixed-versus-uniform improvement was `2.1460` +nats/token with a 95% bootstrap interval of `[2.0922, 2.1999]`. Against the +mean of three exactly same-byte random high-precision layer placements, the +paired improvement was `2.0332` with a 95% interval of `[1.9802, 2.0861]`. + +| Token-weighted measure | Uniform INT4 | Mixed L0 INT8 | +|---|---:|---:| +| Mean KL | 3.149969 | 0.914580 | +| Worst-5% KL | 9.002207 | 4.839139 | +| Top-1 agreement | 0.321155 | 0.665190 | + +The earlier 90-task development result was a 74.14% reduction; it remains +available in +[`evidence/mbpp-v02-development.json`](evidence/mbpp-v02-development.json) and +[`DEVELOPMENT_002.md`](research/DEVELOPMENT_002.md). + +Important boundaries: + +- The accepted result is in + [`evidence/mbpp-v02-confirmation.json`](evidence/mbpp-v02-confirmation.json), + with the full decision and interruption record in + [`CONFIRMATION_002.md`](research/CONFIRMATION_002.md). +- Tokens were scored teacher-forced. Candidate-generated code was not fed back, + executed, or graded for correctness. +- The MSE selector also chose layer 0, so it is the exact same candidate—not + independent evidence that the read-risk selector is novel or superior. +- This supports one pinned recurrent-state fidelity and resident-byte result. + It does not support generated-code quality, speed, peak memory, whole-model + memory, cross-model generality, or a breakthrough claim. + +## Scope + +The supported public surface is deliberately narrow: + +- Python `>=3.11` and exactly `transformers==5.14.1` for this alpha; +- text-only Qwen3.5 hybrid models with `linear_attention` and `full_attention` + layer types; +- physical INT4 or INT8 recurrent-state payloads; FP16 scales are the evaluated + default, while FP32 scales are supported as an experimental, unevaluated + option; +- eager, evaluation-only, single-device inference; and +- explicit `past_key_values=cache` model calls. + +See [`docs/compatibility.md`](docs/compatibility.md) for the validated software, +hardware, model revision, generation paths, and unsupported modes. + +## Claim boundary + +Quantizing recurrent state is not new. I built RecurQuant to test one narrower +question: can sensitivity-guided mixed precision preserve Gated DeltaNet +recurrent-state fidelity better than simple equal-byte placements? For the +pinned Qwen3.5-0.8B-Base teacher-forced MBPP protocol, the frozen policy passed +the held-out confirmation and beat all three tested same-byte random placements. + +That is a confirmed case study, not proof of novelty or general superiority. +Q-Mamba already studies 4-bit persistent Mamba2 states, Quamba2 quantizes cached +SSM states, and other mixed-precision and replay systems overlap parts of this +design space. RecurQuant currently has no fused packed kernel or measured speed +claim. I therefore do not present this release as a breakthrough, a whole-model +memory reduction, or a cross-model result. See the +[claim boundary](research/CLAIM_BOUNDARY.md) and +[prior-art review](research/PRIOR_ART.md) for the exact comparison. -## Research discipline +## Research record -- Model and tokenizer revisions are pinned in evidence artifacts. -- Calibration, development, and confirmation prompts must remain separate. -- Static baselines run before any adaptive policy is tuned. -- Simple averages of decay, write, update norm, and residual magnitude are kept - as negative pilot evidence; they did not predict layer sensitivity reliably. -- Real latency is reported only after a packed kernel exists. -- Failed gates and negative results remain visible. -- Derived checkpoints must retain the base model's name, license, and lineage. +- Independently validate the held-out decision with + `recurquant verify-confirmation`; the + [reproduction guide](docs/reproducing.md) pins both committed hashes and + explains optional raw-checkpoint reconstruction. +- [Held-out MBPP confirmation report](research/CONFIRMATION_002.md) and + [machine-readable evidence](evidence/mbpp-v02-confirmation.json) +- [Frozen public evaluation protocol](research/PUBLIC_EVAL_PROTOCOL_V02.md) +- [MBPP development report](research/DEVELOPMENT_002.md) +- [Current research status](research/STATUS.md) +- [Claim boundary](research/CLAIM_BOUNDARY.md) and + [prior-art review](research/PRIOR_ART.md) +- [Failed proxy signals and empirical-sensitivity pivot](research/EXPERIMENT_001_SIGNAL_PIVOT.md) +- [Scale-format correction and packed/QDQ parity](research/EXPERIMENT_002_SCALE_CORRECTION.md) +- [Earlier pilot protocol](research/PILOT_PROTOCOL.md) and + [v0.1 diagnostic confirmation archive](research/CONFIRMATION_001.md) -## License +## Contributing -RecurQuant code is licensed under Apache-2.0. Referenced papers, models, datasets, -and repositories retain their own licenses. +I welcome reproducible compatibility reports, additional model-family adapters, +and work toward a fused packed recurrent kernel. Open an +[issue](https://github.com/Labeeb2339/recurquant/issues) with a minimal +reproducer and `cache.storage_summary()`; never include access tokens, private +prompts, or authentication files. diff --git a/assets/diagnostic-tail-kl.svg b/assets/diagnostic-tail-kl.svg new file mode 100644 index 0000000..83c7fd1 --- /dev/null +++ b/assets/diagnostic-tail-kl.svg @@ -0,0 +1,84 @@ + + + Corrected diagnostic tail KL by recurrent-state precision policy + On short diagnostic traces, uniform INT4 versus layer zero INT8 and the remaining layers INT4 produced CVaR95 KL of 7.2744 versus 1.2270 for retrieval, 5.4844 versus 2.0482 for code, 5.7644 versus 1.4277 for multilingual replay. Lower is better. These diagnostics are not a public benchmark or breakthrough evidence. + {"canonical_evidence_sha256":{"code_sweep":"cbac481cd90c99e30bf550a8c6026e680471480a99ff60600bbae57daa4b9776","multilingual_packed":"62ac233fd51dcbb722850593dfbbb3c1bec54e189c3f8e3812d1aeaa251b3523","multilingual_qdq":"38522abbd9b8fc593ee2b02d886f9a48a22d3c589d049d256fb8235800198931","retrieval_sweep":"a077f18867c69e5581b4dd003e4e7ed6d03f9843720fbbe2e1522f837518f1de"},"chart":"corrected-diagnostic-cvar95-kl","generated_by":"scripts/generate_readme_assets.py","source_document":"research/EXPERIMENT_002_SCALE_CORRECTION.md","source_document_sha256":"71df981f2f572206799516a2f6ce1c5ed7027bc652763bdea23e5cc47814b7dc","values":[{"label":"Retrieval","mixed":"1.2270","reduction":"83.1%","uniform":"7.2744"},{"label":"Code","mixed":"2.0482","reduction":"62.7%","uniform":"5.4844"},{"label":"Multilingual replay","mixed":"1.4277","reduction":"75.2%","uniform":"5.7644"}]} + + Corrected diagnostic tail KL + CVaR95 token KL · stored FP16 scales · lower is better + + Uniform INT4 + + L0 INT8, rest INT4 + + 0 + + 2 + + 4 + + 6 + + 8 + + + Retrieval, uniform INT4: 7.2744 CVaR95 KL + + 7.2744 + + Retrieval, mixed layer zero INT8: 1.2270 CVaR95 KL + + 1.2270 + Retrieval + −83.1% mixed vs uniform + + + + Code, uniform INT4: 5.4844 CVaR95 KL + + 5.4844 + + Code, mixed layer zero INT8: 2.0482 CVaR95 KL + + 2.0482 + Code + −62.7% mixed vs uniform + + + + Multilingual replay, uniform INT4: 5.7644 CVaR95 KL + + 5.7644 + + Multilingual replay, mixed layer zero INT8: 1.4277 CVaR95 KL + + 1.4277 + Multilingual replay + −75.2% mixed vs uniform + + CVaR95 KL + Short paired diagnostics, not a public benchmark or breakthrough claim. + Multilingual is a correction replay; packed and QDQ token metrics matched exactly. + diff --git a/assets/mbpp-confirmation-fidelity.svg b/assets/mbpp-confirmation-fidelity.svg new file mode 100644 index 0000000..e7aadc1 --- /dev/null +++ b/assets/mbpp-confirmation-fidelity.svg @@ -0,0 +1,60 @@ + + + MBPP held-out confirmation task-macro excess negative log likelihood + On the held-out MBPP confirmation split, uniform INT4 had task-macro excess negative log likelihood 2.949743, while keeping recurrent layer zero at INT8 and the other 17 recurrent layers at INT4 had 0.803713, a 72.75 percent reduction. Across 500 paired tasks, the mean improvement was 2.146030 nats per token with a 95 percent bootstrap interval from 2.092249 to 2.199866. This is teacher-forced fidelity evidence over 30244 reference code tokens. Generated code was not executed, and the evidence supports no speed or whole-model memory claim. + {"all_quality_gates_pass":true,"canonical_evidence_sha256":"2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0","chart":"mbpp-confirmation-task-macro-excess-nll","confirmation_touched":true,"exact_storage":{"fp32_recurrent_reference_bytes":18874368,"mixed_resident_bytes":2564096,"uniform_int4_resident_bytes":2433024},"generated_by":"scripts/generate_readme_assets.py","generated_code_executed":false,"mse_upgrade_layer":0,"phase":"confirmation","primary_macro_delta_nll":0.8037128749489785,"primary_upgrade_layer":0,"primary_vs_mean_random_equal_byte":{"bootstrap_samples":10000,"confidence":0.95,"confidence_interval":[1.9802198561151823,2.086135943630834],"mean_improvement":2.03316749749581,"paired_examples":500,"seed":2339},"primary_vs_uniform":{"bootstrap_samples":10000,"confidence":0.95,"confidence_interval":[2.0922485922992227,2.199865858581662],"mean_improvement":2.146029668748379,"paired_examples":500,"seed":2339},"protocol_eligible":true,"relative_reduction":0.727531178384957,"source_document":"evidence/mbpp-v02-confirmation.json","source_document_sha256":"70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9","speed_claim_allowed":false,"task_count":500,"teacher_forced_fidelity_only":true,"token_count":30244,"uniform_macro_delta_nll":2.949742543697357,"whole_model_memory_claim_allowed":false} + + MBPP held-out confirmation fidelity + Task-macro excess NLL (nats/token) · teacher-forced · lower is better + + 0 + + 1 + + 2 + + 3 + + Uniform INT4 + all 18 recurrent layers INT4 + + Uniform INT4: 2.949743 task-macro excess NLL + + 2.9497 + + + Mixed L0 INT8 + layer 0 INT8; other 17 layers INT4 + + Mixed L0 INT8: 0.803713 task-macro excess NLL + + 0.8037 + + 72.75% lower macro excess NLL + Paired mixed-vs-uniform improvement: 2.1460; 95% bootstrap CI 2.0922–2.1999. + Equal-byte mixed-vs-mean-random improvement: 2.0332; 95% bootstrap CI 1.9802–2.0861. + Read-risk and MSE selectors both chose recurrent layer 0. + Exact resident recurrent-state bytes: mixed 2,564,096; uniform INT4 2,433,024; FP32 reference 18,874,368. + Held-out MBPP confirmation · 500 tasks · 30,244 teacher-forced reference code tokens. + Generated code was not executed; no speed or whole-model memory claim. + diff --git a/assets/mbpp-development-fidelity.svg b/assets/mbpp-development-fidelity.svg new file mode 100644 index 0000000..b25813a --- /dev/null +++ b/assets/mbpp-development-fidelity.svg @@ -0,0 +1,58 @@ + + + MBPP development task-macro excess negative log likelihood + On the MBPP validation development split, uniform INT4 had task-macro excess negative log likelihood 2.964469, while keeping recurrent layer zero at INT8 and the other 17 recurrent layers at INT4 had 0.766489, a 74.14 percent reduction. Across 90 paired tasks, the mean improvement was 2.197980 nats per token with a 95 percent bootstrap interval from 2.080779 to 2.321465. This is teacher-forced development evidence over 5524 reference code tokens, not confirmation or generated-code correctness evidence. + {"canonical_evidence_sha256":"301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488","chart":"mbpp-development-task-macro-excess-nll","confirmation_touched":false,"generated_by":"scripts/generate_readme_assets.py","generated_code_executed":false,"mse_upgrade_layer":0,"phase":"development","primary_macro_delta_nll":0.7664886196454366,"primary_upgrade_layer":0,"primary_vs_mean_random_equal_byte":{"bootstrap_samples":10000,"confidence":0.95,"confidence_interval":[1.965983425400875,2.194512400218734],"mean_improvement":2.078384230534236,"paired_examples":90,"seed":2339},"primary_vs_uniform":{"bootstrap_samples":10000,"confidence":0.95,"confidence_interval":[2.0807794316444133,2.3214647861652904],"mean_improvement":2.197980460855696,"paired_examples":90,"seed":2339},"relative_reduction":0.7414415199379093,"source_document":"evidence/mbpp-v02-development.json","source_document_sha256":"5980fd58aa0933ad97deb896d4901fcd37350c4a57d8a80022ab218aaf77e727","speed_claim_allowed":false,"task_count":90,"teacher_forced_fidelity_only":true,"token_count":5524,"uniform_macro_delta_nll":2.9644690805011327,"whole_model_memory_claim_allowed":false} + + MBPP development fidelity + Task-macro excess NLL (nats/token) · lower is better + + 0 + + 1 + + 2 + + 3 + + Uniform INT4 + all 18 recurrent layers INT4 + + Uniform INT4: 2.964469 task-macro excess NLL + + 2.9645 + + + Mixed L0 INT8 + layer 0 INT8; other 17 layers INT4 + + Mixed L0 INT8: 0.766489 task-macro excess NLL + + 0.7665 + + 74.14% lower macro excess NLL + Paired mixed-vs-uniform improvement: 2.1980; 95% bootstrap CI 2.0808–2.3215. + Equal-byte mixed-vs-mean-random improvement: 2.0784; 95% bootstrap CI 1.9660–2.1945. + MBPP validation development split · 90 tasks · 5,524 teacher-forced reference code tokens. + Development evidence only; no confirmation, generated-code correctness, speed, or whole-model memory claim. + diff --git a/assets/recurquant-hero.png b/assets/recurquant-hero.png new file mode 100644 index 0000000..c83db92 Binary files /dev/null and b/assets/recurquant-hero.png differ diff --git a/assets/recurquant-social-preview.png b/assets/recurquant-social-preview.png new file mode 100644 index 0000000..5afb53d Binary files /dev/null and b/assets/recurquant-social-preview.png differ diff --git a/assets/recurrent-state-storage.svg b/assets/recurrent-state-storage.svg new file mode 100644 index 0000000..255c704 --- /dev/null +++ b/assets/recurrent-state-storage.svg @@ -0,0 +1,67 @@ + + + Resident recurrent-state storage on Qwen3.5-0.8B-Base + At batch one, FP32 recurrent states occupy 18,874,368 bytes. Uniform INT4 with FP16 scales occupies 2,433,024 bytes, 7.758 times smaller. Keeping layer zero at INT8 and the other 17 layers at INT4 occupies 2,564,096 bytes, 7.361 times smaller. These are persistent state bytes only, not whole-model or peak memory. + {"canonical_evidence_sha256":{"packed_multilingual_replay":"62ac233fd51dcbb722850593dfbbb3c1bec54e189c3f8e3812d1aeaa251b3523"},"chart":"resident-recurrent-state-bytes","generated_by":"scripts/generate_readme_assets.py","source_document":"research/EXPERIMENT_002_SCALE_CORRECTION.md","source_document_sha256":"71df981f2f572206799516a2f6ce1c5ed7027bc652763bdea23e5cc47814b7dc","values":[{"bytes":"18,874,368","detail":"reference","label":"FP32 states","ratio":"1.000x"},{"bytes":"2,433,024","detail":"FP16 scales","label":"Uniform INT4","ratio":"7.758x smaller"},{"bytes":"2,564,096","detail":"17 layers INT4 \u00b7 FP16 scales","label":"Mixed: L0 INT8","ratio":"7.361x smaller"}]} + + Resident recurrent-state storage + Qwen3.5-0.8B-Base · batch 1 · resident bytes (millions) · lower is better + + 0 + + 5 + + 10 + + 15 + + 20 + + FP32 states + reference · 1.000x reference + + FP32 states: 18,874,368 resident bytes; 1.000x reference + + 18,874,368 + + + Uniform INT4 + FP16 scales · 7.758x smaller + + Uniform INT4: 2,433,024 resident bytes; 7.758x smaller + + 2,433,024 + + + Mixed: L0 INT8 + 17 layers INT4 · FP16 scales · 7.361x smaller + + Mixed: L0 INT8: 2,564,096 resident bytes; 7.361x smaller + + 2,564,096 + + Persistent recurrent state only; excludes model weights and six standard KV caches. + The Python path materializes one layer state; no peak-memory or speed claim. + diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..159b01f --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,104 @@ +# Compatibility and support + +RecurQuant `0.2.0a1` is an alpha research package. The supported surface is +intentionally narrow because `PackedRecurrentStateCache` subclasses an internal +Transformers cache layer. A passing import or smoke test is not evidence of +model quality, speed, or production readiness. + +## Validated compatibility snapshot + +| Area | Current evidence | Support boundary | +|---|---|---| +| Python | The full MBPP development and confirmation runs used Python `3.11.15`. Packaging CI is configured to smoke-test Python `3.11` and `3.13` on Linux, plus Python `3.11` on Windows. | Python `>=3.11` is declared. Full-model numerical evidence is limited to `3.11.15`. | +| Transformers | `5.14.1` | The alpha dependency is deliberately pinned to `transformers==5.14.1`. Every other release is unsupported until its internal cache contract is tested. | +| Model | `Qwen/Qwen3.5-0.8B-Base` at revision `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68` | No other checkpoint, revision, model family, or recurrent architecture has full-model evidence. | +| Model execution | BF16 weights, batch size one, evaluation mode, `trust_remote_code=False`, eager attention | Use `attn_implementation="eager"`. Flash, SDPA, and other attention implementations are not validated. | +| Full-run environment | PyTorch `2.11.0+cu128`, CUDA runtime `12.8`, NVIDIA driver `592.15`, NVIDIA GeForce RTX 5070 Laptop GPU, recorded platform `Windows-10-10.0.26200-SP0` | CPU and other accelerator support is limited to unit or API smoke tests; the public numerical result was not replicated there. | +| Packed formats | Physical INT4 and INT8 recurrent-state payloads. FP16 scales are the evaluated default. | FP32 scales are supported for experiments but have no full-model fidelity evidence. The packed cache does not accept other payload widths. | + +The latest full-model evidence and exact provenance are recorded in +[`evidence/mbpp-v02-confirmation.json`](../evidence/mbpp-v02-confirmation.json), +with the human-readable result and claim boundary in +[`research/CONFIRMATION_002.md`](../research/CONFIRMATION_002.md). The earlier +development record remains available in +[`evidence/mbpp-v02-development.json`](../evidence/mbpp-v02-development.json). + +## Installed quickstart + +`recurquant qwen35` and `examples/qwen35_quickstart.py` use one shared +implementation. Both default to the frozen v0.2 mixed policy: model layer 0 at +INT8 and every other recurrent layer at INT4, with group size 128 and FP16 +scales. `--policy uniform-int4-stress` is retained only for reproducing the +uniform INT4 stress baseline. + +The command downloads the pinned model and tokenizer unless +`--local-files-only` is supplied. It performs manual greedy decoding; the +full-checkpoint quality of free-running generations has not been evaluated. + +## Generation and cache modes + +The following paths have direct evidence: + +- Explicit eager `model(...)` calls with + `past_key_values=PackedRecurrentStateCache(...)` and `use_cache=True`. +- Batch-one prompt prefill followed by one-token teacher-forced decode calls on + the pinned full model across all 500 frozen MBPP confirmation tasks and + 30,244 scored reference-code tokens. +- Prefill plus a short multi-token continuation on a tiny randomly initialized + Qwen3.5 configuration in the unit suite. +- Batch-one greedy generation and beam search through `model.generate(...)` on + a tiny randomly initialized Qwen3.5 CPU configuration. These are integration + smoke tests only; free-running output quality was not evaluated. + +The local generation smoke used Python `3.11.15`, PyTorch `2.13.0+cpu`, and +Transformers `5.14.1`. It generated three new tokens in greedy and two-beam +modes without downloading model weights. + +The release-candidate full-checkpoint smoke used the pinned Qwen3.5 revision, +Python `3.11.15`, PyTorch `2.11.0+cu128`, BF16 weights, and eager CUDA decoding. +It generated a two-token continuation through the installed `recurquant qwen35` +path and reported 2,564,096 resident bytes, 18,874,368 FP32-reference bytes, a +1,048,576-byte largest single state materialization, and physical reduction +realized. This is a functional integration check, not a quality or latency +benchmark. + +The following paths are unsupported or not yet validated: + +- sampling, beam sampling, diverse or constrained beam search, assisted or + speculative decoding, and externally batched generation; +- training, gradient-based use, distributed or sharded inference, cache + serialization, and cache offload/prefetch workflows; +- non-eager attention implementations; and +- `torch.compile`. Packed cache layers explicitly set `is_compileable=False`. + +An unsupported mode may appear to run, but it is outside the alpha compatibility +contract until a regression test and a full-model check are added. + +## Memory and performance boundary + +RecurQuant measures the live tensor bytes used by packed persistent recurrent +states. The current Python implementation dequantizes one recurrent state while +its layer executes. It therefore makes **no claim** of: + +- faster inference or lower latency; +- lower whole-model or peak CUDA memory; +- a fully quantized recurrent kernel; or +- improved generated-code correctness. + +The reported compression ratio applies only to resident recurrent-state +payloads and scales, not model weights, ordinary attention KV caches, +activations, allocator overhead, or transient materialization. + +## Reporting compatibility problems + +Open an issue at with: + +1. the RecurQuant, Python, PyTorch, and Transformers versions; +2. the exact model ID and revision; +3. device, dtype, attention implementation, batch size, and generation mode; +4. a minimal reproducer and complete traceback; and +5. `cache.storage_summary()` when the failure concerns byte accounting. + +Do not include access tokens, local authentication files, private prompts, or +proprietary model data. Because the package is pre-release, compatibility may +change between development versions and such changes will be documented. diff --git a/docs/reproducing.md b/docs/reproducing.md new file mode 100644 index 0000000..c957fec --- /dev/null +++ b/docs/reproducing.md @@ -0,0 +1,356 @@ +# Reproducing and extending RecurQuant + +This guide separates three different activities: + +1. **Use the cache** to run the supported Qwen3.5 checkpoint. This checks that + the integration works and reports resident recurrent-state bytes; it does + not reproduce a quality result. +2. **Reproduce development evidence** on the 90-task MBPP validation split. + This repeats an already inspected development experiment. +3. **Replicate the completed confirmation** on the 500-task MBPP test split. + The published v0.2 confirmation had its policy, manifest, gates, and code + frozen before outcomes were computed. Now that those outcomes are public, + later runs are replications rather than new untouched confirmations. + +None of these workflows establishes generated-code correctness, lower latency, +lower whole-model or peak memory, cross-model generality, novelty, or a +breakthrough. + +## Frozen reference + +The v0.2 numerical evidence is scoped to: + +- Python `3.11.15`; +- PyTorch `2.11.0+cu128`, CUDA runtime `12.8`, and NVIDIA driver `592.15`; +- Transformers `5.14.1` and Datasets `4.8.5`; +- `Qwen/Qwen3.5-0.8B-Base` at revision + `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`; +- `google-research-datasets/mbpp`, configuration `full`, at revision + `4bb6404fdc6cacfda99d4ac4205087b89d32030c`; and +- batch one, eager attention, BF16 model weights on CUDA, evaluation mode, and + teacher-forced reference-code tokens. + +The accepted run used an NVIDIA GeForce RTX 5070 Laptop GPU with 8 GB VRAM. +The 90-task development matrix took 4,198.6 seconds, about 70 minutes. The +500-task confirmation matrix is a long, multi-hour run. CPU execution uses +FP32 model weights and may be substantially slower; it is supported by the +scripts but is not the environment behind the published numerical evidence. +Treat a different software or hardware environment as a replication rather +than a byte-for-byte repeat. + +Create the normal contributor environment from the repository root: + +```powershell +uv venv --python 3.11 .venv +uv pip install --python .venv\Scripts\python.exe -e ".[dev,eval]" +.venv\Scripts\python.exe -m pytest +.venv\Scripts\python.exe -m ruff check . +``` + +Linux or macOS: + +```bash +uv venv --python 3.11 .venv +uv pip install --python .venv/bin/python -e ".[dev,eval]" +.venv/bin/python -m pytest +.venv/bin/python -m ruff check . +``` + +The package pins Transformers `5.14.1`; the other exact reference versions +above are provenance pins, not the full range declared by `pyproject.toml`. +Record the versions actually resolved for a replication. PowerShell and Bash +commands are shown separately below so their path and continuation syntax can +be copied directly. + +The scripts download the pinned model and dataset by default. Add +`--local-files-only` only when the model and tokenizer are already cached. It +does not apply to the streamed MBPP dataset loader. + +## 1. Use the packed cache + +The installed command runs the pinned checkpoint with the frozen v0.2 layout: +model layer 0 at INT8 and the other 17 recurrent layers at INT4, group size 128, +FP16 scales, and nearest rounding. + +```powershell +.venv\Scripts\python.exe -m recurquant.cli qwen35 ` + --device cuda ` + --max-new-tokens 32 ` + --prompt "Explain recurrent-state quantization in two sentences." +``` + +```bash +.venv/bin/python -m recurquant.cli qwen35 \ + --device cuda \ + --max-new-tokens 32 \ + --prompt "Explain recurrent-state quantization in two sentences." +``` + +Use `--device auto` to select CUDA when available and CPU otherwise. The output +includes resident packed bytes, the full-precision recurrent-state equivalent, +the largest single materialized recurrent state, and the resident-state +compression ratio. That largest-state field is not a peak-workspace or peak +CUDA-memory measurement. + +For application code, use `create_qwen35_v02_mixed_cache(model)`. Use +`create_qwen35_packed_cache(...)` with `layer_specs` only for an explicitly +custom policy. The model must be in evaluation mode, loaded with eager +attention on one materialized device, and every forward must run inside +`torch.inference_mode()` or `torch.no_grad()`. See +[`compatibility.md`](compatibility.md) for unsupported generation and runtime +modes. + +## 2. Reproduce the development evidence + +The committed calibration and prepared manifest are sufficient to repeat the +frozen 90-task method matrix: + +```powershell +.venv\Scripts\python.exe scripts\evaluate_mbpp.py ` + --phase development ` + --calibration-artifact evidence\mbpp-v02-calibration.json ` + --prepared-manifest evidence\mbpp-v02-development-manifest.json ` + --device cuda ` + --checkpoint artifacts\replication-development.checkpoint.json ` + --output artifacts\replication-development.json +``` + +```bash +.venv/bin/python scripts/evaluate_mbpp.py \ + --phase development \ + --calibration-artifact evidence/mbpp-v02-calibration.json \ + --prepared-manifest evidence/mbpp-v02-development-manifest.json \ + --device cuda \ + --checkpoint artifacts/replication-development.checkpoint.json \ + --output artifacts/replication-development.json +``` + +To regenerate each input instead of trusting the committed copies, run the +full calibration and then prepare an outcome-free token manifest: + +```powershell +.venv\Scripts\python.exe scripts\calibrate_mbpp_layers.py ` + --device cuda ` + --output artifacts\replication-calibration.json + +.venv\Scripts\python.exe scripts\evaluate_mbpp.py ` + --phase development ` + --calibration-artifact artifacts\replication-calibration.json ` + --manifest-only ` + --device cuda ` + --output artifacts\replication-development-manifest.json +``` + +```bash +.venv/bin/python scripts/calibrate_mbpp_layers.py \ + --device cuda \ + --output artifacts/replication-calibration.json + +.venv/bin/python scripts/evaluate_mbpp.py \ + --phase development \ + --calibration-artifact artifacts/replication-calibration.json \ + --manifest-only \ + --device cuda \ + --output artifacts/replication-development-manifest.json +``` + +Then substitute those two artifact paths in the first command. A full +development or confirmation run requires `--prepared-manifest`. Do not use +`--limit`, change `--group-size 128` or `--bootstrap-samples 10000`, or add +`--skip-qdq-preflight` when claiming a protocol-eligible replication. The +evaluator rejects those changes for a full public run. + +## 3. Replicate the confirmation phase + +The explicit lock is an acknowledgement token, not a credential. It prevents +accidental access to the test split. Run this only after freezing your own +policy and analysis plan: + +```powershell +.venv\Scripts\python.exe scripts\evaluate_mbpp.py ` + --phase confirmation ` + --calibration-artifact evidence\mbpp-v02-calibration.json ` + --prepared-manifest evidence\mbpp-v02-confirmation-manifest.json ` + --confirmation-lock recurquant:unlock-mbpp-confirmation:rq-v0.2 ` + --device cuda ` + --checkpoint artifacts\replication-confirmation.checkpoint.json ` + --output artifacts\replication-confirmation.json +``` + +```bash +.venv/bin/python scripts/evaluate_mbpp.py \ + --phase confirmation \ + --calibration-artifact evidence/mbpp-v02-calibration.json \ + --prepared-manifest evidence/mbpp-v02-confirmation-manifest.json \ + --confirmation-lock recurquant:unlock-mbpp-confirmation:rq-v0.2 \ + --device cuda \ + --checkpoint artifacts/replication-confirmation.checkpoint.json \ + --output artifacts/replication-confirmation.json +``` + +The frozen v0.2 run completed all 500 tasks and 30,244 reference-code tokens. +Every preregistered quality gate passed. The complete result and its limitations +are recorded in +[`CONFIRMATION_002.md`](../research/CONFIRMATION_002.md); the machine-readable +artifact is +[`mbpp-v02-confirmation.json`](../evidence/mbpp-v02-confirmation.json). + +The accepted run resumed from atomic per-task checkpoints after infrastructure +interruptions. One attempted resume stalled before evaluation because the +streamed dataset loader had no network access, so it was stopped before the +checkpoint changed. A later process exited during an atomic checkpoint replace +after 370 accepted tasks; the preceding checkpoint remained intact and the +interrupted next task was recomputed. Every resume kept the same command, +source commit, prepared manifest, calibration artifact, candidate plan, and +gates. No partial candidate metric was inspected and no outcome-driven rerun +occurred. This is distinct from rerunning an unfavourable result, which the +protocol prohibits. + +## Manifests, checkpoints, and hashes + +Prepared manifests record the dataset rows and tokenization alongside the +model, dataset, calibration, candidate plan, and repository commit before a +long run. The evaluator recomputes and compares the phase, model revision, +dataset-manifest hash, token-manifest hash, and calibration-evidence hash. +Checkpoints are written atomically after each task. Their run signature binds +the phase, model revision, current repository commit, calibration evidence +hash, prepared-manifest hash, token-manifest hash, group size, and current +candidate plan. Resume by repeating the exact command. A checkpoint from +changed code or settings is rejected instead of silently reused. + +The committed integrity anchors are: + +| Artifact | Tasks | Source commit | File SHA256 | Canonical evidence SHA256 | +|---|---:|---|---|---| +| `evidence/mbpp-v02-calibration.json` | 128 | `cc35f4396ef4dd475908d8f96e05fe9c559f13be` | `d3d2f9acf6113ad455cce78d1b957a265c6675236845564d855c7cb537267125` | `7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee` | +| `evidence/mbpp-v02-development-manifest.json` | 90 | `3a3c4a2a11c0822f6c456a74327127f294ce67e1` | `7d51f732f9d0147c485d3bd3214e9bfcd16132de07df342f0c6b9696160aac3f` | `2b13dfb1799472b2fd0006cb87cea60c67a7af37825a87f1686a6fe3d8e38d7c` | +| `evidence/mbpp-v02-development.json` | 90 | `20a5ea95a8ed692600ee1645d2913f3a4b8a6795` | `5980fd58aa0933ad97deb896d4901fcd37350c4a57d8a80022ab218aaf77e727` | `301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488` | +| `evidence/mbpp-v02-confirmation-manifest.json` | 500 | `44d75a2776fa36441e17cc688965c9825c4c1a1c` | `c6a7d0db6ef7577a66ac19fbbc0be166279488f6a6be432b364bd9eb6833f7b0` | `21a6d18c6a0887b1499d156a3d610d4bfafdd59d3557713485b62038e263b96a` | +| `evidence/mbpp-v02-confirmation.json` | 500 | `6bd5bed2b61e192526ba8fdbec8232801cbea843` | `70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9` | `2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0` | + +The accepted confirmation checkpoint had file SHA256 +`df0040cc9cebdbc442992e75d19f9090456f9b249da062c095840e731b6c4609`, +canonical-state SHA256 +`1293d93cb620d2193e9251f49c05d0bdaeebde16d3515c1f0e021c96b5d4fe1c`, +and frozen run-signature SHA256 +`5d15268224357bb078315ef2c2b6e710a7eb8a2734df2527f9637b382951c78a`. + +Strictly verify the published confirmation without loading the model or +dataset: + +```powershell +.venv\Scripts\python.exe -m recurquant.cli verify-confirmation ` + evidence\mbpp-v02-confirmation.json ` + evidence\mbpp-v02-confirmation-manifest.json ` + --expect-artifact-sha256 70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9 ` + --expect-artifact-evidence-sha256 2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0 +``` + +```bash +.venv/bin/python -m recurquant.cli verify-confirmation \ + evidence/mbpp-v02-confirmation.json \ + evidence/mbpp-v02-confirmation-manifest.json \ + --expect-artifact-sha256 70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9 \ + --expect-artifact-evidence-sha256 2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0 +``` + +The committed-artifact check returns `result: pass`, +`artifact_manifest_verified: true`, and `outcome_verified: true`. The raw +checkpoint is excluded from Git because it expands to 34,359,541 bytes. It is +available as the 9,098,655-byte +[`v0.2.0a1` release attachment](https://github.com/Labeeb2339/recurquant/releases/download/v0.2.0a1/mbpp-v02-confirmation.checkpoint.json.zip), +whose archive SHA256 is +`fe2db8b54b0c4ae7f34f0e2b661ebd74e6134b79550c63567288c1d118d0432b`. +Without that file, the command above warns that token arrays were not +reconstructed. After extracting the archive, append +`--checkpoint mbpp-v02-confirmation.checkpoint.json` (using `\` paths in +PowerShell). The verifier checks the contained checkpoint SHA256 +`df0040cc9cebdbc442992e75d19f9090456f9b249da062c095840e731b6c4609`, +its canonical-state hash, and then reconstructs all summaries and gates from +the raw arrays. + +The generic artifact verifier remains available for the development record: + +```powershell +.venv\Scripts\python.exe -m recurquant.cli verify-artifact ` + evidence\mbpp-v02-development.json ` + --expect-file-sha256 5980fd58aa0933ad97deb896d4901fcd37350c4a57d8a80022ab218aaf77e727 ` + --expect-canonical-evidence-sha256 301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488 +``` + +```bash +.venv/bin/python -m recurquant.cli verify-artifact \ + evidence/mbpp-v02-development.json \ + --expect-file-sha256 5980fd58aa0933ad97deb896d4901fcd37350c4a57d8a80022ab218aaf77e727 \ + --expect-canonical-evidence-sha256 301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488 +``` + +The command exits nonzero if the JSON is malformed, the recorded canonical +hash is wrong, or either expected anchor does not match. Its report is JSON so +the same check can run in CI. + +Cross-machine invariant manifest hashes are: + +| Phase | Dataset manifest SHA256 | Token manifest SHA256 | +|---|---|---| +| Calibration | `129698d01bdf7f08989878ff7c980230456095c5b2cae42a010f567ecc49dc1a` | not produced by the calibration script | +| Development | `8fed3da0aae864f4e30c70ad70b0269f759d3592dccb9ab87f24444fa24d65dc` | `3c19f37f3c35cb22f17e66a5438bb9968b214e0aff64067930bc3a7124f63f5c` | +| Confirmation | `060aaff7117dc47af6c01253a912f34b6956241c336bbc7216e73bca8624d2d4` | `199a8836489af9bd0af3fec027e85d57df356bd9919492b24015de51d143f525` | + +`canonical_evidence_sha256` is SHA256 over `artifact["evidence"]` serialized +with sorted keys, two-space indentation, standard JSON ASCII escaping, +`allow_nan=False`, and one trailing newline. It protects the evidence payload +but includes environment and command provenance, so a legitimate replication +on another commit or machine need not match the accepted canonical hash. The +dataset and token manifest hashes should match when the frozen inputs and +tokenizer do. + +## Extend with a new policy + +Do not edit the committed v0.2 evidence files or reinterpret its confirmation. +Start with a separate diagnostic artifact. The existing smoke script can test +one or more promoted recurrent layers without touching MBPP: + +```powershell +.venv\Scripts\python.exe scripts\run_qwen35_smoke.py ` + --cache-mode packed ` + --low-bits 4 ` + --high-bits 8 ` + --upgrade-layers 6 ` + --prompt-profile code ` + --device cuda ` + --output artifacts\policy-layer6-diagnostic.json +``` + +```bash +.venv/bin/python scripts/run_qwen35_smoke.py \ + --cache-mode packed \ + --low-bits 4 \ + --high-bits 8 \ + --upgrade-layers 6 \ + --prompt-profile code \ + --device cuda \ + --output artifacts/policy-layer6-diagnostic.json +``` + +Use `--sensitivity-sweep` instead of `--upgrade-layers` to evaluate every Gated +DeltaNet layer on one of the script's synthetic prompt profiles. These traces +are diagnostics, not public-task or confirmation evidence. + +If a policy remains promising: + +1. add it through `layer_specs` and add focused unit/integration tests; +2. write a new versioned protocol with its own calibration, development, and + confirmation partitions or dataset; +3. freeze new candidate definitions and outcome-free manifests before scoring; +4. write artifacts to new versioned filenames rather than replacing anything + under `evidence/mbpp-v02-*`; and +5. report failed gates and negative controls with successful results. + +Changing `candidate_definitions()` in `scripts/evaluate_mbpp.py` creates a new +experiment; it does not extend the frozen v0.2 claim. Follow +[`CONTRIBUTING.md`](../CONTRIBUTING.md), keep the pull request to one testable +change, and include the exact command, environment, artifact, metric definition, +and claim boundary. Read the complete frozen design in +[`PUBLIC_EVAL_PROTOCOL_V02.md`](../research/PUBLIC_EVAL_PROTOCOL_V02.md) and the +current restrictions in [`CLAIM_BOUNDARY.md`](../research/CLAIM_BOUNDARY.md). diff --git a/evidence/mbpp-v02-calibration.json b/evidence/mbpp-v02-calibration.json new file mode 100644 index 0000000..d982ec7 --- /dev/null +++ b/evidence/mbpp-v02-calibration.json @@ -0,0 +1,1405 @@ +{ + "artifact_kind": "recurquant_mbpp_layer_calibration", + "canonical_evidence_sha256": "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee", + "created_at_utc": "2026-07-22T04:00:28.698473+00:00", + "evidence": { + "calibration": { + "averaging": "equal writes within task, then equal tasks", + "candidate_layers": { + "mse_selected_nearest": 0, + "random_l13_nearest": 13, + "random_l18_nearest": 18, + "random_l4_nearest": 4, + "read_risk_l0_nearest": 0 + }, + "diagnostic_read_risk_mean_by_layer": { + "0": 0.3976491566902005, + "1": 0.1401301747867153, + "10": 0.16849312830404745, + "12": 0.17481756236547222, + "13": 0.15792910501285837, + "14": 0.15042250811392754, + "16": 0.21367933678936848, + "17": 0.22023953422198478, + "18": 0.18419015096258548, + "2": 0.16083438805662986, + "20": 0.17656315345218743, + "21": 0.17779583551245184, + "22": 0.20882127337948972, + "4": 0.13984541433292125, + "5": 0.22826206120144538, + "6": 0.24193228224636368, + "8": 0.13853116560656015, + "9": 0.1755444030371019 + }, + "diagnostic_read_risk_selected_layer": 0, + "group_size": 128, + "normalized_state_mse_mean_by_layer": { + "0": 0.17529198000467544, + "1": 0.025073729904113166, + "10": 0.03510374119450758, + "12": 0.06727648370126754, + "13": 0.03600873059675213, + "14": 0.05937964344148206, + "16": 0.09039201018749285, + "17": 0.10547068898785164, + "18": 0.09114767145793809, + "2": 0.04240674980197983, + "20": 0.08030832431877885, + "21": 0.08185885639637328, + "22": 0.08002353595615602, + "4": 0.058194447097179726, + "5": 0.0629551427194282, + "6": 0.05372850340249499, + "8": 0.04658064299307307, + "9": 0.03702260909906356 + }, + "quantization": { + "high_bits": 8, + "low_bits": 4, + "rounding": "nearest", + "scale_bits": 16 + }, + "random_baseline_seeds": [ + 1101, + 2202, + 3303 + ], + "row_count": 128, + "scored_state_reads": 7134, + "seed": 2339, + "task_token_counts": [ + { + "code_tokens": 20, + "prompt_tokens": 112, + "scored_state_reads": 19, + "task_id": 945 + }, + { + "code_tokens": 53, + "prompt_tokens": 101, + "scored_state_reads": 52, + "task_id": 794 + }, + { + "code_tokens": 95, + "prompt_tokens": 76, + "scored_state_reads": 94, + "task_id": 657 + }, + { + "code_tokens": 181, + "prompt_tokens": 151, + "scored_state_reads": 180, + "task_id": 702 + }, + { + "code_tokens": 38, + "prompt_tokens": 117, + "scored_state_reads": 37, + "task_id": 651 + }, + { + "code_tokens": 52, + "prompt_tokens": 240, + "scored_state_reads": 51, + "task_id": 720 + }, + { + "code_tokens": 78, + "prompt_tokens": 83, + "scored_state_reads": 77, + "task_id": 903 + }, + { + "code_tokens": 133, + "prompt_tokens": 96, + "scored_state_reads": 132, + "task_id": 918 + }, + { + "code_tokens": 85, + "prompt_tokens": 120, + "scored_state_reads": 84, + "task_id": 823 + }, + { + "code_tokens": 62, + "prompt_tokens": 87, + "scored_state_reads": 61, + "task_id": 962 + }, + { + "code_tokens": 24, + "prompt_tokens": 80, + "scored_state_reads": 23, + "task_id": 641 + }, + { + "code_tokens": 36, + "prompt_tokens": 246, + "scored_state_reads": 35, + "task_id": 821 + }, + { + "code_tokens": 22, + "prompt_tokens": 90, + "scored_state_reads": 21, + "task_id": 882 + }, + { + "code_tokens": 71, + "prompt_tokens": 82, + "scored_state_reads": 70, + "task_id": 874 + }, + { + "code_tokens": 65, + "prompt_tokens": 79, + "scored_state_reads": 64, + "task_id": 768 + }, + { + "code_tokens": 52, + "prompt_tokens": 214, + "scored_state_reads": 51, + "task_id": 635 + }, + { + "code_tokens": 39, + "prompt_tokens": 69, + "scored_state_reads": 38, + "task_id": 666 + }, + { + "code_tokens": 35, + "prompt_tokens": 290, + "scored_state_reads": 34, + "task_id": 795 + }, + { + "code_tokens": 26, + "prompt_tokens": 91, + "scored_state_reads": 25, + "task_id": 944 + }, + { + "code_tokens": 38, + "prompt_tokens": 249, + "scored_state_reads": 37, + "task_id": 653 + }, + { + "code_tokens": 22, + "prompt_tokens": 271, + "scored_state_reads": 21, + "task_id": 857 + }, + { + "code_tokens": 68, + "prompt_tokens": 112, + "scored_state_reads": 67, + "task_id": 884 + }, + { + "code_tokens": 29, + "prompt_tokens": 139, + "scored_state_reads": 28, + "task_id": 878 + }, + { + "code_tokens": 132, + "prompt_tokens": 71, + "scored_state_reads": 131, + "task_id": 822 + }, + { + "code_tokens": 64, + "prompt_tokens": 89, + "scored_state_reads": 63, + "task_id": 687 + }, + { + "code_tokens": 31, + "prompt_tokens": 74, + "scored_state_reads": 30, + "task_id": 820 + }, + { + "code_tokens": 36, + "prompt_tokens": 215, + "scored_state_reads": 35, + "task_id": 920 + }, + { + "code_tokens": 135, + "prompt_tokens": 75, + "scored_state_reads": 134, + "task_id": 771 + }, + { + "code_tokens": 47, + "prompt_tokens": 305, + "scored_state_reads": 46, + "task_id": 869 + }, + { + "code_tokens": 36, + "prompt_tokens": 95, + "scored_state_reads": 35, + "task_id": 851 + }, + { + "code_tokens": 37, + "prompt_tokens": 128, + "scored_state_reads": 36, + "task_id": 728 + }, + { + "code_tokens": 39, + "prompt_tokens": 114, + "scored_state_reads": 38, + "task_id": 704 + }, + { + "code_tokens": 26, + "prompt_tokens": 138, + "scored_state_reads": 25, + "task_id": 725 + }, + { + "code_tokens": 49, + "prompt_tokens": 172, + "scored_state_reads": 48, + "task_id": 616 + }, + { + "code_tokens": 46, + "prompt_tokens": 239, + "scored_state_reads": 45, + "task_id": 686 + }, + { + "code_tokens": 44, + "prompt_tokens": 344, + "scored_state_reads": 43, + "task_id": 722 + }, + { + "code_tokens": 27, + "prompt_tokens": 86, + "scored_state_reads": 26, + "task_id": 919 + }, + { + "code_tokens": 33, + "prompt_tokens": 94, + "scored_state_reads": 32, + "task_id": 636 + }, + { + "code_tokens": 66, + "prompt_tokens": 141, + "scored_state_reads": 65, + "task_id": 607 + }, + { + "code_tokens": 314, + "prompt_tokens": 85, + "scored_state_reads": 313, + "task_id": 950 + }, + { + "code_tokens": 111, + "prompt_tokens": 82, + "scored_state_reads": 110, + "task_id": 885 + }, + { + "code_tokens": 41, + "prompt_tokens": 88, + "scored_state_reads": 40, + "task_id": 646 + }, + { + "code_tokens": 18, + "prompt_tokens": 79, + "scored_state_reads": 17, + "task_id": 708 + }, + { + "code_tokens": 33, + "prompt_tokens": 98, + "scored_state_reads": 32, + "task_id": 840 + }, + { + "code_tokens": 101, + "prompt_tokens": 108, + "scored_state_reads": 100, + "task_id": 793 + }, + { + "code_tokens": 46, + "prompt_tokens": 95, + "scored_state_reads": 45, + "task_id": 860 + }, + { + "code_tokens": 21, + "prompt_tokens": 83, + "scored_state_reads": 20, + "task_id": 800 + }, + { + "code_tokens": 56, + "prompt_tokens": 114, + "scored_state_reads": 55, + "task_id": 867 + }, + { + "code_tokens": 184, + "prompt_tokens": 189, + "scored_state_reads": 183, + "task_id": 622 + }, + { + "code_tokens": 35, + "prompt_tokens": 85, + "scored_state_reads": 34, + "task_id": 677 + }, + { + "code_tokens": 86, + "prompt_tokens": 75, + "scored_state_reads": 85, + "task_id": 849 + }, + { + "code_tokens": 48, + "prompt_tokens": 91, + "scored_state_reads": 47, + "task_id": 756 + }, + { + "code_tokens": 59, + "prompt_tokens": 65, + "scored_state_reads": 58, + "task_id": 960 + }, + { + "code_tokens": 47, + "prompt_tokens": 77, + "scored_state_reads": 46, + "task_id": 741 + }, + { + "code_tokens": 68, + "prompt_tokens": 113, + "scored_state_reads": 67, + "task_id": 763 + }, + { + "code_tokens": 24, + "prompt_tokens": 79, + "scored_state_reads": 23, + "task_id": 924 + }, + { + "code_tokens": 73, + "prompt_tokens": 292, + "scored_state_reads": 72, + "task_id": 839 + }, + { + "code_tokens": 37, + "prompt_tokens": 171, + "scored_state_reads": 36, + "task_id": 870 + }, + { + "code_tokens": 47, + "prompt_tokens": 130, + "scored_state_reads": 46, + "task_id": 862 + }, + { + "code_tokens": 22, + "prompt_tokens": 71, + "scored_state_reads": 21, + "task_id": 877 + }, + { + "code_tokens": 77, + "prompt_tokens": 87, + "scored_state_reads": 76, + "task_id": 672 + }, + { + "code_tokens": 28, + "prompt_tokens": 198, + "scored_state_reads": 27, + "task_id": 894 + }, + { + "code_tokens": 65, + "prompt_tokens": 171, + "scored_state_reads": 64, + "task_id": 911 + }, + { + "code_tokens": 56, + "prompt_tokens": 139, + "scored_state_reads": 55, + "task_id": 907 + }, + { + "code_tokens": 31, + "prompt_tokens": 106, + "scored_state_reads": 30, + "task_id": 744 + }, + { + "code_tokens": 16, + "prompt_tokens": 161, + "scored_state_reads": 15, + "task_id": 858 + }, + { + "code_tokens": 32, + "prompt_tokens": 143, + "scored_state_reads": 31, + "task_id": 908 + }, + { + "code_tokens": 44, + "prompt_tokens": 124, + "scored_state_reads": 43, + "task_id": 695 + }, + { + "code_tokens": 36, + "prompt_tokens": 92, + "scored_state_reads": 35, + "task_id": 674 + }, + { + "code_tokens": 37, + "prompt_tokens": 73, + "scored_state_reads": 36, + "task_id": 937 + }, + { + "code_tokens": 227, + "prompt_tokens": 201, + "scored_state_reads": 226, + "task_id": 783 + }, + { + "code_tokens": 13, + "prompt_tokens": 117, + "scored_state_reads": 12, + "task_id": 792 + }, + { + "code_tokens": 33, + "prompt_tokens": 96, + "scored_state_reads": 32, + "task_id": 848 + }, + { + "code_tokens": 26, + "prompt_tokens": 81, + "scored_state_reads": 25, + "task_id": 973 + }, + { + "code_tokens": 53, + "prompt_tokens": 172, + "scored_state_reads": 52, + "task_id": 648 + }, + { + "code_tokens": 49, + "prompt_tokens": 117, + "scored_state_reads": 48, + "task_id": 772 + }, + { + "code_tokens": 50, + "prompt_tokens": 75, + "scored_state_reads": 49, + "task_id": 764 + }, + { + "code_tokens": 33, + "prompt_tokens": 130, + "scored_state_reads": 32, + "task_id": 710 + }, + { + "code_tokens": 28, + "prompt_tokens": 310, + "scored_state_reads": 27, + "task_id": 615 + }, + { + "code_tokens": 23, + "prompt_tokens": 145, + "scored_state_reads": 22, + "task_id": 929 + }, + { + "code_tokens": 22, + "prompt_tokens": 183, + "scored_state_reads": 21, + "task_id": 872 + }, + { + "code_tokens": 41, + "prompt_tokens": 93, + "scored_state_reads": 40, + "task_id": 804 + }, + { + "code_tokens": 31, + "prompt_tokens": 90, + "scored_state_reads": 30, + "task_id": 713 + }, + { + "code_tokens": 81, + "prompt_tokens": 309, + "scored_state_reads": 80, + "task_id": 709 + }, + { + "code_tokens": 26, + "prompt_tokens": 209, + "scored_state_reads": 25, + "task_id": 854 + }, + { + "code_tokens": 49, + "prompt_tokens": 310, + "scored_state_reads": 48, + "task_id": 698 + }, + { + "code_tokens": 11, + "prompt_tokens": 132, + "scored_state_reads": 10, + "task_id": 847 + }, + { + "code_tokens": 87, + "prompt_tokens": 111, + "scored_state_reads": 86, + "task_id": 627 + }, + { + "code_tokens": 42, + "prompt_tokens": 255, + "scored_state_reads": 41, + "task_id": 936 + }, + { + "code_tokens": 26, + "prompt_tokens": 148, + "scored_state_reads": 25, + "task_id": 827 + }, + { + "code_tokens": 48, + "prompt_tokens": 90, + "scored_state_reads": 47, + "task_id": 930 + }, + { + "code_tokens": 114, + "prompt_tokens": 129, + "scored_state_reads": 113, + "task_id": 751 + }, + { + "code_tokens": 118, + "prompt_tokens": 82, + "scored_state_reads": 117, + "task_id": 707 + }, + { + "code_tokens": 31, + "prompt_tokens": 178, + "scored_state_reads": 30, + "task_id": 946 + }, + { + "code_tokens": 136, + "prompt_tokens": 83, + "scored_state_reads": 135, + "task_id": 853 + }, + { + "code_tokens": 33, + "prompt_tokens": 96, + "scored_state_reads": 32, + "task_id": 824 + }, + { + "code_tokens": 102, + "prompt_tokens": 107, + "scored_state_reads": 101, + "task_id": 755 + }, + { + "code_tokens": 27, + "prompt_tokens": 231, + "scored_state_reads": 26, + "task_id": 705 + }, + { + "code_tokens": 25, + "prompt_tokens": 140, + "scored_state_reads": 24, + "task_id": 614 + }, + { + "code_tokens": 28, + "prompt_tokens": 85, + "scored_state_reads": 27, + "task_id": 835 + }, + { + "code_tokens": 43, + "prompt_tokens": 109, + "scored_state_reads": 42, + "task_id": 785 + }, + { + "code_tokens": 31, + "prompt_tokens": 319, + "scored_state_reads": 30, + "task_id": 662 + }, + { + "code_tokens": 23, + "prompt_tokens": 111, + "scored_state_reads": 22, + "task_id": 732 + }, + { + "code_tokens": 18, + "prompt_tokens": 118, + "scored_state_reads": 17, + "task_id": 679 + }, + { + "code_tokens": 132, + "prompt_tokens": 161, + "scored_state_reads": 131, + "task_id": 895 + }, + { + "code_tokens": 27, + "prompt_tokens": 134, + "scored_state_reads": 26, + "task_id": 727 + }, + { + "code_tokens": 41, + "prompt_tokens": 109, + "scored_state_reads": 40, + "task_id": 629 + }, + { + "code_tokens": 23, + "prompt_tokens": 75, + "scored_state_reads": 22, + "task_id": 789 + }, + { + "code_tokens": 117, + "prompt_tokens": 268, + "scored_state_reads": 116, + "task_id": 819 + }, + { + "code_tokens": 32, + "prompt_tokens": 120, + "scored_state_reads": 31, + "task_id": 886 + }, + { + "code_tokens": 183, + "prompt_tokens": 113, + "scored_state_reads": 182, + "task_id": 669 + }, + { + "code_tokens": 123, + "prompt_tokens": 182, + "scored_state_reads": 122, + "task_id": 846 + }, + { + "code_tokens": 45, + "prompt_tokens": 128, + "scored_state_reads": 44, + "task_id": 809 + }, + { + "code_tokens": 56, + "prompt_tokens": 91, + "scored_state_reads": 55, + "task_id": 797 + }, + { + "code_tokens": 60, + "prompt_tokens": 82, + "scored_state_reads": 59, + "task_id": 803 + }, + { + "code_tokens": 26, + "prompt_tokens": 160, + "scored_state_reads": 25, + "task_id": 644 + }, + { + "code_tokens": 25, + "prompt_tokens": 82, + "scored_state_reads": 24, + "task_id": 647 + }, + { + "code_tokens": 57, + "prompt_tokens": 68, + "scored_state_reads": 56, + "task_id": 605 + }, + { + "code_tokens": 91, + "prompt_tokens": 236, + "scored_state_reads": 90, + "task_id": 630 + }, + { + "code_tokens": 59, + "prompt_tokens": 78, + "scored_state_reads": 58, + "task_id": 868 + }, + { + "code_tokens": 32, + "prompt_tokens": 278, + "scored_state_reads": 31, + "task_id": 902 + }, + { + "code_tokens": 44, + "prompt_tokens": 72, + "scored_state_reads": 43, + "task_id": 955 + }, + { + "code_tokens": 33, + "prompt_tokens": 85, + "scored_state_reads": 32, + "task_id": 640 + }, + { + "code_tokens": 45, + "prompt_tokens": 86, + "scored_state_reads": 44, + "task_id": 602 + }, + { + "code_tokens": 36, + "prompt_tokens": 183, + "scored_state_reads": 35, + "task_id": 712 + }, + { + "code_tokens": 27, + "prompt_tokens": 117, + "scored_state_reads": 26, + "task_id": 606 + }, + { + "code_tokens": 51, + "prompt_tokens": 118, + "scored_state_reads": 50, + "task_id": 656 + }, + { + "code_tokens": 104, + "prompt_tokens": 89, + "scored_state_reads": 103, + "task_id": 671 + } + ] + }, + "claim_scope": { + "confirmation_touched": false, + "development_touched": false, + "protocol_eligible": true, + "research_result": false, + "selector_only": true + }, + "environment": { + "command": [ + "python.exe", + "scripts\\calibrate_mbpp_layers.py", + "--local-files-only", + "--output", + "artifacts\\mbpp-v02-calibration-full-v2.json" + ], + "cuda_runtime": "12.8", + "datasets": "4.8.5", + "device": "cuda", + "device_name": "NVIDIA GeForce RTX 5070 Laptop GPU", + "model_dtype": "torch.bfloat16", + "nvidia_driver_versions": [ + "592.15" + ], + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "torch": "2.11.0+cu128", + "tracked_worktree_clean": true, + "transformers": "5.14.1" + }, + "source": { + "dataset_manifest": { + "config": "full", + "dataset_id": "google-research-datasets/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "calibration", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "row_count": 128, + "rows": [ + { + "sha256": "18a4b60ed2d7426288593f337a0d0ce2fd0bd16f8ecb6b396069457de662d67e", + "task_id": 602 + }, + { + "sha256": "6bab0180f1a5abeaaf4414589ec875dbc6fcbc7abf17f56e836b380848feb570", + "task_id": 605 + }, + { + "sha256": "b340a4d0a70a96de7b32ccdb303c14854ea25630964f0149c1ddadbd4bdc76ac", + "task_id": 606 + }, + { + "sha256": "11232053152c07a60b8e2e79815d3de04a1443a87a65774ef87639fedcf630b6", + "task_id": 607 + }, + { + "sha256": "df1d9295892a3ec0212556c0905a23db8fc1eab76a0f0ea6c219307d3d4db6e7", + "task_id": 614 + }, + { + "sha256": "bedd17b437fab4573218c8f6c42328991af93325f68a3b76eda1d36d048a30e7", + "task_id": 615 + }, + { + "sha256": "5f1b71dafe9e85d5f48be7f279c5634adae641fdfb87e6909a997bcc5b74acae", + "task_id": 616 + }, + { + "sha256": "57ac285cb7a025d45282c3f6abe281fce91e4da9330e282165c7a0c99907f09b", + "task_id": 622 + }, + { + "sha256": "23062ec4d03f9ce58874abccbf75f3436b384930c2352538c01ab0c52cfae270", + "task_id": 627 + }, + { + "sha256": "48d564ef9d4278625af5579de41175d80500357cebd54dbbe30c34e6e6017f64", + "task_id": 629 + }, + { + "sha256": "b8c6d761d7946756d948fa254a4dd95255620f840e21bfa47bc49452c63f17d6", + "task_id": 630 + }, + { + "sha256": "4770efd6c106a46a9324fbc5fb83a8755e9fd687b484752a0299f70ec0cbbeeb", + "task_id": 635 + }, + { + "sha256": "02970640b1788468a6f2ff70b282e0f78ab45bb155d30974046382cf62b3f9b7", + "task_id": 636 + }, + { + "sha256": "491d1321a7d87bbc4aa8b65850260cb0b82d1b5c0ff1627689c7c82cf5c5da53", + "task_id": 640 + }, + { + "sha256": "bcdcd2a5d06d5b9ef6eebefd3ea184b60f0ec91a678b3708006b239ca8993977", + "task_id": 641 + }, + { + "sha256": "4453b1489fe4024bd04a184ac776f8c49a877602c89658ad181d0e9b574a66ae", + "task_id": 644 + }, + { + "sha256": "a0a6132d99c64acc06c5947116f86df29c7e7c4b10e449d8357c859a0cac8ecb", + "task_id": 646 + }, + { + "sha256": "69f26a25c79f72d70f09311f96a1e6371dc30c019f8ded18f970243ab490149c", + "task_id": 647 + }, + { + "sha256": "68b51e215f65f6b72cff28976c5df8a8c3e53852b7003829476d7ba13c64978c", + "task_id": 648 + }, + { + "sha256": "df3782543fe3ef4a8443a7737e1f3d7678c974984d7960d19a53c0a94ff8c046", + "task_id": 651 + }, + { + "sha256": "346aee9efc80955f531c1574db8b5c2eb25b48876c76a0aba1a35f15138800db", + "task_id": 653 + }, + { + "sha256": "7f8f1b79fdcebed75ffa633e7acae366be57906fe3a7c540eeab2a5b6f5dcd97", + "task_id": 656 + }, + { + "sha256": "807f4d8170d4920740bd290f081110e02695892d741647dd9c06eddcde0a659d", + "task_id": 657 + }, + { + "sha256": "20ff8b409982378f49061b5cf94d89613d1239da5fdab8121a1b014ec5f9d9c8", + "task_id": 662 + }, + { + "sha256": "b4f5989005c921c3ab94ab52c8115e79f99a22390bc1d6e6235d36fd02687fb9", + "task_id": 666 + }, + { + "sha256": "20486484c2b7f27cb2eb2101c7b707cc4cdf748de58a09ad2dddc6997d089654", + "task_id": 669 + }, + { + "sha256": "2d42c6ddb97ff8b63a27004ec719c5254e84c0d8e0d971aa7879608821a3eab1", + "task_id": 671 + }, + { + "sha256": "82b87eac2a21c807c65cb68047ed2cb5ce3d8bc453bb6a99205f6a9555843afd", + "task_id": 672 + }, + { + "sha256": "cef670bd00614b270af30dcc9941df4e1c429422d8f5ee4791b13be512cb018f", + "task_id": 674 + }, + { + "sha256": "0e7a43cda14ead948c5a011dae76ec4cd91111fcb644abd2afca0356375c99bb", + "task_id": 677 + }, + { + "sha256": "8a74422c7a3cedd583cdc334004eaac60a4c79bc14517cab488741e67f62b0a8", + "task_id": 679 + }, + { + "sha256": "d41cc9253a016b071472b34642a8e1ad5f66b319f20719e9cc839eaf305ae5a6", + "task_id": 686 + }, + { + "sha256": "350e5f813e96ccbba22a8db978e72840806c6e6fe4be28a06905c3eae5cb68ef", + "task_id": 687 + }, + { + "sha256": "2ad8a07f09fd2969bd7cb2ef8f21ec19a85fe90060baedaf171f96e4606b6f64", + "task_id": 695 + }, + { + "sha256": "03f32966879f76e560c9d681e0606c9e43b7a3ec16cef3834336ca1907f0f0b2", + "task_id": 698 + }, + { + "sha256": "54dc13e4c5b064b80966b9f315eed560502e37213bd74a5b712fcfc01cb605f3", + "task_id": 702 + }, + { + "sha256": "dfedafc5ad5c0985fcb5f3451f9434628f9d43077fe643ca1d5026106b5caccc", + "task_id": 704 + }, + { + "sha256": "e72224e1ea5cf51e4d9536da838a5ba80df5683c7b670279dc884694d4f28b59", + "task_id": 705 + }, + { + "sha256": "7715b06bec0d8d30f4661ddd980830ac6a6a2e56478d23f2ec86a912089ba25f", + "task_id": 707 + }, + { + "sha256": "bccec7302a85edcfc051afb0894c6915c3fad68dd121d9cbb453d1b03511fd3b", + "task_id": 708 + }, + { + "sha256": "75f50148134355f44c0010350a3b783fef43dc41ee51e48aa4d6e1c76d2f6837", + "task_id": 709 + }, + { + "sha256": "9d60a0c3316db6d1074b90bcc06a5e5c23f55f86f7a2e0db30c463e17c8d8dd9", + "task_id": 710 + }, + { + "sha256": "37506bba8a16943c651e26b16a3671cf5c8002a3db262ea3034825c82c8c4643", + "task_id": 712 + }, + { + "sha256": "b48fc42d297500a6fe097476c89f088c1311784e8bd1bcf6b79d4e9fedb6b019", + "task_id": 713 + }, + { + "sha256": "3efcea05b02d799e12cb49c5d26bbbf2c00972fd466bc30add66ad09a754fc14", + "task_id": 720 + }, + { + "sha256": "9bbd665af173f07fd00e8e9f14d16fe42d02fe7a4060be3ee8656296f1c24015", + "task_id": 722 + }, + { + "sha256": "4def8e85bd92c9756de5a6ce85bf7cd73aed45a8e582120e1d8de8f1eb70868e", + "task_id": 725 + }, + { + "sha256": "0ba2390b8450c2d706cf103c9d926f660643edb4bf937c7120e05ae7dcd89e33", + "task_id": 727 + }, + { + "sha256": "1c0a375e35a7a918b8caaaf3314b75c36dd75e95f4ce01fa24ac80bb7c5af8cc", + "task_id": 728 + }, + { + "sha256": "d8b59f11844633172b0c86bdbefc19a87cbf9de3fb20c033ad26dcdecef2061d", + "task_id": 732 + }, + { + "sha256": "b6cfe3dac4270af8bc80cd469cf6ac9c996354660864448756c25b09996ed5d7", + "task_id": 741 + }, + { + "sha256": "b783efeabb19cddcaa1cd60c417641723e50d2a6a6c772df5b639a676064b053", + "task_id": 744 + }, + { + "sha256": "4be0d45499d58b4ba46de13a38e004d6b29bd5279a273f40c7828970a41090cf", + "task_id": 751 + }, + { + "sha256": "342167819b2e0845ea5bcd26ec47183d0e2ffa07e727d8e7245708f320c100d7", + "task_id": 755 + }, + { + "sha256": "c548acfae5d44e40dadf7340e6a95770267132f0d9ebaa56c06dddce84eeec0f", + "task_id": 756 + }, + { + "sha256": "1fa1bb3481bbffa4496b6e035777caf912a0f650985b08ec3a9041900e9e73e2", + "task_id": 763 + }, + { + "sha256": "0df88505b64d45a45f50f4fdcf078c7d8695eb7d3117f9ee328941bb96955b9e", + "task_id": 764 + }, + { + "sha256": "c5dc00c3b5e0db444f9493621a91b46ccb74e1245d5369c204fe345fadd67883", + "task_id": 768 + }, + { + "sha256": "7470507a8c6c27b54bb1a56ef07e012d75c78bacbf8570aff8d0bb5f9919ac67", + "task_id": 771 + }, + { + "sha256": "ccac8e8d77476d212a2e56a1661aff437a4ff40d99796dcf86bc65cea561dcf5", + "task_id": 772 + }, + { + "sha256": "91f613a6e3222715388e93f2384571e45c95b6d8c1e076f55eaa804ebf5cf286", + "task_id": 783 + }, + { + "sha256": "eeb0f1c9acca121d1d69b7a85127add543fd58839dd1e6256606a4cca432f4f0", + "task_id": 785 + }, + { + "sha256": "573aa7da9b3d8423c03b243964bf76b81ebb1c6bc7d321a15511f5fec7d665d6", + "task_id": 789 + }, + { + "sha256": "399dee06f330be92cabacc7047ed35d013d3f5134ed909669eb721c1476032cd", + "task_id": 792 + }, + { + "sha256": "af6a0c224e5118d6554a8c07cf9fda753414d8b0e23784969d6df6b4ee365191", + "task_id": 793 + }, + { + "sha256": "18b5d375c0d8a97b8a071d73e606ae091afe52f218959ae7e262ef7e2b155d50", + "task_id": 794 + }, + { + "sha256": "0da1480294ac295b6ceebf822dd2b512863c26b314e43f95ca6e5945e6ecfd82", + "task_id": 795 + }, + { + "sha256": "d6ca683dbbde0e79359ac838adedd6019c813004fb7da48a0f999a68204c67d2", + "task_id": 797 + }, + { + "sha256": "b5f6d0ebbd110d5f0ca866dca13ee76da881c253a88f922299e9315586bd8474", + "task_id": 800 + }, + { + "sha256": "9b5f42e1959b2f2a9a243bbfb3cc6af3b3fed1b6527d17b86509b73e7716a077", + "task_id": 803 + }, + { + "sha256": "c6bc9b29581330ff72d7c4fa53e8f73b45a402559ff93569641a51f9a4439024", + "task_id": 804 + }, + { + "sha256": "66e49c89e8421283d55bc358a5c2059c3d62784446f0e586f3f2d77aef6ff151", + "task_id": 809 + }, + { + "sha256": "96d970edf165405b1f5aed50a909af70627a54536c67607012e53b4b38f33ea1", + "task_id": 819 + }, + { + "sha256": "9ba789ae1cb3f3e21ec124ab7bd3447665019e54aa77fdb409c6d5a871994809", + "task_id": 820 + }, + { + "sha256": "f9644cedeecc273044f3c51acd0911010eb58194da4b902644a7bcb8654bab02", + "task_id": 821 + }, + { + "sha256": "ba6eaa8bdd27506a5372e9e6beeb080d23bbbdc6996661d75ab26290d05fb3a4", + "task_id": 822 + }, + { + "sha256": "1d7c5a34b0c84f2c09e769a2402bb5b70e959fc9aff9cd444d2385915cc1f50a", + "task_id": 823 + }, + { + "sha256": "4c537391db5aa03bd9ba4cf70c3676d598a3434e2de0c93a2b41aeb0b8c888f0", + "task_id": 824 + }, + { + "sha256": "0c1ebfbfd1ef30b7b921789d7b7d2be991b1039dea30480c3b7c20732dbea8c7", + "task_id": 827 + }, + { + "sha256": "22fc3ce5e518c2f5e0a3b49b4f606a9a2ff6eb1b365a5cd2b763ef5c4559da26", + "task_id": 835 + }, + { + "sha256": "9a3a8dc7b0c01f525b10282b3be30fe899e1cf70289e5ec5cf6d4ed5d2627d0f", + "task_id": 839 + }, + { + "sha256": "a49784c13a6113a3ded3161764a8f37473a15f3760e0cdf01927f30bfe040c18", + "task_id": 840 + }, + { + "sha256": "bc0169a770b6be2b03fa9501e85c89697f6473aadf85a06fd85ea717b8e0e2e5", + "task_id": 846 + }, + { + "sha256": "13e1a19869e5aeb1b5e3ffb05043dcf0c2a879554353a4efb64e2f0c9079de99", + "task_id": 847 + }, + { + "sha256": "80baece0fdc2ab7d4fd5918508ee9d39b9a9fd3edb73aaa4a9a8536e0a2d3e91", + "task_id": 848 + }, + { + "sha256": "653a2e01264ed1d36f7adb97f2bfdac1abe506f1f935c2bcaf227a3e9c135353", + "task_id": 849 + }, + { + "sha256": "b1761ef5396f9ba11c5047b5cb0ff0955f3c8ad67f1c8449a4d7b394f8009076", + "task_id": 851 + }, + { + "sha256": "f02e13461a874ce9d2f4aa9c1f69b37d56e9891f3ee53fc2558d8188d127e654", + "task_id": 853 + }, + { + "sha256": "2253f072deaccdcde508bf44f7518d62b5f2488bc8d56844c81ed6679b07c4f5", + "task_id": 854 + }, + { + "sha256": "e9ac9acf8ca396274eef0b4479cfe2c0126ab6656222828b6198a497fa46f913", + "task_id": 857 + }, + { + "sha256": "0cc21cce1fa59e5fc55789a03b8a2ddde0febaee7eb61c60408007bf3e8f2af2", + "task_id": 858 + }, + { + "sha256": "c249175bd739930b950995bede307495b169c63f709eb2e41f0fc5597efada0f", + "task_id": 860 + }, + { + "sha256": "9e4b6ef7ce7d4eda8a241226cc66b0486cfbb42e1348463e9c82657769beaf95", + "task_id": 862 + }, + { + "sha256": "63b248175efd8f2e22ec48e9f77bbbcb488acb1cb414d122e610216aab83ca7b", + "task_id": 867 + }, + { + "sha256": "ceca3057ffcc25c3be3630cf0ac223f753c73c9b508975c7b182e53ad636e5e5", + "task_id": 868 + }, + { + "sha256": "be0c42f832fb5afeb271845eee64ce97f919537ea266e41c881b798c9d15cabc", + "task_id": 869 + }, + { + "sha256": "282f8ea69b300531e40d724fdef1e898a57b74470ae43578bb5dbdaa5aa2637c", + "task_id": 870 + }, + { + "sha256": "53e7b753ded18d7df62a9bd8295fac3f626fbad103eef1f691d295f2dd0c6d97", + "task_id": 872 + }, + { + "sha256": "843ce7cbd3bb988467919c2b3a57c97b88623e0547dd8c401a826c4356ffb331", + "task_id": 874 + }, + { + "sha256": "583c205150f59734648d69899b8b9616e600131ee0e0071c7a2395d0e5578319", + "task_id": 877 + }, + { + "sha256": "88b2f4e5c166757c111f0baff74c20fa4d8f13a4fa13d8403c7cacc4f0558e9c", + "task_id": 878 + }, + { + "sha256": "f09b59b65ffe6d64b07085b045bedca52d872b52785eeae60543e10ce35e4bc2", + "task_id": 882 + }, + { + "sha256": "947efb4b2967163d5a5e7816e091b8b4a2a58c73e479c66ccc1b65fd7ea79cbb", + "task_id": 884 + }, + { + "sha256": "4c0d89e359074acbd4139fbb1fe87edaf1cbde75ed617be49db266a613b09d91", + "task_id": 885 + }, + { + "sha256": "0379930ec9ba2b17b39c37a9745484aef7192df91bbe7bfcb8464bf235b1edc5", + "task_id": 886 + }, + { + "sha256": "d92f7d1ba82cab3763014bea1d719a7771ff72490b774c548c1b77f8aff3e41c", + "task_id": 894 + }, + { + "sha256": "d5ad51d886a926e7a003445a1f0d69cf187f31a4626988613747bb245f5f45e1", + "task_id": 895 + }, + { + "sha256": "25fe9442603dc64889c96d39d99224249721d9f761090a378dc18800eeb83879", + "task_id": 902 + }, + { + "sha256": "a58f7ba4c00be27de2a85b091d06e2bd8ad57f4f7a3859501b027350a2731307", + "task_id": 903 + }, + { + "sha256": "073a931c88cc6d01d730c5050afb5ff700a3ef9aaad7deeaa623aea044fd7968", + "task_id": 907 + }, + { + "sha256": "01792d7274226e621ca8404f5ababe9477ef4a7337cb6fc4961dc9e99f46c0dc", + "task_id": 908 + }, + { + "sha256": "30393686473b7c4700ca1dc290d7690b222d96c763fae28843ea4197858dc5c5", + "task_id": 911 + }, + { + "sha256": "6d3c59ac8dba827b2c8d4ba8307892558d3832c1e622e12f9c8d3ae7371cc4d4", + "task_id": 918 + }, + { + "sha256": "3ef4c44d5746bddb64e9a28c351fb9f56fc451c34c20190a01f4793509a939e3", + "task_id": 919 + }, + { + "sha256": "1539ae14cc92af940de077b6700a7a17ca2649aa26e165e15f6761cabde1363a", + "task_id": 920 + }, + { + "sha256": "35822c08ae141bce472f4f9b8731ea70def350d27cea8744e9976f997d1e986a", + "task_id": 924 + }, + { + "sha256": "a1c8042026c613c27e438a4a9cd39df456e6e56c76707514006d73a3c15a4207", + "task_id": 929 + }, + { + "sha256": "3dc0c98bd4aaa0d4a6036dfc33ca07f4da9fc1e7810e066ba9d96b3638960b3e", + "task_id": 930 + }, + { + "sha256": "6d1a0c6c6a46c2a091052ac90fcf480c78d8644596464156142b8a453479f032", + "task_id": 936 + }, + { + "sha256": "e53eac8aef8a3000d0b10751f54a055fa47e42eeee174b476f3a5b99cb1960db", + "task_id": 937 + }, + { + "sha256": "9a7fbac2ccd13b8940f81ef75d20a4521cad8d39ccfe78714d85c49f712597d0", + "task_id": 944 + }, + { + "sha256": "3208a23529c654eb5f21fcab152e38a4d46e193c11bfba5e3efe6a614339e6f3", + "task_id": 945 + }, + { + "sha256": "526fa30a76ec79477f4332c99e846bb297aef3949a13d0b3a1e8409b16129dd2", + "task_id": 946 + }, + { + "sha256": "85415b60358ed5c36762d2a692f5ea0d242a388c2badd4434cf411684fae7898", + "task_id": 950 + }, + { + "sha256": "d9c425370e8db5b625d6aea44f6e36df58be3f2925624237542896034a2ada25", + "task_id": 955 + }, + { + "sha256": "b03ac32ece778320e5c766293410b13fb5e53fb34a2e9278b2f7ec958e217363", + "task_id": 960 + }, + { + "sha256": "0728aea3b66dd335a36520abdd9acd08d5f3452a4d2ceb85a238fff0fe5f7f18", + "task_id": 962 + }, + { + "sha256": "f8275aebb615918f13a8e66c45b338f579d4542889f23ddd2f5e6f38c97cb286", + "task_id": 973 + } + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": "rq-v0.2", + "source_split": "train" + }, + "dataset_manifest_sha256": "129698d01bdf7f08989878ff7c980230456095c5b2cae42a010f567ecc49dc1a", + "model_id": "Qwen/Qwen3.5-0.8B-Base", + "model_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + "repository_commit": "cc35f4396ef4dd475908d8f96e05fe9c559f13be", + "tokenizer_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + } + }, + "schema_version": 1 +} diff --git a/evidence/mbpp-v02-confirmation-manifest.json b/evidence/mbpp-v02-confirmation-manifest.json new file mode 100644 index 0000000..bd98db6 --- /dev/null +++ b/evidence/mbpp-v02-confirmation-manifest.json @@ -0,0 +1,6136 @@ +{ + "artifact_kind": "recurquant_mbpp_prepared_manifest", + "canonical_evidence_sha256": "21a6d18c6a0887b1499d156a3d610d4bfafdd59d3557713485b62038e263b96a", + "created_at_utc": "2026-07-22T05:30:07.348281+00:00", + "evidence": { + "claim_scope": { + "confirmation_touched": true, + "outcomes_computed": false, + "phase": "confirmation", + "protocol_eligible": true + }, + "environment": { + "command": [ + "python.exe", + "scripts\\evaluate_mbpp.py", + "--phase", + "confirmation", + "--calibration-artifact", + "evidence\\mbpp-v02-calibration.json", + "--confirmation-lock", + "recurquant:unlock-mbpp-confirmation:rq-v0.2", + "--local-files-only", + "--manifest-only", + "--output", + "artifacts\\mbpp-v02-confirmation-manifest.json" + ], + "datasets": "4.8.5", + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "tokenizer_class": "Qwen2Tokenizer", + "torch": "2.11.0+cu128", + "tracked_worktree_clean": true, + "transformers": "5.14.1" + }, + "schedule": { + "candidate_plan": [ + { + "default_bits": 4, + "name": "uniform_int4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + { + "default_bits": 8, + "name": "uniform_int8_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + { + "default_bits": 4, + "name": "read_risk_l0_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "mse_selected_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "random_l18_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 18 + }, + { + "default_bits": 4, + "name": "random_l4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 4 + }, + { + "default_bits": 4, + "name": "random_l13_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 13 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2339", + "rounding": "stochastic", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2340", + "rounding": "stochastic", + "seed": 2340, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2341", + "rounding": "stochastic", + "seed": 2341, + "upgrade_layer": 0 + } + ], + "row_count": 500 + }, + "source": { + "calibration_evidence_sha256": "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee", + "dataset_manifest": { + "config": "full", + "dataset_id": "google-research-datasets/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "confirmation", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "row_count": 500, + "rows": [ + { + "sha256": "2f6a29231bbebfdb0b17c2e267f26f9b35cfd6e6724771647ff1bb77821c40ca", + "task_id": 11 + }, + { + "sha256": "69701847454dfe9133e911d6e636c93515010268a4e768ddb80452c3d13b6614", + "task_id": 12 + }, + { + "sha256": "363abf0efa8bed1a2643c6f4a0912c40686d7721a7a24026fe5dcb2301dbd492", + "task_id": 13 + }, + { + "sha256": "5ebb47a4e9ea4f41cff8bdd7e41783b40429299b3d764f568bfa4dd31ae5fa24", + "task_id": 14 + }, + { + "sha256": "018ccd1c6fb7ec04498dfd5758eb1b8daa39b7ec39e49cc2020178f3bf69ee33", + "task_id": 15 + }, + { + "sha256": "8e4122f4410ea296503267f6ab8dd17f75ae966a3312022560075810a596c1ee", + "task_id": 16 + }, + { + "sha256": "b96c8dfb7ee4fae1195ad89a7f95d6e872988541238679783c855e829e7383fc", + "task_id": 17 + }, + { + "sha256": "705af8c177b2f31958e0633f222ace66b96f41b5922c4176b3822d9c9d22fb06", + "task_id": 18 + }, + { + "sha256": "fdbbfc6f6ec0567d2a5d14f5e3c1bea1af31517de6fb21ff0512c1c1c9448218", + "task_id": 19 + }, + { + "sha256": "be366443f7b156b46f910e0b178411a943894cdea3d4528903f3e23fb9846cdc", + "task_id": 20 + }, + { + "sha256": "c8c319ead869d4b9dba5d362042b55f8d8e7ba0a96dfdae33824b46a6a82e9e2", + "task_id": 21 + }, + { + "sha256": "9ff2683e64c8a4148d7ad62110756a1e3d54297ca285c313c223e596b9260fff", + "task_id": 22 + }, + { + "sha256": "6ab376afe986e3bf251ffd655081a82ce0745c0f6cabe8180d6bf9c0204e6d52", + "task_id": 23 + }, + { + "sha256": "8787c6a4ad8a8ce82cfe3f1983aae531a6a9134e85a981290865dc41996277e9", + "task_id": 24 + }, + { + "sha256": "66a9e6dad977f0a86a21d391656edde9f8558c2125c161c57bbb2d936560c839", + "task_id": 25 + }, + { + "sha256": "81c94a6c19f80640a985bd88e07e3be904962964e57e954fef3be94d683e3401", + "task_id": 26 + }, + { + "sha256": "212c739c9b40f26cc81ac28928921662cb6fe027df124f0c380e31251cd8f636", + "task_id": 27 + }, + { + "sha256": "5e29a25185d0953eb0b5062ac3a40a0fd14a1f94532cea97e892280f5412e6f1", + "task_id": 28 + }, + { + "sha256": "2bc9923a1e44dc11dc70732ad4da7a852f03d737eee531b92b499413466fb6a8", + "task_id": 29 + }, + { + "sha256": "4d763680cadb6bdabd5932a74fbaabb8b992e924abc9e080466cb3ee61f2305b", + "task_id": 30 + }, + { + "sha256": "215c397ffc2e58c32569eba2fb2972fe4303a83213d679f25cd99fa66949267d", + "task_id": 31 + }, + { + "sha256": "cd45e5f183ef7cdfa051dc6750b6a34677b3e0160ef4eff1f364a9f20e1da93f", + "task_id": 32 + }, + { + "sha256": "02760c978d7d50dde29c53b8f140c264ecd87231dfdb7209afdd68b6be4cf8bd", + "task_id": 33 + }, + { + "sha256": "dccdef71e53d0c026eade07be98bae9e95678ee67bc551d4e9bd0e779616eb7a", + "task_id": 34 + }, + { + "sha256": "aa4724f597c4b7636cde1ace9d5807c24bde42f8df576a6fc46d86effb1eada4", + "task_id": 35 + }, + { + "sha256": "7d141a9e2334529294d140fa3ecb1572bb5bce5a77b755f9659451e08c43ac23", + "task_id": 36 + }, + { + "sha256": "7fa06d17c47d679a01a9aa55d9a37e43397e5b5c3d76f906dc53b5fc1ae0746c", + "task_id": 37 + }, + { + "sha256": "009de5516476736101e9e9b904a1d335290578c978e4b932bbd8ae4475fb7082", + "task_id": 38 + }, + { + "sha256": "5aaf4c13f871082a2196f39d96126ea13d7b472823a1bf5de325735e83aef23e", + "task_id": 39 + }, + { + "sha256": "682bc367abdb33a52378f0007d0cb9e71140268bbb01cebfb3d6b36eeb360a35", + "task_id": 40 + }, + { + "sha256": "178b361b3fa5a13f7a3c1cebccbab59e583fd22f1d8fdbab74846347bc29f4d6", + "task_id": 41 + }, + { + "sha256": "0a25a29a2ae9a0f5e59686918169356e953d76d8ccf3121ca4787f808f33539d", + "task_id": 42 + }, + { + "sha256": "206ef439e2f71af51d06bdfa1fe38be444d5d4b2dee58b5db4cf26f663311720", + "task_id": 43 + }, + { + "sha256": "f9ccea714fa76b4ba14787d2cb5997fb1e02f4467cf393076eb799c45e0dc863", + "task_id": 44 + }, + { + "sha256": "b1f2a1bac9e4adecbe6b9e4bef031f4cc1e73721847ddec28a4c309212036395", + "task_id": 45 + }, + { + "sha256": "8d7e5e8686b6dfa0f706a3b6d4cc1a49a273514469bd6c43f3f13234e670b021", + "task_id": 46 + }, + { + "sha256": "870418aa2ef21be37b06635fb9e848864c057ae70a937ddd37709abbfbff0077", + "task_id": 47 + }, + { + "sha256": "907304b908f9017806de07794bf75c35585e902d791cbef3ccf57bac2d632c70", + "task_id": 48 + }, + { + "sha256": "2a31c05527a2774c21f9168760205bf8743cf3f5445d20766d3b0b731351d49f", + "task_id": 49 + }, + { + "sha256": "a9552c1ffb0db7fd510527b21de25ade98178d8a9b46f2553afc1b5540b32105", + "task_id": 50 + }, + { + "sha256": "5f0589cebbc69aafdac77e431f895c2ffdf6c7243caf4fbad9b6d189cd1adb14", + "task_id": 51 + }, + { + "sha256": "c92f7b1e1e9627db3d20d82187d1b6df4faae30b324c32a16a82e8b4e1b97e6c", + "task_id": 52 + }, + { + "sha256": "c3b0a8667f8158d646833fa00f05157b35437f850f7e8077050f5bd3003c592d", + "task_id": 53 + }, + { + "sha256": "4a05ed70d38d8b295de6219759399afd42854f8a9566aef386551e00666d14af", + "task_id": 54 + }, + { + "sha256": "0b23f9a4cb09a1f4cc67b9603dbade3ad6dc4f216d06b83e4f06a688c7c514f5", + "task_id": 55 + }, + { + "sha256": "c7d06955b7fd4893bf0555aadbda7aedfaaa38077981964911b91b5b717b06df", + "task_id": 56 + }, + { + "sha256": "2ee4f615555caad541348003db089ed872d7147436e107e3ba4d7fa837bfdca6", + "task_id": 57 + }, + { + "sha256": "e4b5144effde366f2c024520ab1ec1445f66e7ef96342b21768a9b4e5b7c2e19", + "task_id": 58 + }, + { + "sha256": "bf58ee5a45cfd877586297d1557e3d1b56ac567e951687d211d25fc739e5626a", + "task_id": 59 + }, + { + "sha256": "7b27e236b88d2ce24290ed3fc9a53cbb1d7a27fe855d5260a8b9257328887df8", + "task_id": 60 + }, + { + "sha256": "33e134893d3b545343aeab19299e4e6ca19830c425a0a2e9cc41ecd2cfcd6f16", + "task_id": 61 + }, + { + "sha256": "86e84db560a6f6ea38fad389d80f73a073f9521bb876fbae6c7d2ec4bb45755d", + "task_id": 62 + }, + { + "sha256": "9e2f485080777b58b4070f034769331fd080af2d0fd47a0ba0d01e649e8a3a79", + "task_id": 63 + }, + { + "sha256": "472c3560ce1f1ae28fe42f276d5ef28331679a2ef811152bd16260b51a7e08e2", + "task_id": 64 + }, + { + "sha256": "5ed90b65985a25fc82a206389e3637c2880b448dbdc941e364a47a53e86c5b9e", + "task_id": 65 + }, + { + "sha256": "d0e029585fdba93009db39daa3c95368cac8e76c2a09876963ab3eeac19c4fb6", + "task_id": 66 + }, + { + "sha256": "5a5baef60b6a65436227783ad3b4c8d9aae54ff6b04e2c00a02e33c07351e520", + "task_id": 67 + }, + { + "sha256": "a6af1498663b624ec9c5b9c1ab94087c50b6646cc4f21d3a13ab6189aaae28fb", + "task_id": 68 + }, + { + "sha256": "dbec07be6235f84875a337f1919d906eed2665b752413a7cf99cdb48c39c7404", + "task_id": 69 + }, + { + "sha256": "163d6586c717ac7c18d4fe7912f016d6df888ac2e4a5a5a9f9236527b83c3fdd", + "task_id": 70 + }, + { + "sha256": "cae64ed67ab5d6ee3f5e1d087675761ebcb6376e33220d002dab381afff651f3", + "task_id": 71 + }, + { + "sha256": "995a6cca8fc82d0461c3fe2f3664a6539eae8ef1a7e9d69f40fb92380966f062", + "task_id": 72 + }, + { + "sha256": "14ece3e6418c7b26888aa5a38e78457962fb5c1f5a90b6a7a903801512ab7a85", + "task_id": 73 + }, + { + "sha256": "bc83ca7d863ec7432d8446776d0e933464798d7478b1db96d16f0d85d53a6a0e", + "task_id": 74 + }, + { + "sha256": "7e1e7b7ad6e35e1ad9ce612946ad96d47853b0b831ccef00d8d31be29ca11b07", + "task_id": 75 + }, + { + "sha256": "3f925078ef0fae17ffc99fdd1bcbd78bcf27477e6f2207ed4e31a1f5d88a2548", + "task_id": 76 + }, + { + "sha256": "6ab23be621addc74ae3988f9fad8094f429e762cf5d1041ccdeb4731c5aa6c0b", + "task_id": 77 + }, + { + "sha256": "6d753274f7969d468eb521c0ce58a659ec3c0d03786ec53fe60b11131bdf4dd7", + "task_id": 78 + }, + { + "sha256": "4d3e158b3e71580e3a2a2a7fd51b7fe7beca87de6abb1bcde3304a7c73151d98", + "task_id": 79 + }, + { + "sha256": "5272d05dad6bd25ac4d3ad58beba38ae46c94d4b39b1b8705f6da343d6f7e09a", + "task_id": 80 + }, + { + "sha256": "6c570df4ab02dc78eccec01881546b1673c78120d06cd728f98806f4c4a548ff", + "task_id": 81 + }, + { + "sha256": "c2505b8a66671da0a6cb3260c64f2866567ea65180da1033d709c0e10e87ce9d", + "task_id": 82 + }, + { + "sha256": "048ba5c05339029399a264a3aeb716fd4273765ee8b542a0920676967c8b9066", + "task_id": 83 + }, + { + "sha256": "dd9fc6ecbc3597996e311b9f1f003dd103c554037c7b8be76e954abc3f10e75a", + "task_id": 84 + }, + { + "sha256": "cb6b21693cf0819e22a49b3ae15a9d1adb22e9b36ebd7d115392b03c4c9fe78b", + "task_id": 85 + }, + { + "sha256": "f8ff537cbe7d5b7bb67cfcdd988b38fa226b7cb57da7d44681254de9a65e503a", + "task_id": 86 + }, + { + "sha256": "25a4ec259efdcdc017d62698a5d60585b71ef9b02477f1968bcfc35dad86cc6f", + "task_id": 87 + }, + { + "sha256": "771ed5605e7c5e5d9213c2ff044b83a640f323152aad3ef603b52ceaf019f8f3", + "task_id": 88 + }, + { + "sha256": "02e8bf268416fb14180c7b8a8acbeaa4af888cac0eaaff3cdbf7b855e5512357", + "task_id": 89 + }, + { + "sha256": "b98e46a78cf547b27f14587b81ddc8f93303fb0e64e6cec7c568e5092f605325", + "task_id": 90 + }, + { + "sha256": "ab49ac7c8231642d472966e59d95fb39c51782c70e4519f011310cce247b507e", + "task_id": 91 + }, + { + "sha256": "02ea226ff2c90e62967ab5413734dc1a587075f6a12e198f83a0f76dcd628484", + "task_id": 92 + }, + { + "sha256": "4131ab62c6c84221299aaafcfaa5c8fc2d0375ed351b14bf322c8ad642441c7e", + "task_id": 93 + }, + { + "sha256": "d42f50b469810b0e575b7abb8cc8e172249fdefdf99bb763ee9c2102459bf0b1", + "task_id": 94 + }, + { + "sha256": "26572a8a0cf47a7b96b4bd392213e9a66b75a0eb619a7e0eb77aa21a8c5fd833", + "task_id": 95 + }, + { + "sha256": "704746feaf484fbf86ef2a395edc7a7b561b2f08fe27f1bdf6d55bd232d05f92", + "task_id": 96 + }, + { + "sha256": "4d265ae7b7585759134b446e588fded424f5f0350808dab7c5b380d045c3a401", + "task_id": 97 + }, + { + "sha256": "15436d55206e5095462f78bd610aa5d6c1f130625ea63a358ef52cb46a37baab", + "task_id": 98 + }, + { + "sha256": "e60395697035bd82a20b53f424aaf68e19c80a66c2426484fbc204bee08e9689", + "task_id": 99 + }, + { + "sha256": "ab0ff6ce8c83b157c3d2dec65bac320934c9eb69a87dfa12a7f1042d00a5104e", + "task_id": 100 + }, + { + "sha256": "13fca81b40435a74b8b6790d67c74ff0bc2854531ecf825f50edc77d213ec90e", + "task_id": 101 + }, + { + "sha256": "53d3b51aaa3e61c3a9d767bd002396428ce211eb29da068f812f26cf90a71f3e", + "task_id": 102 + }, + { + "sha256": "27ef5636f3e1f33fc642ae45b1535b15e949e8497c1673e597938fa0c34425d9", + "task_id": 103 + }, + { + "sha256": "2924e5101914d9c3a3123c08a0f4ee42067f1722d19d5d9c61a1bb7266265efe", + "task_id": 104 + }, + { + "sha256": "823f28eec0ba779713ce1a000215d105a3040958e59a93357a6da436b461144f", + "task_id": 105 + }, + { + "sha256": "74301b21f36e8486110c706581f472523601c79f4767c4eca7a23aa4a91f7676", + "task_id": 106 + }, + { + "sha256": "380a03031053e153a203e3af252c5863fb268625d45945141a0a304cf775cf4e", + "task_id": 107 + }, + { + "sha256": "d5531ed21205a5d60352855a47b1c627a90a60d9641fc8dc88c3838cef4b7d53", + "task_id": 108 + }, + { + "sha256": "3bf618870f43656c259bb30a7f5d8631282a03c7f519ecee0c64f59b8b2ce136", + "task_id": 109 + }, + { + "sha256": "e6dbaa4a22ed5bdd7ab864942b8aa0c0044ffdd94fc96d396cda6e8006fa45bd", + "task_id": 110 + }, + { + "sha256": "4baf6b8a0444929743dcf3488310bdb4d177fb92e76e535df9730d01cfc0276b", + "task_id": 111 + }, + { + "sha256": "80e3fef8e5d172bdf4d7770fc3ffd8fe02931317096f75b98fb3f7e17cca02e8", + "task_id": 112 + }, + { + "sha256": "f4481235f45a83bdfe00e359b7500562efc26ef38d2e1fa1af2d72998bee63c8", + "task_id": 113 + }, + { + "sha256": "783c5601d9451aa103c5a217e6b965248c2d6f879c41b74ed2b8bb5c6e3ef606", + "task_id": 114 + }, + { + "sha256": "d5421e5c55296231524f67533b697a81d9ca26ccbcc14172a2a3d6d587ffce4b", + "task_id": 115 + }, + { + "sha256": "24e88b1e95597ec6efea83399873a8a72825f19df9a4484c755db9a7c7937eea", + "task_id": 116 + }, + { + "sha256": "13cb9c1e9768119c3dfbad381d1bbdcae8ff027957f8272c75e8e14fc1b36d03", + "task_id": 117 + }, + { + "sha256": "c02627535f4b4c9fdbd8519dd35a26b2354bd445fe90cdad8e8057de5cf0e786", + "task_id": 118 + }, + { + "sha256": "c137e29604fe5876b06630c740aed84d2f807031141687c7b2910a7fa538dc54", + "task_id": 119 + }, + { + "sha256": "9031f102b1230be91e8916ead0e9b156393f7ddcdb70b86aedc3ec9621d3195c", + "task_id": 120 + }, + { + "sha256": "d8b1a4b63046753b46f8d9c949e146687dbd5ac95777c13efdec136323b5355a", + "task_id": 121 + }, + { + "sha256": "c6a68d8757ba5642fc1b462fd1e20ed539d1cca315d63cf8365da62d5412c481", + "task_id": 122 + }, + { + "sha256": "120959347d4ef625b5cc7e7f937fa3fd8b80fa2ff5571279d6ad61fa61eeecbe", + "task_id": 123 + }, + { + "sha256": "f740b3dbbd4c05f286d8071ae99d4e5919acb73f33fc86fe98a3af63496e1a5a", + "task_id": 124 + }, + { + "sha256": "0943210c451ca802e2e3efb463932dca33c936d0d0defbf6545d3f5321d1855e", + "task_id": 125 + }, + { + "sha256": "05a9d552d96e355a7a16c1e4aba2921b4c21edc40f32078aaa8b3a04cc397577", + "task_id": 126 + }, + { + "sha256": "2d2ae75a8a2826a37c80d1030539df94332b535e26f797ed49a3effc3300135e", + "task_id": 127 + }, + { + "sha256": "8c27f7c9a36a3066a7eaeb7109fac13b3142f953ca144770d28ae81495e39d6e", + "task_id": 128 + }, + { + "sha256": "1a844a84218a805c097ef35531319fd808b8f237a6e149805161158a7a7a26b6", + "task_id": 129 + }, + { + "sha256": "999821329bec7721e2a7e36e4ff8146c8e35c0dd3e960de622eabdf90bca0bdb", + "task_id": 130 + }, + { + "sha256": "24d2e2b8a3f9bdf82cc43b49b856bddaf76d05581443513751a2e7231ffd3e78", + "task_id": 131 + }, + { + "sha256": "be99aaba1de6c8108beb5d29a9d228474efc278c5b457e27d6d4b83345358abe", + "task_id": 132 + }, + { + "sha256": "7bd8e14799247cada71c1ed9ea0e893724ae9f113a95a533cf125a892dc30a05", + "task_id": 133 + }, + { + "sha256": "3821e32efa6fb8d5daa3976a4b38b22622bd9c076468ff69877aa79898e91dc9", + "task_id": 134 + }, + { + "sha256": "311477e634bc30e3c38b0d3cd841e7ce46621663fad87b5562d857f229158bc1", + "task_id": 135 + }, + { + "sha256": "ff64235c93476f0e989635d4498b7548c176cfca8a99bc76e8570de2b3fa7e93", + "task_id": 136 + }, + { + "sha256": "9442f82e59db4825384b253025fb5e8e2125a1970d7d7ad16c4bae67e7e8baf6", + "task_id": 137 + }, + { + "sha256": "83ee57e3168feec39507e03d212f840d9abcf3b921c671fca8d1548ce725b6e1", + "task_id": 138 + }, + { + "sha256": "fc5ea864e3ee5f1243c1b2890d89387dd0c5785b8830dd1fb0c3a5b4295c9271", + "task_id": 139 + }, + { + "sha256": "2493fc54053b527aef5cae741556d59c71f4c080c5fd7ccdcfd043df66aa148f", + "task_id": 140 + }, + { + "sha256": "f816c18b373eac8c9cbb867878f08a9de3988e141093ff51a189cfb4a47fbddc", + "task_id": 141 + }, + { + "sha256": "7e9583f5c048b1bc583747db8ee33019fdf2bec68feeedc9d1b0229cf1eeafbe", + "task_id": 142 + }, + { + "sha256": "8f15463fa74d9a6401c928627304dea026e68ed220cd1ab2a367672f5807e6b1", + "task_id": 143 + }, + { + "sha256": "47aa4a7f81473acb006ce6655c740a01cd2595cc6c4953a40b2025c70f51912c", + "task_id": 144 + }, + { + "sha256": "28f022689390fa11f26734e58f3414efff6cbfc70f8f882c1bbe5f46e9d84b56", + "task_id": 145 + }, + { + "sha256": "c49fd3f990c0bd76dbe7743a03e7f1e7480d3629fe86662a9b85ea61cbe457ed", + "task_id": 146 + }, + { + "sha256": "e062544f369a935a41eecb5903781645eb88074292e964972f16d443b0d6e2ca", + "task_id": 147 + }, + { + "sha256": "11c0ab80bc80cac210b3735ee7e73d38846f2148d559ef528a2e296ef44dfa83", + "task_id": 148 + }, + { + "sha256": "c237acdd9bb305cae18bac0bfcde60262573e07439c17f6258a63974e96a1e78", + "task_id": 149 + }, + { + "sha256": "55b7f2aa839596e044c5c692e22cbd32f684cd11ba0fad7910fb3fcd7b4e37c0", + "task_id": 150 + }, + { + "sha256": "c5b1cc0e09d58f9ae5f926f1c97708c3d67f7d6d0a8fc9a1ee0d75fa5dd6556d", + "task_id": 151 + }, + { + "sha256": "4a6f20274f7f8623b19918c5e1ec2f32764ebb13c29a90b0756980ca624ecaf9", + "task_id": 152 + }, + { + "sha256": "d7489cef07092c64f7a36224d0b463b3a7414c9a635399c653740ed9a82c2512", + "task_id": 153 + }, + { + "sha256": "7a09a77ebef8461c3df5ca0e0b94bdf8193b21661b8c7288b640582105776a4f", + "task_id": 154 + }, + { + "sha256": "050beae947159ab1382fbdde49f5c257e0c4e32127c710f3b2c10e9c02d4de94", + "task_id": 155 + }, + { + "sha256": "476ebc7da4c853e029a3f2b5d7291818f6ffe643f13db4125f2a0d9642111369", + "task_id": 156 + }, + { + "sha256": "2b3a442fe56306279c7308c6d5f17c0f7cf0d8201a702fea94fa6b27d6b6f6a7", + "task_id": 157 + }, + { + "sha256": "0741a465476a7d71c6482d4dd5f74edf629f2d7e2c020ba50337ef409bafcd7c", + "task_id": 158 + }, + { + "sha256": "2000f1d47fef625b730a76ded04fd24dfcf2de212f9464c2ad3081ba3adedada", + "task_id": 159 + }, + { + "sha256": "54cd63229b66bc905c46cb12f231a0a7ea808ce86c353cb2f73f3b3d33843f9a", + "task_id": 160 + }, + { + "sha256": "21cb75acf3c93f065303983a4a818bb3c319fbe5a6f03234c0ac261217535718", + "task_id": 161 + }, + { + "sha256": "e9b3378fc810d27fc3ea9808c2cfd663df99f47b41b0f8461c05d964c58e3487", + "task_id": 162 + }, + { + "sha256": "2678238a3cfddd410a5beb44ad9567ae7023542fe135b78aec309da00d01b4bc", + "task_id": 163 + }, + { + "sha256": "4a93f106b7f6b76d78355179b60ab6fc1270661a7da19be801a794b11c5ba4d8", + "task_id": 164 + }, + { + "sha256": "29bcc66380a822cb34a4204390989eea5d1d6de7c05be38f825bc186f0963f62", + "task_id": 165 + }, + { + "sha256": "c55f5b34a8a2f8359e0db294ff4eb6dc150c8bbc0f05fbd0e58f2bf213f8970a", + "task_id": 166 + }, + { + "sha256": "e3c5cfd263b198cce950f1bd5dfc1de998cad87d4ea39709a41ddcb590f927d3", + "task_id": 167 + }, + { + "sha256": "8c753db16553a82c3e893012632cbd5a82ceb0348cd2b620c241e4277fc8434d", + "task_id": 168 + }, + { + "sha256": "51103c0568757a15aca3288c8690ddba84d5ad4b8a5d4a173b3d1498dba95968", + "task_id": 169 + }, + { + "sha256": "5a7130e56261023a7a9af9647f8353bfb370a7b806ea5fb68434f55f0fb2491c", + "task_id": 170 + }, + { + "sha256": "bf803d4376ec1be41854a125a47cc5bf8ac49a52b74df6f5e4f0a67df8a8a59c", + "task_id": 171 + }, + { + "sha256": "54c0888dd2c8c2aa99c21e526153d1cd4353981c2ced2c96250d69bdeaf095e5", + "task_id": 172 + }, + { + "sha256": "973c9f7c8512078d946c5ee1c2971cc01f4da88862ffd48c2ffa6929f121f5bd", + "task_id": 173 + }, + { + "sha256": "df6608a5d3f60cdb79c16b56241cd4bbac523d593dd0ddb898b64627c132a808", + "task_id": 174 + }, + { + "sha256": "c44b6fa384ad71da45894353369f1aeea685aee9a950824f7edb0d09a8e112fc", + "task_id": 175 + }, + { + "sha256": "170d0225314287b10082e0cc4618858ff5b518a2c5a2bd2016882d4060874593", + "task_id": 176 + }, + { + "sha256": "1ad7fbb80912869942b7c1a6d909a1fd0d7b106f94dc1b92d85a6e57f9d926c9", + "task_id": 177 + }, + { + "sha256": "8fd023bb68accffa9114c792d76a4aae2a872e496cea74810d5beeb02d533e1f", + "task_id": 178 + }, + { + "sha256": "f63d6519ea4be2358a70497b66d1b10442ff892e5d93c853c237e6edf2eda88c", + "task_id": 179 + }, + { + "sha256": "b90a0fec32e4f57161074a84ad7804c6be794ada3c27b2d00bc9e76ae3accfad", + "task_id": 180 + }, + { + "sha256": "a6241a1b151f178fda03690b01cb31effca05fc8be21675eb98ca7dec566c156", + "task_id": 181 + }, + { + "sha256": "0cf1a5a203bf22c21f174c5f9c3f10b78cff0e00e893cab3c2d37d10ecf37273", + "task_id": 182 + }, + { + "sha256": "6c045b07b241b62f0116414e0f2de5f747ce2664706bd53c985a447eb07a1add", + "task_id": 183 + }, + { + "sha256": "fe8432431767e168a054cb9f7579a4117851eb1987c3829256cd5324b7d0abb3", + "task_id": 184 + }, + { + "sha256": "d3b49c071fd026a4fb97766d0f2b5110c702e7cd1c748a2250b60b8cb19c3a2b", + "task_id": 185 + }, + { + "sha256": "f9c72af9d9283176427f75ad58d6cd3f388bc7fc41704e9ca6e1a393c5985dbd", + "task_id": 186 + }, + { + "sha256": "3bf24e849f6e0d54f150ad902ff274f9556761d0adcb32a15d3ce0f396121ee3", + "task_id": 187 + }, + { + "sha256": "ca2004404c39601a10e21abf078fe1461bfca104acbab91eac0c6fe541bfcb69", + "task_id": 188 + }, + { + "sha256": "9c04c3013905867455d36aa1d2fb472fdb6b9dd47bbca70049fa21682a7175b8", + "task_id": 189 + }, + { + "sha256": "2e45424e9b432f19b9a51f30db80fb2edfa6bc1b129b5b2dacab038abc6cbd9a", + "task_id": 190 + }, + { + "sha256": "85adc139072b74675a71fbaf07c4066a772fa25bc44672d46c3b810a3896f539", + "task_id": 191 + }, + { + "sha256": "9d023e6c419c566332cace172fb5514db165ab6e011792faac2cbcd4bbe05833", + "task_id": 192 + }, + { + "sha256": "8553628cfabe9185cb6b6e09f77b72f6d90b7f225246881ccfd1ccc1e9b288cf", + "task_id": 193 + }, + { + "sha256": "a2734f1c04fd7e6dfe5d33b3717bfbc7c5de9dc21816a4c4abef3694096e5520", + "task_id": 194 + }, + { + "sha256": "e8e12d64fb8e2e32c3302027fde3e77976401283f2938fb314ba191541dee142", + "task_id": 195 + }, + { + "sha256": "2364b4a4b6c7bc97cb76c840bfc077f13d44f79813d8366a2b215b54ddd30037", + "task_id": 196 + }, + { + "sha256": "5eb78a0b48e90693efd1cd7f32851e57d032efbd6e05db3f6e43e6591124a726", + "task_id": 197 + }, + { + "sha256": "8fbe6d489ae07c7606f6b837e463e92df271ff0e4f3e765dfa91561ef4853269", + "task_id": 198 + }, + { + "sha256": "38ef3f4ca2766b9a4259981e7a5598b733f86a00f2fa9ff78a84ee130b8a428b", + "task_id": 199 + }, + { + "sha256": "70ecac3fba135b8f75f627c59920c2bdef412d85a3570d5cd1c93e67efbe90ef", + "task_id": 200 + }, + { + "sha256": "fd61c5f1bc5a385d917a1f983e1e5a330b55fb12d5933459119980a4904526c1", + "task_id": 201 + }, + { + "sha256": "f89d460da6b0853684386bfca30a4b42448878e8f9636de3770500d0b74fbf1c", + "task_id": 202 + }, + { + "sha256": "80f11f6b5057b2e466db28155459822e0ffa253daa879e9c13ee4628a7685814", + "task_id": 203 + }, + { + "sha256": "7cc78f32fde8466cafc6526297f450feb431db494e62d8a97a48d0b8bb238979", + "task_id": 204 + }, + { + "sha256": "71a7ad804a167d337188cb02b93f3401322e35823289b1d1b3c372a3167379c5", + "task_id": 205 + }, + { + "sha256": "5238b2a3719afbfc752042fe5802df5a2a34ecd84c0190f4e33238db9a9ddd0a", + "task_id": 206 + }, + { + "sha256": "43c8a66107dacec77b40ac8fe1c4b27350b4f5a742b37ac950f285bcc5933982", + "task_id": 207 + }, + { + "sha256": "0d8b71f7c07efa5b15240e33902b5ad7eac144ca40efe93812ccfe124df91741", + "task_id": 208 + }, + { + "sha256": "6de8256560ff8299ea962a76b585ebea5661821512d1a65d7d8cef42d5838437", + "task_id": 209 + }, + { + "sha256": "2bd9bb0e36b7cd08830d193235b67a933ec08895dc2aeb60b3fd88f48526ca4f", + "task_id": 210 + }, + { + "sha256": "3843729048980d804ad35bd60a6526415c4a951e15459ac60324f4aab3f9b6b1", + "task_id": 211 + }, + { + "sha256": "cfd41fddf21f2d186644f2ba72b29acca18b502a49f8b3f9bd188c4243d97093", + "task_id": 212 + }, + { + "sha256": "dbb88bb03d414fc310e2a59e7abf3e70382f4552ecd0b62d7ec8b7893f787fa6", + "task_id": 213 + }, + { + "sha256": "3c4941b00096f46643bce962a62672bb37a59d6e1e309c606ed890e67f128b7d", + "task_id": 214 + }, + { + "sha256": "327ee93fd78735f1a6511c84e74921fb96777828ff8ed765e515c716e525d761", + "task_id": 215 + }, + { + "sha256": "524227a9717f5911af7a477f3669e4ecef72c20c48f3d32de9b8c2a91f423b97", + "task_id": 216 + }, + { + "sha256": "cabea26272b503e37aed663d1581153f7924c4575306163cac9b9a4f19edadb3", + "task_id": 217 + }, + { + "sha256": "6517220c5a05a0cf7720b8a6a3184da3a997439dc0a5d12d02fd51106a42e053", + "task_id": 218 + }, + { + "sha256": "14e685dfc0764f4e060ce081721570448a4cc05648ff974d00f22bc8940fe324", + "task_id": 219 + }, + { + "sha256": "82a360c9a073bfbf0059d99a5d93d1764d99bc9c9899010b8ba3794e573d9680", + "task_id": 220 + }, + { + "sha256": "dbdd12ff871833c0de1f1d5b1709aee344009fde6dd1bc812b73e1e9283dedcf", + "task_id": 221 + }, + { + "sha256": "690346901931e24a3fabae155ab36455a69e95e07980fd8dc88f2f979f61c978", + "task_id": 222 + }, + { + "sha256": "8708a014dfd0b446c90b258c77775c8fdeb032fbdcbb1ac74455a04bb4c1f504", + "task_id": 223 + }, + { + "sha256": "e5b185f6361b38031c3524e28324c5fb3a572018bedfadf748d19226a06afe6b", + "task_id": 224 + }, + { + "sha256": "f703efe5275f4a094d86256b8868306650706216405077726570522bcf3c2f83", + "task_id": 225 + }, + { + "sha256": "246d1121aed5f93e5da420c60c98df579736c59a8fa005e5f16fc582248714a2", + "task_id": 226 + }, + { + "sha256": "2463cebfe43ad6bdeb076292da2d6eea5e76052aefa60150e06bf33a3af51f17", + "task_id": 227 + }, + { + "sha256": "edd7c15994fdc6efef3334ebba3b2b57185e82f10983df8119a7ebbf89db61e1", + "task_id": 228 + }, + { + "sha256": "e610488735f3f63f6065ce6e7b3151ff2e7492d5606f4c015aec822666db99b3", + "task_id": 229 + }, + { + "sha256": "b9c3b7788b79e161ec4e1985a6fb75d15863e84067865708ce955cc0abd63239", + "task_id": 230 + }, + { + "sha256": "2e2148af44e0a8f59a15f3cc826038985963f85930524df2b9422a6fc0a7e39b", + "task_id": 231 + }, + { + "sha256": "1d28fae3bb1335d765a48bca116ff7460f1ebdb9be535bbd16ed186dc4bd529a", + "task_id": 232 + }, + { + "sha256": "2e7b8df9ccd380e69558721ee2ee142603009dc43f390a49f5ea4582b2f3fda8", + "task_id": 233 + }, + { + "sha256": "f2559e68b39ccb8c5c421b3b5a38a413221834e3b9869aa2cbf64660c7404828", + "task_id": 234 + }, + { + "sha256": "8f6ba9aa1762ddd7e9f83ccadf883d8f73dd84e4eb162a9a880d7dfe6d71cc83", + "task_id": 235 + }, + { + "sha256": "c91fa6515e6861035b85ad8fd8fd7318ae06a35e1fd8013925fa7369585f2503", + "task_id": 236 + }, + { + "sha256": "550002fa0cd08ab6e51efdf52891b8d51ea205ffe26caac5f75a6a24912f37bc", + "task_id": 237 + }, + { + "sha256": "bf55d5e07d2feefb499afe94dd6caf1f6be6bc2516c3fe1cacc2181f1918309b", + "task_id": 238 + }, + { + "sha256": "2a7f878119acf25d53d17fb549eac1899fef52ad2c9b0b60768a0e718d084bf1", + "task_id": 239 + }, + { + "sha256": "f93c9da1a5de9648988dde7f7a9ee71d2ac295768f4b8d51df5da78d3c2d8437", + "task_id": 240 + }, + { + "sha256": "c18ec2070de3cafde537543d5b16293d8212b4d38e36ccf2ce43750706792747", + "task_id": 241 + }, + { + "sha256": "d827a98c206e55b21cb348af83262379bdfe21c8cc65a058ed772481a406c4b1", + "task_id": 242 + }, + { + "sha256": "e656e616ec4aa4d7f30ef3d9ed925a5ca7743043ae96df47537342a05f5ab473", + "task_id": 243 + }, + { + "sha256": "7691ba2cf1ebcc1a1b0f2124fd7b21b830d0a2e008f454f07e5230ad1f5ebd1b", + "task_id": 244 + }, + { + "sha256": "20ef0092979a437ad23865f6eb294ada45c156cc78bc8a256f6bb5c15de13691", + "task_id": 245 + }, + { + "sha256": "5b8fe1ed0503d1d96b6276a5ab726c91d5711e7971a5a429c6090cd5b5ab9163", + "task_id": 246 + }, + { + "sha256": "3d1caef92c89cbc99842120ed162e911c3d179b7ae1c3f30e24072b5aac8e7d8", + "task_id": 247 + }, + { + "sha256": "147e7afc930ea89ff1143aa1cc6540bff8100bebe03d60510971d998e7ec96a9", + "task_id": 248 + }, + { + "sha256": "fc2181ff89e89b8c1246a56d638e76a6ad294568f1abc504b1c7a8744c038200", + "task_id": 249 + }, + { + "sha256": "15d753c2d9a994f5a5e631dea8fa1c725725baee2aadbd4330487cd8b9b1850a", + "task_id": 250 + }, + { + "sha256": "ebd4fc8cb01abb17bde8acdceca42b5163837611a7ee33129174c467f20adcfd", + "task_id": 251 + }, + { + "sha256": "19b799513e56cd92490857a42b3a105836c51cd66da7c66a8eee60cd68a9f79e", + "task_id": 252 + }, + { + "sha256": "0f58e14c5ab619e1295761a91ac1f2259eef0a0b5c083437daa886b284cc7a63", + "task_id": 253 + }, + { + "sha256": "319181b67d9e98bf63885418561f9a991cbb12729fe4e212ef83699dc921cd4e", + "task_id": 254 + }, + { + "sha256": "9125c82eb74146a4cb85c693e800fbd93b4a3527c83e001ed6bc35e39467f2e7", + "task_id": 255 + }, + { + "sha256": "b4aea978719f9cb1475f221854af913288c28c2fecb2b79875d88d8500b8cbd9", + "task_id": 256 + }, + { + "sha256": "fb0e85415d4a83f4d55315b84d5dd022ad54adaf7b3a5c40e983cb52807ee2e3", + "task_id": 257 + }, + { + "sha256": "25f98f17f1c15116fe7646d8270fb8afdf7887d7e3c4e6c2cfb0da7005d6c717", + "task_id": 258 + }, + { + "sha256": "b7dd5b7de8aa2b07015c9b0caa2d7fe99201b05f3f23521b28a635e0e0a8d995", + "task_id": 259 + }, + { + "sha256": "96f0dd3617f4d1e9777ce7ef83adde00c7815739328b6aa86069b19dde896194", + "task_id": 260 + }, + { + "sha256": "802c0b39f263187802fd2c0b92e8d97ebece7ba7d21a386c242c258807cba3e9", + "task_id": 261 + }, + { + "sha256": "3f346fc5605a2d7d23e96673d5da90c44d1e84ecdeb5ada0c5bd3ba3897658e1", + "task_id": 262 + }, + { + "sha256": "1e7358836748876bc2670a0fa1b35c4d052768be12b47137ad27f6a54c31678f", + "task_id": 263 + }, + { + "sha256": "bed4d32d90966acfe2f2213526611e2ca7f34ebcc722a8a597f02bf6299ac915", + "task_id": 264 + }, + { + "sha256": "b569fda57ca6578d8ea0e8eed4b42f69496401491e8a852e3451d4c5bc2b5363", + "task_id": 265 + }, + { + "sha256": "16b0159a777674d34d73bd163e4ee24bf3dc71ace62eab528f80419afd706086", + "task_id": 266 + }, + { + "sha256": "94f0f0e9a97f3aa87e7a7ff98309cdc36a4a62f66f18379e2aa1143a947139c9", + "task_id": 267 + }, + { + "sha256": "7d40ffde20e0204fb1ca47ded9a5effb4d50b907e25065bff7e4ba1438df0fca", + "task_id": 268 + }, + { + "sha256": "b16f4f5ef64cdd19e5eb481ee1a66aeada1981774c79ee031acc1ba1395c5fb2", + "task_id": 269 + }, + { + "sha256": "1a3077a7c720fabd63eb4ff1bf361b34460561c2bd3e34f8766954bd21c76196", + "task_id": 270 + }, + { + "sha256": "f3e44b4e61a94ae392ad260bef41f3a02538825713cb3afa148024e958eba59f", + "task_id": 271 + }, + { + "sha256": "806519fddcd7a32defd20e48d6a0a3be33b61f7095013b29fe783885f6bf2703", + "task_id": 272 + }, + { + "sha256": "cce63f2198749f630e00a4fc7335e1b30de019090ea8908e3a643687b1e44323", + "task_id": 273 + }, + { + "sha256": "6e6b04d34dc592d25d63262dfec633a73205b7476ef1c84e093e2f3b063b2fad", + "task_id": 274 + }, + { + "sha256": "576831528ac961df9df2e12e2d5d22934b36b9cb51e4867e28d10b18d01c1ad8", + "task_id": 275 + }, + { + "sha256": "4ad4ccee8051eaf28b2af36b9ede35f74b404016e95d3424b482c0c0908b12d5", + "task_id": 276 + }, + { + "sha256": "d524f85f1ab6d99e7c617433346c738fef31d87c4f18f14a756e079f16186d9d", + "task_id": 277 + }, + { + "sha256": "822b142b1294dd9bf50e31ef00548b691bd57a5ce10ea23c39899ad12abb9e96", + "task_id": 278 + }, + { + "sha256": "8bfdc3cfb3f40a7e0dd37ff14ea1d9cc4ec100dd92b86d941ea30df203270476", + "task_id": 279 + }, + { + "sha256": "3703231f72aa5bc7bdaa8b8b18fa7888e6d13f8a63df01eaf389fa314d1a441e", + "task_id": 280 + }, + { + "sha256": "60c0524fb232761fe64317053466c70f694edfa77082ea73b9a49b135dab8cb2", + "task_id": 281 + }, + { + "sha256": "d2241d1a85127c1659c23f44041983b19ee5cbcb95a5edb69c36c0bb96b8d4c3", + "task_id": 282 + }, + { + "sha256": "5490874303af7a93a33964466fdedc374e05f7dfab331d4c6e56fcc65e51645a", + "task_id": 283 + }, + { + "sha256": "0c10c9d1d4ae8e1d29f82b18dfe0236dab44ecef6e5d8efa3317016269fe5e78", + "task_id": 284 + }, + { + "sha256": "13a8732809db094d657c5a3a00749c51ca10c8db8ea685b1c94cc34b33171193", + "task_id": 285 + }, + { + "sha256": "72af9eafb8501ec4f30b44727150964a569f93eb641b9e91c32d11f3e8ad73dc", + "task_id": 286 + }, + { + "sha256": "b47f48253aa1fbe2ffbd95e8a56846acc6a4790337ca4ca4ff67dbdca6ab1b1b", + "task_id": 287 + }, + { + "sha256": "15f880a7c4af4fa97e8b93df0a0df6cccc81bc93c12e252787e9781e71ffd8d1", + "task_id": 288 + }, + { + "sha256": "afa075279a3217cfce2462447504a298347c489c0f867008390e02908132a9be", + "task_id": 289 + }, + { + "sha256": "2060112b1440f412693512863466676c9ef1927748038e24e0530881aaa69e4f", + "task_id": 290 + }, + { + "sha256": "0835fbfe45ae0bec588efaf8e677a4713266ff7f33f5411983e00f21968b4fb5", + "task_id": 291 + }, + { + "sha256": "9c4497472121f46364681d46b51f65fa3a7e43521276d44fdd4184e8ce573bcf", + "task_id": 292 + }, + { + "sha256": "eb5394c2209d93efbc45a1127bc14626ccea628c01d3726840c73f623140a68d", + "task_id": 293 + }, + { + "sha256": "f6018aaf61c665641c8431592203e6515d1195aaf03084b62a3002bc8faba1c0", + "task_id": 294 + }, + { + "sha256": "bedfe0bbd89ce95272cf7700a98eac14cc1f5c2f6c8c7459d6644a06aa2e5cd5", + "task_id": 295 + }, + { + "sha256": "7deebed09b0a00080b7aadd114efdb7657ce3baecc019faec749be5c2d3b2de2", + "task_id": 296 + }, + { + "sha256": "e334bd45b69d5cfab68658b7fe59b2919b15e3b68c211eba9dbd102adcee8e0d", + "task_id": 297 + }, + { + "sha256": "e4536d3708847cdd3d0e298961a4830cd7f86e2f499a1062998d9e98789f36a8", + "task_id": 298 + }, + { + "sha256": "8d25b6ef08a8181222780ea1532dc38561062a14da45202a82eb0bd8ee37ff2e", + "task_id": 299 + }, + { + "sha256": "21ec0578ed01492eff92d7356d316f6ad27869a479c1f3149c60358c7c908bc0", + "task_id": 300 + }, + { + "sha256": "abcb9938ebceccdd541a15886dd56ba549c2b5ee386617dcbddfdb6a9bafe21d", + "task_id": 301 + }, + { + "sha256": "3fbf0e50f66ad0fd77b3242d9a0fc02108701491b86cf066dc053037181c4dd5", + "task_id": 302 + }, + { + "sha256": "d66028b612c489097eb709368c5b1864d0d7ba837867dac091105473542d1932", + "task_id": 303 + }, + { + "sha256": "588a90ee693ca462a2a30bf4afe3d02e94e215b776b6d932806a82f17b20a4f3", + "task_id": 304 + }, + { + "sha256": "599d437f9462248fc5d60f70537f9d7683b82dcde18422652a04e7c6e8620568", + "task_id": 305 + }, + { + "sha256": "4ec6aed035b183f25a548c65e5b26ce36e2f881620eb28f8dc8e922be6a25112", + "task_id": 306 + }, + { + "sha256": "74678316d4cbf5b251099bc648ccf8600bfad13108bcaadedb1e87f7391e8d15", + "task_id": 307 + }, + { + "sha256": "028339f380ad488f5567761626cac374e89f51f35dafc06c5f895345759eb07b", + "task_id": 308 + }, + { + "sha256": "80245ee3803cdf01b99d7ca0d0d45f7c5098d86df2dc435d6b9c6174e257845d", + "task_id": 309 + }, + { + "sha256": "ff880d9097ac27a2fd7eecbd39dd0e340c60ebe44d6027e247dd62dffc46d585", + "task_id": 310 + }, + { + "sha256": "8c346f5684e6b5944b02b2f3654a2928cdb50a7d33e4199e490cc0da388c6fb3", + "task_id": 311 + }, + { + "sha256": "58358d84fd7305a04e0f12a77fc5d46089c3e883a35fa2d3b5fb3f9644abb1c5", + "task_id": 312 + }, + { + "sha256": "43790774f6b1b218acb5e2cb4538f217b18995ddbaa1d707b8f0ee5c4e310d2e", + "task_id": 313 + }, + { + "sha256": "2bb31a2c3964fc250d32e5cd7747693954ba5984a43a2698e511fa8e953322ca", + "task_id": 314 + }, + { + "sha256": "f1643afd2b1e000fee72880930817911457318719ea04677edc79af740848be1", + "task_id": 315 + }, + { + "sha256": "c2f38ae4b96890e38e2d0bfe611db2da6bb93434447533c4d64cc7522196cc7f", + "task_id": 316 + }, + { + "sha256": "2ab1afc0469480799d6158f98e0ee06763b57dd84fa601d88b6ee706f412be60", + "task_id": 317 + }, + { + "sha256": "fdaf4c762208f99a4a540aa92c20905a33a01b997ad3625e3806c723a15fc285", + "task_id": 318 + }, + { + "sha256": "9ef0f43909b7de780b0ffdb21c92d33e51864e863b0fccc19c2abb7e072c12b8", + "task_id": 319 + }, + { + "sha256": "fe7cd8c6925f0ac78916814a8c8c4804912d423d41806580c96ac9b97027740a", + "task_id": 320 + }, + { + "sha256": "0198e293bf2b11a8c08ae3d8e76dc18bfa522d371f88dbbbdaad0b756b7efac1", + "task_id": 321 + }, + { + "sha256": "9619394380f45f5665e32aabacbe96d7aa1b80a8363e4021ee061b919678eff0", + "task_id": 322 + }, + { + "sha256": "2ff2380786352d4e1dac1e7803d08859f1cdb72f447266ee4c901219e79faa8c", + "task_id": 323 + }, + { + "sha256": "4e475ac3b9a4bb472188bf02ce53fc3662bcb43d90978493c36f6302cbd6ddc9", + "task_id": 324 + }, + { + "sha256": "f853790752583f64e086fa9167d179cb017bc71165b02a5d4057805f37d1a4fc", + "task_id": 325 + }, + { + "sha256": "5ec6c2b31ecaed6b80a6b7d7ca25ddc183094ed2971b045b731a13868eb9bb56", + "task_id": 326 + }, + { + "sha256": "40dd8688924b4f5d02d081e819b9e3c273b5ba9c58506f817eac51cfba08054b", + "task_id": 327 + }, + { + "sha256": "5d1fb48f0ca9f4175aac5c8d5439ce49ef46f22cd23cd1e24b2218d2a52943d5", + "task_id": 328 + }, + { + "sha256": "ed7f77aead2d8462672bad73e6e34bdc51b25f3977e7447b51d0374e1d371a06", + "task_id": 329 + }, + { + "sha256": "6945f99dfd906c96a80b742a3fe71a33fc8ae3e92dfbbcde5211c89d533f29fa", + "task_id": 330 + }, + { + "sha256": "e09fe0cc4eb796ab0174c4dd1a4e20f986dfe3766c230b8a1f2e9c6526739df5", + "task_id": 331 + }, + { + "sha256": "872a0d97802e3edffb25689f2d9ddeec1f67beb62ed243b0b8cb6e59020277bc", + "task_id": 332 + }, + { + "sha256": "c807de13315dd573f06c0cd4ec1d6712c61d85f5a321b2d5ca92dc9575b706d5", + "task_id": 333 + }, + { + "sha256": "4c0be99a89e67cecd3c3681ce1b06ad89d69c9dcb80ceb1902e601eadc88f1fa", + "task_id": 334 + }, + { + "sha256": "b906e34a5a242e20e424540611ced997fa24388dae5f07549613c69dbd836a0e", + "task_id": 335 + }, + { + "sha256": "5aa1a0b5a0f6a623b3993b590a5f4f4e2e6fd0d53d11860496e4b8377415e719", + "task_id": 336 + }, + { + "sha256": "c84e776c68c5cb666f778e5f676116f0190d9049d510de25c4eac7a9caf8a0f0", + "task_id": 337 + }, + { + "sha256": "cf0a0deccd52af8605f4df1ef7c10307443ad90458852fe5e24642086cd2645c", + "task_id": 338 + }, + { + "sha256": "37db2c5de7780bf16b7e638b6c8ae3be007f0dbad9c58dc709818b7bce3b727d", + "task_id": 339 + }, + { + "sha256": "ebe35ebda7a0e46c9a96ed2f2a1bcee80bae946c075d53720431d77d21752121", + "task_id": 340 + }, + { + "sha256": "ca0764e6c60a9b68db2b9563a408398a9b2d807f0f54e9ded016c60079788f6f", + "task_id": 341 + }, + { + "sha256": "77dfa50cafb3690ca8156c8b0070b0da8982be0c3ecfa125f404c01588480bf8", + "task_id": 342 + }, + { + "sha256": "78a922bfac4c984433ae881316475f9fa6cd475647a632793e0e9d71ba543d22", + "task_id": 343 + }, + { + "sha256": "00b8566589e206d13beedffbe6c2eeb4f567f29c2787302d13699ce4793ca810", + "task_id": 344 + }, + { + "sha256": "913fa20da1f77c6cca78fa2f10425206f132af2fde9faf7b6449bec8fd8db024", + "task_id": 345 + }, + { + "sha256": "8a93ffc38e9de1708863464313a18914b233cef42d84973231263c6371e287b0", + "task_id": 346 + }, + { + "sha256": "03707dd85fe7378082ea92eb0443e2d5fbc7128215a23d4eddb3e86542b746d5", + "task_id": 347 + }, + { + "sha256": "d141b5e2afaa5926bd1d607be3b4bdd1eaee1b4ff0632f45cee48909f43191d3", + "task_id": 348 + }, + { + "sha256": "4433bb86b5b45ce189af3ff4689f2fbd20ab2033dac61b4e06fe7d79840f32b8", + "task_id": 349 + }, + { + "sha256": "15e99544384c556e20f477bbd4544dac723f6445bd74ea34be1e6fa45335e465", + "task_id": 350 + }, + { + "sha256": "25eb09f36f9af217505397e3d8c2f4541caeaf8b14613b41b00b51d4415a0b50", + "task_id": 351 + }, + { + "sha256": "bb3ea61faf6d2daeb717914d6ab85279681b799f7fdad53c1a105476ad97b37d", + "task_id": 352 + }, + { + "sha256": "5c261a73d5152778f8029bde8f02ae280c997774be6f57245cb99dd24cbf0318", + "task_id": 353 + }, + { + "sha256": "49aa39ce941a3521e4c88a04e89c403859b034730db2c21d4dbeaf3c3183572d", + "task_id": 354 + }, + { + "sha256": "0a81815f31a8c0502c4455437f79af1c3b5b28f4bd99c7321adbae925f5f5ea2", + "task_id": 355 + }, + { + "sha256": "a106a9f410cc8f6a345a0639f06e8b6c83188ee6a21e79792d95e5419c3a04f1", + "task_id": 356 + }, + { + "sha256": "a7574ed518bb42a9f894897873e164586787e96451e5d78a7076bcb63e23189b", + "task_id": 357 + }, + { + "sha256": "edd2e0079d54976ddcfe9b20b6b5919906cddf6e94f5500c4886995db78276d2", + "task_id": 358 + }, + { + "sha256": "d157c11ac76aa251062ee6b1fd108dac7868843530d82e7f167a526d27e893b2", + "task_id": 359 + }, + { + "sha256": "9aab2e8bfdd4f3b137b4448eed06259384ba45692a9c058c37abda10061ddfac", + "task_id": 360 + }, + { + "sha256": "7ad6cb2cc83b07965ec36dad914f07ea4e2310712edd124e319b77e3ef76287c", + "task_id": 361 + }, + { + "sha256": "cfe68ec821203a89385791481376e73690c9619b1efb1c9989d04a1180dca411", + "task_id": 362 + }, + { + "sha256": "1d9c11fc7b028311854ecc3974da899e2660b31d4fd1fcff2c97cf1ee3a0cd2b", + "task_id": 363 + }, + { + "sha256": "942402c8d6ab5c8dc057df573756222f7745878e239c0d0ae2af0266d179b341", + "task_id": 364 + }, + { + "sha256": "7f1872b2a89fc624016b5dfc33a04b02241bd12a611a9f278bae7fa860320a88", + "task_id": 365 + }, + { + "sha256": "210e8e3f73c48d3357f58926c645857d6058a68ce1b75fb853e90066f799efba", + "task_id": 366 + }, + { + "sha256": "be7456d2ce42dd05c8061f226b98ebf8408c45ce1f89f188ea8f772eadfba369", + "task_id": 367 + }, + { + "sha256": "97d36952d01115c3b56fb45b6648ad273129e18391a586c5c2e8e2e48670eb8d", + "task_id": 368 + }, + { + "sha256": "8405b6d90b1688225a891f34dfbba0581dab88ec939968d3514c0af832681135", + "task_id": 369 + }, + { + "sha256": "741ff253a563fd3a46736ae470a5e5636f4058861e7b44187f3fee8609d0b2c7", + "task_id": 370 + }, + { + "sha256": "dbb6bf44b681816812d24e1c85ef8586bed38cdc72bebcf135191851313f8751", + "task_id": 371 + }, + { + "sha256": "d77de30dfb38d286a7c0dc7dec657b8deb688b2b3ea4584c66c7a4b3fccaf09c", + "task_id": 372 + }, + { + "sha256": "f89131c0c70c00526ced382a6d6c58ea41dd8d892a851aca6ee89d9572a132f5", + "task_id": 373 + }, + { + "sha256": "352c2126fb8c014a14d94a19905ade9bb210c1555c5ac1cc8094ee5375715d98", + "task_id": 374 + }, + { + "sha256": "e485cf48124dbeecd2f3a01150d5dde008cdf88ed1efcbd975c3c1698b72f87c", + "task_id": 375 + }, + { + "sha256": "c721c557341deb2cf72c38c7217793c15736b55e8450c1e11be69b1ba8c6c666", + "task_id": 376 + }, + { + "sha256": "dd43cd1c8d6f20da3fffe9a9ea873c65193e9dc997a7c7c3e611cb00473cc025", + "task_id": 377 + }, + { + "sha256": "12d431116661022dd07545451d9d1386445ec10993213984b29477b03f7d62d1", + "task_id": 378 + }, + { + "sha256": "8f1c3a8f42992424e6aa70fef206a804d9b78d03d8c2948631e6b7239a525bcd", + "task_id": 379 + }, + { + "sha256": "efa85785210bba60bc0f521b47e1d9e675e05aba2cb5f21cf6c81a27e18f80ba", + "task_id": 380 + }, + { + "sha256": "c12b111a98846bec34098e03974c9f2c0c9c1212f6e4514875a4f6bc86289233", + "task_id": 381 + }, + { + "sha256": "dc9182879b66972afe02eeabd15113ab7e2f1e1568e0e68db6451773b218d64b", + "task_id": 382 + }, + { + "sha256": "9551f88c9d8b8d41d71ce0ebd47ee6643bcd2409a8369059c58f3953be694837", + "task_id": 383 + }, + { + "sha256": "628d47d0b2a72f70e18edfdf85ad30fe9a428f15b128ed0efc0389ee102cb327", + "task_id": 384 + }, + { + "sha256": "a6d73e35b8d8a66c7924c1dd38ba57a884667055995d626e51007e038b2a832b", + "task_id": 385 + }, + { + "sha256": "e7121637308f5b2a366b793ed7253037801f5bacc9a3b40e7e2e13eb2dd16e6a", + "task_id": 386 + }, + { + "sha256": "54c18d01847c67ae6e9c691c36eeabfdfc6b78f38eaba25ea59bf2a22f18edf5", + "task_id": 387 + }, + { + "sha256": "250d665208d3771cb6f8a8ddf6d5ad6ddfb20317aea19280715c67e782cc6607", + "task_id": 388 + }, + { + "sha256": "ca0f67414b9f8ce4ef2bf351bc2be79b22dfdcb8399c33ff67f13dea9fe5f9da", + "task_id": 389 + }, + { + "sha256": "f803403e005d94ec891028012aed6c4d87c27f5f23dc366e981317659ba1ccba", + "task_id": 390 + }, + { + "sha256": "bc7a033d6f5a7c9a155252d76064d0611a365af7d3d0c98e9ec948e066f231d4", + "task_id": 391 + }, + { + "sha256": "98ea9707d826c34a41fe877ea35947409c416d2d22913c6b4e2b287433648cda", + "task_id": 392 + }, + { + "sha256": "ff435b6557f27e35771b1a15006d786d52c347cabdf3027be46ff6ab116008c4", + "task_id": 393 + }, + { + "sha256": "09d6585833400d2e122558b6d80a57c90b9c1b9767f1bef1e6a6ba98dacb7ccc", + "task_id": 394 + }, + { + "sha256": "9c5bb4295a2e879e18d52d71bb0b951f87a7c7a4590c436246a852d75eba868e", + "task_id": 395 + }, + { + "sha256": "644f1bf7b575c915371a3103337ad232f56b06c3b79ccb2ac0384cf10d940657", + "task_id": 396 + }, + { + "sha256": "8e4f8ab9760c117a1824c6bb7db20bcfd353ab20e3d595a495c95bd46b357b33", + "task_id": 397 + }, + { + "sha256": "d340c5cb4c7685a2ebb9e01c8d6d6fa10c7a965a4f378b1d661b33d0331b109a", + "task_id": 398 + }, + { + "sha256": "cb7d4d03da7adf20c73fc7206d0be2e65713a82c9a47bd655a26ae94d95f55a8", + "task_id": 399 + }, + { + "sha256": "ae81c38b2c00d00b2b7040e14eaaf70636fa6a20b92572ae859b972de982a3b2", + "task_id": 400 + }, + { + "sha256": "577603435dcf0022bd5d05ef0235ca3afd73f08a8d599a01f35bbd8e551c7aaf", + "task_id": 401 + }, + { + "sha256": "0e64f3ea7246c08db578b365ac79e3d773cbac74f0780752e3d259ba4ebee3dc", + "task_id": 402 + }, + { + "sha256": "7e6412995a40210f133f023a8cec4b50100c3f36862106e7b36965805058b441", + "task_id": 403 + }, + { + "sha256": "1e1f2fde9648b40cae968eeca68df7910b8cd86a593ac4a62c740c81b1922b26", + "task_id": 404 + }, + { + "sha256": "70071e06087cec4cf88f1cc16de6929fadfe9b33889144aa936c8d3fb3075a5a", + "task_id": 405 + }, + { + "sha256": "6bbc422f02bf1e74e2496ea78055528f7f1822348f99af8e1b0871139c3c4f78", + "task_id": 406 + }, + { + "sha256": "fdda6befb7380acb6c3b6a49e6ea960e6ddf444fec1747516ba431d6e77a96bd", + "task_id": 407 + }, + { + "sha256": "756569fec5d3f8ce6135e4a734867bf0716c8d658b2ce6f6e2d57c63d24c6058", + "task_id": 408 + }, + { + "sha256": "6b5e83867ba953ea836aea2871337cfef6355c144684192167adc590b10b3b97", + "task_id": 409 + }, + { + "sha256": "59c5c07c575bd15f09a7d57b8beccb7b1090aa344b18ee4db42deb9989e96ec1", + "task_id": 410 + }, + { + "sha256": "0179c29cbd42c2046ac7dbc0c68f56ad5e15d4340a7e5a95ee1c691903536792", + "task_id": 411 + }, + { + "sha256": "4e30a719595c2b8a7280b83f40522b2e5078da5aa456f1adb221f167d48731d4", + "task_id": 412 + }, + { + "sha256": "55f1e43d53a66428eaaf18eb02a27a0285197a8286fae13d415feb5e8957c51f", + "task_id": 413 + }, + { + "sha256": "aff705795c68ce211052e44e3bfd57214b565ba33db47f94aadfab179c5b47f5", + "task_id": 414 + }, + { + "sha256": "1158cda6a946280aa807ff5a9759f2d8cb1baee193c2d46739d6e1f4f2eefbeb", + "task_id": 415 + }, + { + "sha256": "5b7ff6ed0eed5ea54ce95eb5741e002e2d203e85371729f71a45f48c7c7f2e02", + "task_id": 416 + }, + { + "sha256": "60e4a46f78a80395c234a293bd1e8d8bd06ebcfcd601aefea4d27f4e439e6f76", + "task_id": 417 + }, + { + "sha256": "d50a6919f4a5e0910ade0c1b959c6c391fdacb4a9151b02d84e41e8802763f16", + "task_id": 418 + }, + { + "sha256": "79a575df50f4d9b220729b0a6fe0fd3cfc855509ef4fb4698314bf52aa1784da", + "task_id": 419 + }, + { + "sha256": "7f13a7500a59d696ae8904ad214000ddfea1b186bdfb42022d4e3be17456d847", + "task_id": 420 + }, + { + "sha256": "d5e210428d0578963941379cb2242d9aafb46fa4d3965602f45e6323b6f9b63a", + "task_id": 421 + }, + { + "sha256": "0eba342cef414d69277ee052d916efef69268d38b1733e936e005f7c299ca07e", + "task_id": 422 + }, + { + "sha256": "3fb84b23ea74883614a27f2de80b7b578d9badb2a0e9912b97f246e391473fe7", + "task_id": 423 + }, + { + "sha256": "2e87915df4b259a1f1a88a0869cea8d521845e3f5e690294fe80d73c26f3d063", + "task_id": 424 + }, + { + "sha256": "8f2fbbc3a4cc308fb9351ad9ebfc91f7c8000a9250161c8b46525bf016ec62cf", + "task_id": 425 + }, + { + "sha256": "4a39015b60cf1837f16fe9b41ed76c980db660173ee69f114feb9022fdc2c401", + "task_id": 426 + }, + { + "sha256": "572ccd8d94c819abcc887a5127a5ec7d2818b6e9a8bcceba9988af9df06e1164", + "task_id": 427 + }, + { + "sha256": "82fea8cf0fdacc58cecb1c057bf0b72cb89774ffdf2234f896ac00734451317d", + "task_id": 428 + }, + { + "sha256": "76c5d64da0e111299a737c30bfcfc5b2f312b5153afc32c286bed167f44e375b", + "task_id": 429 + }, + { + "sha256": "e0e267a2ffd8f4b22deed3e17fdf132b12b57e323570ac65b50d4f3460de6e00", + "task_id": 430 + }, + { + "sha256": "29dda656e6906c514bd0a24c86e9906e91e5e6252efed721fb9fb1f30201a494", + "task_id": 431 + }, + { + "sha256": "0ccfadde33c2cd7cc9c7297cc9da7afe3c33174cf8374c9eb04f3017ba978d2e", + "task_id": 432 + }, + { + "sha256": "91b57007b7ee1c1aa97f8d14fa958cd3942ef326c7dc9c62fd597e2dca55dbcd", + "task_id": 433 + }, + { + "sha256": "d52408b1b8082bb757ab1a713f88c131a03879c264756130ff4bf03a80833c12", + "task_id": 434 + }, + { + "sha256": "5ae2bc49b5d418a5b73f2c2a2172f3ff83b60736fc4e81a5d9fc8cd43b6f097f", + "task_id": 435 + }, + { + "sha256": "90e16b1e649208a0a42b9521df66a77d7ff2b8bbec2fb1a698058c04df2500c4", + "task_id": 436 + }, + { + "sha256": "c7cc2d93c25efc0e25f9784d015df7a5e06f0c1334bec875c2af42fe7fc63f6f", + "task_id": 437 + }, + { + "sha256": "0df5effaa4842e28c0ac03d3e06c50af4f5bfcb71ff68e1ef50d359f89a49afe", + "task_id": 438 + }, + { + "sha256": "4aa5251f2c99769e18869d45ae93d80bc4f7f8068e413b605ea67b2b41da08ff", + "task_id": 439 + }, + { + "sha256": "42f060f45f76843bbc09ff282aab9b1aee642763d4f8b3688d3eaf3f423788ee", + "task_id": 440 + }, + { + "sha256": "b964e8555b10a0d7308b677019b44aaf1e14879a4bcbf8deba64a2029956f2d1", + "task_id": 441 + }, + { + "sha256": "6a0f47baef1ae5e919708989d28e2f350009bc69420421e14476a45e580bd04c", + "task_id": 442 + }, + { + "sha256": "32274088c83531f0957d90f1ef2e9c078ae1c03f8bd315d60e125aebbee1aeeb", + "task_id": 443 + }, + { + "sha256": "b64e4e008307c3509c5a60f5994a158dfab19d6d0ad7d7e1d034305651da2637", + "task_id": 444 + }, + { + "sha256": "f782f60c38b93e5dfbb4912c7212a5c32ef3c57919f76990118d358ae704e67c", + "task_id": 445 + }, + { + "sha256": "c3d2acf33428d45671dc365e0f4b7f41ea5c50b82cf6033417e8c58360c088c5", + "task_id": 446 + }, + { + "sha256": "8f4bfd5f3f3c0fde6d216d5722574fe46346a4cade4836b27276d3adfff6b32d", + "task_id": 447 + }, + { + "sha256": "bcba6b1d1d8f953e142c3fc58d63845980f3ee05840fd0cc44f2aac77bee1366", + "task_id": 448 + }, + { + "sha256": "6001737c658d40251410d05ae17a76596b0b5a473f475f3f9bce2dde728ef110", + "task_id": 449 + }, + { + "sha256": "2f262a1fe0aa292d1c05c6c7a54b11497eae618bc99c212d2597f7653fc540c9", + "task_id": 450 + }, + { + "sha256": "35e752af37ccaeee3318768767c41a4d98c3100de8b3d05191bde3ebe39596f5", + "task_id": 451 + }, + { + "sha256": "9794bdf965e51e9775d2953c06e87ca82a61b280009eff3ed7dd0c1b28bddddb", + "task_id": 452 + }, + { + "sha256": "9d7e39f13669d309a8628459e32d913fb39788cf3cc7baffdd208caea37fa85c", + "task_id": 453 + }, + { + "sha256": "66928d9b49e34c9854d364e49ed4b0bbfec402e851cb28980b8b63704e30d059", + "task_id": 454 + }, + { + "sha256": "7eb4f7077232b544b25930c0f174008271b47397497655149f68fdd7740d9b50", + "task_id": 455 + }, + { + "sha256": "a52a10792cc945ec9e85b393707ce5a90f7cb09f6badf59c6dd440c79d3a6ae5", + "task_id": 456 + }, + { + "sha256": "15c11f878ed7340009751e41cb45116318009559105b11595a3e00b5ff25688e", + "task_id": 457 + }, + { + "sha256": "bc396deb81854e793ac1587a32c0b16b24ec40a97393b4384680a954d726fd9c", + "task_id": 458 + }, + { + "sha256": "05d7b8a2f7bcbc1e17e62759d12d37b35ccbf613fe441634d6e676240107e24d", + "task_id": 459 + }, + { + "sha256": "0b54dd55c706306ae8e621501912717e681c08de7eda1e0cefdcf830ca554937", + "task_id": 460 + }, + { + "sha256": "738535c0d1745f9a1ff76633dd8682058b020c56f50e80e1f919efadef1707c9", + "task_id": 461 + }, + { + "sha256": "57b8abd98652d60594a67a4d0777cb031c62c7eb6b19c1f86f956511946f0f76", + "task_id": 462 + }, + { + "sha256": "cb1fb2e12395a713434e064edafc76c981dc68f7f9fee112e6959036039b459c", + "task_id": 463 + }, + { + "sha256": "984987d382d3cb45d6cf5790ce57e13c9ea0f59fff56df420545e0ab0834b0c9", + "task_id": 464 + }, + { + "sha256": "794be688ff200c8c76627cf673620f58499723f3b9b196d7a6b4128ad95f41d3", + "task_id": 465 + }, + { + "sha256": "d2579a273aeff89a7fa13d785457817f7421060f4ce653fb71d5501ee2d0d4fa", + "task_id": 466 + }, + { + "sha256": "24c85b3c08e271ea0db463dd1d844cafae596388961a55328fbab8be851c843c", + "task_id": 467 + }, + { + "sha256": "7042d4cc071d0acb5562591a94b54ded66ef31eb8f79bad3d48ff25f4cffc641", + "task_id": 468 + }, + { + "sha256": "12ccda53757d733485a9b0154fb3c93ffaaa4c54d88931e7166fbc935c89ba13", + "task_id": 469 + }, + { + "sha256": "edbcd0153b737021feb8896a0275d0459d90a07e3b3457c762e5a714107f6078", + "task_id": 470 + }, + { + "sha256": "4c811034d89bfcaf14b54b0bddf449cab4db37b3c66f197b8f498a5b68062beb", + "task_id": 471 + }, + { + "sha256": "b47647824b3ce2a37fe28dd071fa31e881a2f66d7f54f43bd95c6b6a5b61eada", + "task_id": 472 + }, + { + "sha256": "6f036ae099de2abac390a0c7cbbe7c0934c92e3c2888f652c6facbaafae17b69", + "task_id": 473 + }, + { + "sha256": "afc5bb6b9220189f56fe9a703602093420ff79d98ef812ffc1eb25506f26b414", + "task_id": 474 + }, + { + "sha256": "085441b4c11acd255ee4e7f3456405593012a2a494d742233f3b14f1b0e78337", + "task_id": 475 + }, + { + "sha256": "804ec9d1ffe208cbe60cfc729abab380c502c5126017dcf376f7c8781e7b852f", + "task_id": 476 + }, + { + "sha256": "be92e18e21c9526de06052f9d9b2a0ce26a97e2fea74bca165bb77adeb38f238", + "task_id": 477 + }, + { + "sha256": "1a3abf4b1aab9f0e25b34a5c39480b54b66e6ced2231ecb79d4247b6a801b29b", + "task_id": 478 + }, + { + "sha256": "d7614c25225fc25c1b0a7436854ef495a3906fa49f4f3d48798e4e804da04646", + "task_id": 479 + }, + { + "sha256": "906e6ef7b33aa2335fdfa4d4ec2a5b15ca00138d60580998e9f6be5c7153ec22", + "task_id": 480 + }, + { + "sha256": "eda124daaeb803b2df1f4026d47d51ad3dd086869265dc7df82bce9391dcde0d", + "task_id": 481 + }, + { + "sha256": "3f5bce00a9180f1cc74ca5ba3ea74e23d21c0d50f4e63e7029c46bfce3bee24e", + "task_id": 482 + }, + { + "sha256": "3b08049b726c50b7b6140e3e29aebc7ffbde4b3a7377181b3937b3e72197b657", + "task_id": 483 + }, + { + "sha256": "f020b03336fdabc3e84987d913b07c2b51d427b865f3cff92095c08b62b5fac9", + "task_id": 484 + }, + { + "sha256": "a20341c859c43d2f5d35236b81d4c8c1b4275b2b24223a2721c64f7fb1458d87", + "task_id": 485 + }, + { + "sha256": "9e851c71920a7f8895737f873dd1edfe858f2e5ef70bc27bbf5fc4a3bedaebbb", + "task_id": 486 + }, + { + "sha256": "1671e7eecd9a87f428209990045f310f94a93deb18c72670b9b8fdf3ef2390d6", + "task_id": 487 + }, + { + "sha256": "b1196a28fc4ae41522543435bb921b5c78d5ecbf29582eef08c9b153e1a65070", + "task_id": 488 + }, + { + "sha256": "6a609aa375933ec2b32f40860a04dd500d92be3e525f0f603786a1f88b8ff7f8", + "task_id": 489 + }, + { + "sha256": "764315278a99dd3ef7ff7e6c00afdef2b9a3a24a7d3bae43825dde160e72a61e", + "task_id": 490 + }, + { + "sha256": "59fe5074d624e7a1b3a538f78afef34d7247e86f60c0478cb53f4faaa2950704", + "task_id": 491 + }, + { + "sha256": "c6e907a3d108eaa8a0d4f7381e5210ee74f030f05ae9f81f9826d0fc7d589906", + "task_id": 492 + }, + { + "sha256": "84cb0e1d47122c054e5a084c77ef0962d06ba45025b8918d9eb7e802ec68e08a", + "task_id": 493 + }, + { + "sha256": "dde8e7712b499c0b4cc14bfe1708a98825c6c14ebf979664a7eedef2e07c4d8c", + "task_id": 494 + }, + { + "sha256": "aab4144e484a8525e196ef8ddefec5b927f72c9a089c189f2fd4a845de9eb450", + "task_id": 495 + }, + { + "sha256": "189642f6c1b43939304530fda78a9b92e8980d27e1589cb024c456c20a8445c8", + "task_id": 496 + }, + { + "sha256": "0ec7d8f5629efa5720389ca52dce888e5d5cf76f66626c290b98ae96b7314d23", + "task_id": 497 + }, + { + "sha256": "4aafd44f85d8fa3ba6a35237b4a8b0664ebf4d70b858d675b4c28552da6fa9be", + "task_id": 498 + }, + { + "sha256": "bbe93b89c36edee65aa8734055d3860ae37d5bb8f1610f4c35d024ddc4ea0cbd", + "task_id": 499 + }, + { + "sha256": "f184ca62b8ff80a9fa5229f24948dd1f0323ce551d6f1fb2be52c7befca787a7", + "task_id": 500 + }, + { + "sha256": "1e1f67f6795affec921655e9c5ce1f1a0a88e5cbf5246ad0bad6f225696b4d72", + "task_id": 501 + }, + { + "sha256": "7412ea1c47189b902527f543e3d7f5b545f23cc3f9c7d42a30d84e965526c2d5", + "task_id": 502 + }, + { + "sha256": "e7cc091a492314b808f6af95689f6e5ff5d5337494bf0a5c825c414d90f5198f", + "task_id": 503 + }, + { + "sha256": "8b04ed8ffb03266a82cf671c0a31ed72c82076a1e2c09f50455482e80f4b67e4", + "task_id": 504 + }, + { + "sha256": "0e7d4d0b672b93a0fdbbd5afac27e2ac7eed5c27e967fe79b74a5c5425fef0fd", + "task_id": 505 + }, + { + "sha256": "24d81a9357afd3cada0e73f2bef736c397261dc2099465c5553ca5b041df3493", + "task_id": 506 + }, + { + "sha256": "f98aa30f2e58adf274be27279e70a5912952fd736bbf4383e5a0fa4f1a2d13e1", + "task_id": 507 + }, + { + "sha256": "231d6d8ef2e3da18cdac979b4c28ed8d7f3dcaa27094af862101de56fbceee00", + "task_id": 508 + }, + { + "sha256": "5dff11bb143c42c4100e35322bf44a10bfa2a1ff0c88c7dc23d0d4c00b52b94a", + "task_id": 509 + }, + { + "sha256": "6a5aadf88bdd30a3448fb69d1e3b40320ca0a7adb8e0c42c1d1745e8b390a691", + "task_id": 510 + } + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": null, + "source_split": "test" + }, + "dataset_manifest_sha256": "060aaff7117dc47af6c01253a912f34b6956241c336bbc7216e73bca8624d2d4", + "model_id": "Qwen/Qwen3.5-0.8B-Base", + "model_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + "repository_commit": "44d75a2776fa36441e17cc688965c9825c4c1a1c", + "token_manifest": [ + { + "code_token_ids_sha256": "ca4074a90d9924be4e36b616edb1fb00ec12a5eaffefb9a5813387c761c50e1e", + "code_tokens": 98, + "prompt_token_ids_sha256": "ed99353e8634f79ee55af2198182202f7bc84427c2cddc8fa4d3b3218a39fb67", + "prompt_tokens": 88, + "task_id": 11, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "7a07bffbee0607ebba8bc719eef65c351d89ea0085ed1c92059ab5834c3d3184", + "code_tokens": 19, + "prompt_token_ids_sha256": "41b434ec1ea4056dcae6131dbaff63088c7c6833fbeb8a77f7a5b79cb51a7e7f", + "prompt_tokens": 215, + "task_id": 12, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "221fcb82068107d797be46f12073f32e9dfc949992ce8866a7607e2cdf599f02", + "code_tokens": 39, + "prompt_token_ids_sha256": "a39d1e784403627fea22a54a99d9e2019cd244fbae7a651745ba75453137f7f9", + "prompt_tokens": 239, + "task_id": 13, + "total_tokens": 278 + }, + { + "code_token_ids_sha256": "44956cfe26f8b9cd8a218f2820997aeaa1f378147a54093cc84ee699320a882b", + "code_tokens": 24, + "prompt_token_ids_sha256": "d01e4094a64e7225e4851ec7ce56a839092be6086c65dcf43aae5a97197c31c1", + "prompt_tokens": 87, + "task_id": 14, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "4a7d3767866cd6d449afa9a55314639dd9911a22c6b582fafd7889ce784b0d51", + "code_tokens": 25, + "prompt_token_ids_sha256": "b26b6fe54ca0665182b853eda401a9f250df526f9ef89c04db06ecbb201e0093", + "prompt_tokens": 113, + "task_id": 15, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "3764692433fb372c33425f9a53ad24eb14e5db843302adfa5b5b0a8a53c951e8", + "code_tokens": 55, + "prompt_token_ids_sha256": "6ecbf53756b49711a63907e2332b7ba1c4f1109dabd7a407de2d664f8bff5e57", + "prompt_tokens": 92, + "task_id": 16, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "0c4a09601bb3445d380f00408d51bb28d3e58451474426fa7e64388426f76934", + "code_tokens": 16, + "prompt_token_ids_sha256": "15b1a6bc09a362631db60b7b45984c6f30e4cce2649aa5dcbc6c62873dcdb694", + "prompt_tokens": 68, + "task_id": 17, + "total_tokens": 84 + }, + { + "code_token_ids_sha256": "c0728f0361691ca53a8576262c93c131ac3c0674ad5e839ea8f0b887e17b804a", + "code_tokens": 199, + "prompt_token_ids_sha256": "689290e79a13cd85182e1e53a4cc5c49911dd6bb0c8a29b3eef06db4cb8034b8", + "prompt_tokens": 102, + "task_id": 18, + "total_tokens": 301 + }, + { + "code_token_ids_sha256": "f66a26c66f1854dc46e3b08a790cb9d0df0ab49128a51e8be04e69ee0c017df2", + "code_tokens": 28, + "prompt_token_ids_sha256": "e9b78ca3e0110cda78ea2034bab7b33dc5f6d48fb563cecb1b6d1e917acf1673", + "prompt_tokens": 101, + "task_id": 19, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "ae9c0dfe9a3c6f29c3b4fbba71aae76fad03543b6e6e12d4e34a26ae50c22ec5", + "code_tokens": 87, + "prompt_token_ids_sha256": "f4849f630e91bb0427fd5ad7e120f8b34b1a1a70aeea09b3a60baf05aa29f92d", + "prompt_tokens": 80, + "task_id": 20, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "20f4393df1dbea6bce755816285ad46507dd58921703dcddbdded90ba91e085a", + "code_tokens": 35, + "prompt_token_ids_sha256": "b91035c6853da4eb360e2a07ccb945b7af361dfc3f62ad66542394454629331a", + "prompt_tokens": 108, + "task_id": 21, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "96c2f4cdf35461af186b19a7eba036772a5470849ceb4c32ebf1b4a2f9ca145e", + "code_tokens": 62, + "prompt_token_ids_sha256": "941b1bd73064348187aa42b1c4b840da4ee1b598de449c02b13c97236f566bf4", + "prompt_tokens": 113, + "task_id": 22, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "2acd5f2c91c0cf905a0c4d65b15711b63a05c1ec3ea03e3b40fa3e447f9c3a95", + "code_tokens": 58, + "prompt_token_ids_sha256": "85816f3811b40926707769c69129140f7d1183071503dbdba043813cd6367983", + "prompt_tokens": 149, + "task_id": 23, + "total_tokens": 207 + }, + { + "code_token_ids_sha256": "f6fdf9cb53b2c54485de4e4bdba6ecd540a2e0a81e11fefc5b6a92be6961d77d", + "code_tokens": 79, + "prompt_token_ids_sha256": "dfb771f01c701c76191c084dad9b1f2907fb2fc4b78c1a2aa7c29c3d8b26fbd9", + "prompt_tokens": 87, + "task_id": 24, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "b1fd9b33ab2d8e2dd7e7931a44b6f92280b1390875aa6f120df42eeb92a08c31", + "code_tokens": 58, + "prompt_token_ids_sha256": "0af9bf0df6730c0bae8a2c97fc6383f3934c8e07e3731d186863dd5b04f675a9", + "prompt_tokens": 104, + "task_id": 25, + "total_tokens": 162 + }, + { + "code_token_ids_sha256": "db7853bd6a3ae3c0e856a916b9b6803f8dc163d4bfb7a0795dfde8fe535291c6", + "code_tokens": 48, + "prompt_token_ids_sha256": "3f29dbeb6aff516a38f8c773d454d279fe014adb5ce2b4cce51ccfb96ad66a71", + "prompt_tokens": 142, + "task_id": 26, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "fd1309c323771b967dad115ece05bd67273d805f680d339f4c23dfdcd9c27dd4", + "code_tokens": 37, + "prompt_token_ids_sha256": "ab1dd51def39597fcf1d5ff1dfe6dbebb650d0eabb7ccad55efb1dc117f22faa", + "prompt_tokens": 116, + "task_id": 27, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "f4cd18271b3fafb0029071d6e5b40458c6a1810cc42aeba39bde3ce36d2927d7", + "code_tokens": 61, + "prompt_token_ids_sha256": "b2cdefbda762a3291840c7a63a182e116a981c65ae85b490c02d7c1a324746ae", + "prompt_tokens": 80, + "task_id": 28, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "8264181d8b34af2d9cbd07da5e4084489d6ab91622bad459dd7bd0da39c73991", + "code_tokens": 77, + "prompt_token_ids_sha256": "0bdd84ec7d91f960f2ee48ce688e7099459977d2ffbe014bdb7bc8c2ef047a90", + "prompt_tokens": 137, + "task_id": 29, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "a7483fdaddb05b76d4a2be4bd59083d949ca756d9798897ca8ee1f227148873a", + "code_tokens": 98, + "prompt_token_ids_sha256": "de38a7613d5d82cc419bc02c0acc47a14a7d0369135adf20825aa1506cc0d81e", + "prompt_tokens": 95, + "task_id": 30, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "ac6d875ff1fad87c9754785301d1d6157743fd9c5f066d864db10465934f6a07", + "code_tokens": 162, + "prompt_token_ids_sha256": "012c336cc3b3b7ce9e3765952be744a1e5eb0f71dee14ff538402c340b0dbd86", + "prompt_tokens": 316, + "task_id": 31, + "total_tokens": 478 + }, + { + "code_token_ids_sha256": "10e3a04899828e4c297b205675e770b686b8972ff197e45a36ed0ae1ec08918e", + "code_tokens": 103, + "prompt_token_ids_sha256": "4a9d2bf3169137f973c1378d5d0e7549fbd97d4e06ad15c5d0dc40729464b122", + "prompt_tokens": 81, + "task_id": 32, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "79bb64818ce913fb005b1128a6381acb757468e48a4a3c12bfe666406b86ff8f", + "code_tokens": 73, + "prompt_token_ids_sha256": "21e72dbb837c0e026946f0989c9a460e3463668805005c4b79c7a7a4f1bd53c2", + "prompt_tokens": 84, + "task_id": 33, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "084005ab9510b7e85c731d8b89475152925bcca967173b05ec5552b539a56599", + "code_tokens": 119, + "prompt_token_ids_sha256": "dcc46d60716f9dbe17e266f0ddab1b550005733b6288092e0c8acac261d1fa64", + "prompt_tokens": 98, + "task_id": 34, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "d4e1f618f7cca6007e0f09abd1ae5a67f53af7276e39ea630cce5e8b65a159b8", + "code_tokens": 17, + "prompt_token_ids_sha256": "3662e296713e8f1edfd7bbaaa8dd668c0dbb5498232b35b3e18ea32d6eabcca6", + "prompt_tokens": 73, + "task_id": 35, + "total_tokens": 90 + }, + { + "code_token_ids_sha256": "200f6db7298be70ecc782e8d0a30761a8d562e2a4ea2e7412f0ff0ebc7644832", + "code_tokens": 56, + "prompt_token_ids_sha256": "f347eb604a7f9da27f6104b7e432fcdab391c2db3b8b3ffb51799443b0e86d3e", + "prompt_tokens": 95, + "task_id": 36, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "b15d7bf5a1c94f21a03001d4aee2cbbb1bf592636ce2ca6041019195a2dc5dfd", + "code_tokens": 57, + "prompt_token_ids_sha256": "aa474b2c164d94f4ce6c82da5e9e4cf4c5bdb9cc66614a584100f3c204d8e857", + "prompt_tokens": 217, + "task_id": 37, + "total_tokens": 274 + }, + { + "code_token_ids_sha256": "a48da0e5a048b4f9d6194945a3d9c526ce0dcf5612cf75ca0ced9435efc8db00", + "code_tokens": 63, + "prompt_token_ids_sha256": "897a4ee4fd5f6e307ea63bb0b3488b3f675c7b93d2c76e1702385e23f36ccc49", + "prompt_tokens": 114, + "task_id": 38, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "a6f227b2f487bd7b3c0a936e4be584ddd5a23e847c718170591325a654a6cab7", + "code_tokens": 200, + "prompt_token_ids_sha256": "210bab400ea5da5536cc28e598d53dd54f20b203f68f7777d1756cee5b059d76", + "prompt_tokens": 93, + "task_id": 39, + "total_tokens": 293 + }, + { + "code_token_ids_sha256": "377f905274a38c1b43b4d4dfa9c12200a1aea0ebe52ee934125093118e1ad4a8", + "code_tokens": 30, + "prompt_token_ids_sha256": "ad2cee03328388d1f57dd321d28242f388500e3cfd1ec100e6f07fb2a0db24c7", + "prompt_tokens": 344, + "task_id": 40, + "total_tokens": 374 + }, + { + "code_token_ids_sha256": "6436487de97d6c255c86b19fdab467c77f3ff77912bef299ad851fae7d6d13da", + "code_tokens": 28, + "prompt_token_ids_sha256": "f64a1b42d9646080299a6166b75c09769a173875fbe0f717ec72d9b85fbe51e4", + "prompt_tokens": 150, + "task_id": 41, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "b7a3f210a3585f4b8f55727852e815e4e3e10aee32dddf485a4f548ee1dd5d18", + "code_tokens": 26, + "prompt_token_ids_sha256": "385daf033e6f9e8fdf41af1f4fbe97c7677c8237d1157e77db0bef410c72b97c", + "prompt_tokens": 107, + "task_id": 42, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "aa53a04cc981295b35d37380f9f955fc351a9a0b50d95b5af42edf9f674183b8", + "code_tokens": 54, + "prompt_token_ids_sha256": "44a392e9d847d83c24c3254e6ac10ca75b1a7668f5f1939afded586d8e4a3e15", + "prompt_tokens": 88, + "task_id": 43, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "44c824bbfc11af6312d49c7677ec43f0de083212acc643ff3ca18ef412c7a5a1", + "code_tokens": 47, + "prompt_token_ids_sha256": "1b1117a2ae53889adcfe9d326a46b4d18ad5849b90f6c7ad31c130c1eaa9a0ce", + "prompt_tokens": 81, + "task_id": 44, + "total_tokens": 128 + }, + { + "code_token_ids_sha256": "d0dfaa1cad80b3ac30ee4992c395ff2d36a13c00cdf161cabb8f9da0675e23ab", + "code_tokens": 93, + "prompt_token_ids_sha256": "4e498a9df7f26dffc85989a37aeeb767cabf24a9aad8138d0a3657c1f1866592", + "prompt_tokens": 100, + "task_id": 45, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "9ab4c157f93c4edd52f0b7dd90abe2a3f2e56e5b6b45c697572a11f50d34ccee", + "code_tokens": 30, + "prompt_token_ids_sha256": "c663f8c87c27556ef65d7c0374375ca2f20dcd62ab0affc905cc157f6e999ae8", + "prompt_tokens": 94, + "task_id": 46, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "a07e2b3d43a3b1b2837b3ae326e16d3c7390ca30ea5e909b2d2ab0ad326794c0", + "code_tokens": 89, + "prompt_token_ids_sha256": "b34cdf5c66d268933013da91cda1877cbdde70c40eaa30cc5ad47ea940fe9d29", + "prompt_tokens": 86, + "task_id": 47, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "fd17e090ed65c9b06e57387112805f960f73fa94bc73464bf1432f966a81c6ef", + "code_tokens": 70, + "prompt_token_ids_sha256": "bfb67484ef69e10dcb9d8682434e5341d8164c379fe58d6534f0ef91254f543d", + "prompt_tokens": 82, + "task_id": 48, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "f97eb04b3722a2502920a008089c2e0dd8b5dd708f8fd779378d32bf92e8e44d", + "code_tokens": 26, + "prompt_token_ids_sha256": "e7d215ee203a8178b569941d75974534a94f0d2adeda524f76b79cc48cc6644d", + "prompt_tokens": 194, + "task_id": 49, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "1b9fc36cc01bd8d59ec0e95e3cb73d2047d3352ace9d4cd9551f9d3a3c2de915", + "code_tokens": 48, + "prompt_token_ids_sha256": "716debad76c783b9d4da0f2eb306c46698b5fa084b1328bb22f987e21cd65505", + "prompt_tokens": 166, + "task_id": 50, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "0d569c061b3db691dad5a7e1fbe219d9b5ff92824583579a335870389cdce499", + "code_tokens": 29, + "prompt_token_ids_sha256": "0197a61bf27df1bb477044885b773b43193f3c8a0f7336fc2e33fa7f10ce0512", + "prompt_tokens": 82, + "task_id": 51, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "be19dbc524f163e3ddddd94839057d7ea46a7d7b4c026d9f970e3555c0824937", + "code_tokens": 17, + "prompt_token_ids_sha256": "1aa0dea339e491a4b40cf40f825b178858faaea433b5c82155ee57f84354144b", + "prompt_tokens": 86, + "task_id": 52, + "total_tokens": 103 + }, + { + "code_token_ids_sha256": "58e786b9131ffeff92479ba3859c080d8257df33dfbccc2ad6fdbfc40433f8d9", + "code_tokens": 37, + "prompt_token_ids_sha256": "5e2f0557ce5033dabf63973ca38d907a27570aae5c5f5cf21ed2a9b4fb7f3ed9", + "prompt_tokens": 86, + "task_id": 53, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "2f2da2e7070d444855c0e1b6274b9ce79e57a8e63179c2dfea699528dfc55bf8", + "code_tokens": 121, + "prompt_token_ids_sha256": "4137ead7c7dd71a7f07e63349c3153d1ea09f0418658597e01e0e2482e0ae2dd", + "prompt_tokens": 176, + "task_id": 54, + "total_tokens": 297 + }, + { + "code_token_ids_sha256": "5328ce52815f42e654c1f443300859c15a34796f32fea3b00f18c88f1b6fdff2", + "code_tokens": 30, + "prompt_token_ids_sha256": "5da18dbc6d4a14366a61c23c9c2017c194c50ec973ddd78124cec6c099400cc5", + "prompt_tokens": 80, + "task_id": 55, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "d33653ab859bfa48ae2e43465b43f6b19d3b0f7634b1e57791d294b7de0db1c5", + "code_tokens": 74, + "prompt_token_ids_sha256": "63f364f6ab6a42463946724a6cb25cd055414ff33d016b0818041b51c02f2570", + "prompt_tokens": 71, + "task_id": 56, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "7f3b8195c3c4302172fec3d44ca1c916374f2a11e453589baae327683243bfc5", + "code_tokens": 53, + "prompt_token_ids_sha256": "33975f33d9ca2399cc6d53c3e0259c19dc29cf270567cce6ba7dbb1885dc496f", + "prompt_tokens": 107, + "task_id": 57, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "1f0f4b0df382dce3e83abb9712e143bd7928cdfb5b5d86fffa79ecd885926f39", + "code_tokens": 21, + "prompt_token_ids_sha256": "edcbc5d459be24f8bd6f406a36d483796018ede1e2f2a5c384f3201a44f18f94", + "prompt_tokens": 84, + "task_id": 58, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "f31e878fce3a1891f8071b7caff1ea84aafc5d58fe59a1ac11b9b1ef33f785ef", + "code_tokens": 20, + "prompt_token_ids_sha256": "0b11a37d25b39c23f1927614979b9cc2822c921dc4988f1a7f1c59cdf30b188a", + "prompt_tokens": 77, + "task_id": 59, + "total_tokens": 97 + }, + { + "code_token_ids_sha256": "a443d585d7a87d6d79211e285b0e01d783ca6222494ceb95fcae55772021c6f2", + "code_tokens": 122, + "prompt_token_ids_sha256": "b6e757c7b148040afdc04f90f979ddb3e1ef77e7a449160c5bb71615a73bc4ad", + "prompt_tokens": 145, + "task_id": 60, + "total_tokens": 267 + }, + { + "code_token_ids_sha256": "6ba5e7ff66a83b07a9991a00cdee94d982a25397674e5432dfd4e3a27dcc4c38", + "code_tokens": 94, + "prompt_token_ids_sha256": "e95ddd07c1e0da2b2caca29823bae5ab11f50f559cf00fd45298af0f69e0151b", + "prompt_tokens": 99, + "task_id": 61, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "ee94442188523f891922c3afbea433be47546deea7e30eb64f0100465148b9c9", + "code_tokens": 12, + "prompt_token_ids_sha256": "9184a820d5228322a4d1d657bc310b1bf6751ef716efd639200f0d9078d15b6a", + "prompt_tokens": 104, + "task_id": 62, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "1db308823f4593774dc50735df950bdfd8682241f70aa1e6bfac23946f4fd78b", + "code_tokens": 38, + "prompt_token_ids_sha256": "815b393dd2fb94de79c3b07904a2368f11039f19fa36cd70d7028405fa1c8eb6", + "prompt_tokens": 151, + "task_id": 63, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "c8a59e076296ce8a47087028a792c70ffa21dce70ed9aa07b3939db6c2a662e4", + "code_tokens": 58, + "prompt_token_ids_sha256": "4116bdacb62fbd2cdbff5dc924507c9c85fbd7a69f27e4c189298925c516e597", + "prompt_tokens": 196, + "task_id": 64, + "total_tokens": 254 + }, + { + "code_token_ids_sha256": "16105f9807e84284ae6417b45a092d735f521a0f6f56b9947f2ae168a78380d6", + "code_tokens": 54, + "prompt_token_ids_sha256": "ec16922ad5c4c8ddc8ffa92d30a377defbfa5bd9f3c550d06ba6ac0261b48955", + "prompt_tokens": 116, + "task_id": 65, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "e1c9c52bed6e427dc9fb811b9fbaff64be024891b78bbf76c7dc50568202d83e", + "code_tokens": 40, + "prompt_token_ids_sha256": "8cbacf868f4620e15dffada8a159bca3ac668cada91f03e2980607626943bb97", + "prompt_tokens": 86, + "task_id": 66, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "e882b415b049dde1d6c5060f61d59d4488c3292c8b2e7bbb4dddea3aa7daf597", + "code_tokens": 115, + "prompt_token_ids_sha256": "3f9edda1dfa2825fd7b067941c5a5d4805615a5ff62deab9405d28098d0b608d", + "prompt_tokens": 128, + "task_id": 67, + "total_tokens": 243 + }, + { + "code_token_ids_sha256": "c70fbcb4bd4cf364cffc3be84c7814f758014aeb8643dbcaf163a6044e8c668d", + "code_tokens": 60, + "prompt_token_ids_sha256": "d50ec4cce1c2b583fdabdeb2e09336affa1837152c28e1d07cdd5692ebba36f3", + "prompt_tokens": 102, + "task_id": 68, + "total_tokens": 162 + }, + { + "code_token_ids_sha256": "f2d0502e46fcc155fccb765c5391c0a73ae7267bab60e360b9e6fbceeff727e0", + "code_tokens": 124, + "prompt_token_ids_sha256": "7e5e54100a17a3fc7bb14fb0de59c6863c32fbc8aa11d5ce99527c5fc47b63a6", + "prompt_tokens": 104, + "task_id": 69, + "total_tokens": 228 + }, + { + "code_token_ids_sha256": "919e961af9f7d90f8becb688e35c761d4ec0bf39a30d45678cc43b0ad33af917", + "code_tokens": 91, + "prompt_token_ids_sha256": "73f541e9ec6c1ba3942b10d9542b355b93b083ba87eedd34e5c671b0a5b32778", + "prompt_tokens": 146, + "task_id": 70, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "2542f94e49df4904fcf3eb34e8ac90b33c087027f3b4e3f6b35d39793469d130", + "code_tokens": 125, + "prompt_token_ids_sha256": "c74181d2aac145606b088dcabc2aa93c593408ee73e60322ab1d7be48a1b9b7f", + "prompt_tokens": 164, + "task_id": 71, + "total_tokens": 289 + }, + { + "code_token_ids_sha256": "3850074acef019a6d2afa5604e982a6069b09681af51d2cfaa76ad771045bf9b", + "code_tokens": 26, + "prompt_token_ids_sha256": "9540fcbf020c8a67e9d9108f12cd85c0935c63a6ccac86664e1502def5b4d03b", + "prompt_tokens": 79, + "task_id": 72, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "9770220e9770ae5712379cffee311f930586560e79b9c94e8915cd444202e611", + "code_tokens": 24, + "prompt_token_ids_sha256": "81a2f7b24bb1fa111ef571f3ba4e741c8abba293d94536e4ede3fe1ba2d4b24f", + "prompt_tokens": 159, + "task_id": 73, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "c000a031e5beb3ed4c5cf24f3a1d3f76b272b280fea56818163ce001dd200a90", + "code_tokens": 180, + "prompt_token_ids_sha256": "ea20709fc745eab3a8f932013aa6ecbe7e5bed7fee7d28e3a2cd7dd3f96b66d6", + "prompt_tokens": 107, + "task_id": 74, + "total_tokens": 287 + }, + { + "code_token_ids_sha256": "874c164a8413e60415fed343cf9cf0bbd4d3b8ffaf17a3a4fc07e5bb5fd45e38", + "code_tokens": 40, + "prompt_token_ids_sha256": "0856e8a0f86c222b1f11695ec7039f5a80673a7adf6b9ba1144a5196f81c55be", + "prompt_tokens": 201, + "task_id": 75, + "total_tokens": 241 + }, + { + "code_token_ids_sha256": "9a5ab9bc318685221ae3a00aece4853a9a2f1b451e0627c8624d4cd5487df8ab", + "code_tokens": 72, + "prompt_token_ids_sha256": "34ffd5a0735716c8aea2eab69629cc50552a2020823d8fb9fd0fa19bab9320ce", + "prompt_tokens": 80, + "task_id": 76, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "9b6b8fef67034cd6fc2896cffcc2b540831f350609d4e331836879fda1b3ff26", + "code_tokens": 20, + "prompt_token_ids_sha256": "c4f9c132c553b040a95f02af9be15b3aad7eeda10a208eb039986986ad25d910", + "prompt_tokens": 85, + "task_id": 77, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "7a2c05dcc15b5d5d4b342f4a4652db0e3410631e8d6756bd784012c72bc85f3f", + "code_tokens": 76, + "prompt_token_ids_sha256": "9f32c67684d87dafd9f0b29fd241cda54729c6ae067b24d5fd48a01f2580cf21", + "prompt_tokens": 89, + "task_id": 78, + "total_tokens": 165 + }, + { + "code_token_ids_sha256": "f8c5ca3dba3ba832e3f68155b6ea45bc226e460d6d4008d5ecaac96849342199", + "code_tokens": 42, + "prompt_token_ids_sha256": "ec9611eab01932858b739d549ac0eefd5f62974f5ef61389ce19ecf053b55577", + "prompt_tokens": 71, + "task_id": 79, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "e439c21eda7f9ea423cf4107a7e860ba7f24538a03464f72a271b86f8eab378a", + "code_tokens": 28, + "prompt_token_ids_sha256": "e9cf18c783b3abf183b0e4ace9eb051fee74723583f71da9ef028e7d3aff23b5", + "prompt_tokens": 83, + "task_id": 80, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "73a78ea6a11845bfd9fc2d848990f3c858f4248c95f126fd06edd7f631cfb08f", + "code_tokens": 58, + "prompt_token_ids_sha256": "d1e840b887a00933a67c4a5774437b8288e0e8d1b8bb2aeed47397accf7ad9c9", + "prompt_tokens": 252, + "task_id": 81, + "total_tokens": 310 + }, + { + "code_token_ids_sha256": "916f0220c27b1c281a41ad390ca7b870755c237f98631f272c7b6e31609c2532", + "code_tokens": 25, + "prompt_token_ids_sha256": "d32748a28cdab0135c671d9beba2b42f96189138dcabda89ed848ae7a268cbe7", + "prompt_tokens": 112, + "task_id": 82, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "a9096fa457b3c7e373c9e7861a5a412f885237141e969f9c58b2f5a7d0eaa276", + "code_tokens": 88, + "prompt_token_ids_sha256": "0ac3e057973a992cb2e7f92d07b63ffce8d93339706ad7c7acc7ed915605e72f", + "prompt_tokens": 79, + "task_id": 83, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "8df335928e123755be904d009a612aeac5f40e49b85562d4aef9f32675dcaf2c", + "code_tokens": 42, + "prompt_token_ids_sha256": "3270e0d5b2b5f0c304ca90ed62e66021f6c97792fe8443d5fbbbfe06e324672f", + "prompt_tokens": 70, + "task_id": 84, + "total_tokens": 112 + }, + { + "code_token_ids_sha256": "c675cf5eda90b7b82bc0514da5ac294f6f65c245552d740c0a6bce4ffd90d03f", + "code_tokens": 24, + "prompt_token_ids_sha256": "6a37b6d7e968320143eb78eaa91c166aafe3591842cce8e5fab66040eef1421a", + "prompt_tokens": 118, + "task_id": 85, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "a73935b567a9ceaab9383a8fe3b4d8eb2e69ebf158959102c641320cc3d507bf", + "code_tokens": 24, + "prompt_token_ids_sha256": "5783709cd4625dc7d491074cf6b415f7bcb750ed24276ecaa42d229d19334abe", + "prompt_tokens": 78, + "task_id": 86, + "total_tokens": 102 + }, + { + "code_token_ids_sha256": "eed89bb525900899baa2242ff332f28ed6862dcfc8d7284a6eb7e87bd83ef3c8", + "code_tokens": 43, + "prompt_token_ids_sha256": "94eb8be73dcc0ccd4b975d48aa83c2a71a92fae79264ce90849ca49f41290525", + "prompt_tokens": 310, + "task_id": 87, + "total_tokens": 353 + }, + { + "code_token_ids_sha256": "11af968b33151ace9b5131d4cdc08f8d8854111927921305703da84a970a0583", + "code_tokens": 24, + "prompt_token_ids_sha256": "8779ddd9f0763372a9bfe9331c60e9b3557fe31e5f3ebd008dcd534db5143f88", + "prompt_tokens": 220, + "task_id": 88, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "e1d18724171881fd8aced45af5c1a08cbd61a11d86cc5ad85187bd4e9c2a400d", + "code_tokens": 14, + "prompt_token_ids_sha256": "cc04d950ca3e9e286cb0b912bab35dd94bf7b742ce93e811af7a836655de0781", + "prompt_tokens": 72, + "task_id": 89, + "total_tokens": 86 + }, + { + "code_token_ids_sha256": "9d8311191c99cbb4c0b5218229daec1143e19cdb2559fb6d5e89fcd9749d9205", + "code_tokens": 41, + "prompt_token_ids_sha256": "e7deb2c9cd0cea443af1ce0ea4c8b25e447014a67efa876a8619df04871a2c2e", + "prompt_tokens": 83, + "task_id": 90, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "4d6e2570c07c0b4fe8ce75e041256f52b55792021cc1881cb8f2d8c68156e942", + "code_tokens": 32, + "prompt_token_ids_sha256": "2cddd75f63a682a8057ee1c480563458d86420a8e40629c3432a0307f3cc3736", + "prompt_tokens": 113, + "task_id": 91, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "3eeb5f3de473dc1e96f6ac50f4f4546cdad752e4b85766173377782ab33ce1b5", + "code_tokens": 53, + "prompt_token_ids_sha256": "e7abb7b690292ee9d260fcb7dce173c01f97d814c0df3e753f6b0b6b20d58c1a", + "prompt_tokens": 85, + "task_id": 92, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "51a9d2b9d6a4c919c7cad1737769e77c0525b60b683a5bd12f15eb961a2ea171", + "code_tokens": 51, + "prompt_token_ids_sha256": "8c399e91f489918832679021affdb6c4dfb1f4ac59e9b0899ac000f45d901567", + "prompt_tokens": 80, + "task_id": 93, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "adf34317830481522a1887bf5b65ffe1f36cf162ccdd640a9498126cf37d400f", + "code_tokens": 36, + "prompt_token_ids_sha256": "f3a9a8c9fc0e1cf71bdbc0f2d18fd12a2dfc353220ced456a1e3fe52512d7db9", + "prompt_tokens": 153, + "task_id": 94, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "54b4abf6744d8d389ccaf8687941e46aeb569c2cdba30eb3788fa5fe936e203d", + "code_tokens": 24, + "prompt_token_ids_sha256": "699d75b92e11ec17e1e042775c8f37c27d836e8d93b0d8a623c814189da3e416", + "prompt_tokens": 103, + "task_id": 95, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "98b72f4568ee3c30cf02cafaede5aa425114aae99f11f38decc83e7825ba8682", + "code_tokens": 39, + "prompt_token_ids_sha256": "f87c8ff4b4e11a7c780f6b65f67b97da4f3f5b3fef2afe56c740dad7b4e3004a", + "prompt_tokens": 71, + "task_id": 96, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "292969477ee44c5d19674f97232ff9038e50a17305f0b9274354be7cd502690f", + "code_tokens": 84, + "prompt_token_ids_sha256": "a2a8cc78a2c3e409084301e283b962b7f7663b83d9ca8b76191ceaeecc74d99a", + "prompt_tokens": 330, + "task_id": 97, + "total_tokens": 414 + }, + { + "code_token_ids_sha256": "9007a4333b815b1827f5dc5cc72f85f78a6ba66917edf3b2abaebd8f574ad698", + "code_tokens": 32, + "prompt_token_ids_sha256": "9e5404764764474f325c5a4654ed2f733a86ade6f8a7f62f657c46bf105fe5d6", + "prompt_tokens": 111, + "task_id": 98, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "ae6dbc9b8c15edc86d4a87b3c0e6a455e88fba0e3ca7f58cd381ed4654b7b9f6", + "code_tokens": 19, + "prompt_token_ids_sha256": "377c17235fd40a9cc91da3f68b3c853aa9b689f5e13d1d2cae48c59f0dec6f3a", + "prompt_tokens": 86, + "task_id": 99, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "e91e0510930051525c5f3b09373c3e12a4bdd5e246b84b8f1ee83144487d340a", + "code_tokens": 50, + "prompt_token_ids_sha256": "9027f1ddca1663ee908227a774d331e0c5b9058e2b5da39708345226a9a6f6ec", + "prompt_tokens": 86, + "task_id": 100, + "total_tokens": 136 + }, + { + "code_token_ids_sha256": "97d799a2ddd249f52f17e5f0dfe3554af676e26f6505e93b92fea83b11f1045f", + "code_tokens": 71, + "prompt_token_ids_sha256": "ca383495b10d22d89821d81ab2ca70caec36803e5c7bcc22cf6673aef5994076", + "prompt_tokens": 123, + "task_id": 101, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "264fc707f93279ecc2c2c32a9e03c7d7fad75b5d44c5be8df38fb818e4bb130c", + "code_tokens": 28, + "prompt_token_ids_sha256": "9b597d762024ffb86715e237f857651d202a6b978c8fe45e3e2e2fbd9d7ad65c", + "prompt_tokens": 81, + "task_id": 102, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "59cb42d2a76c4cee786f9d5deb090cc246504a49034ee544bb509f58d67ba5da", + "code_tokens": 76, + "prompt_token_ids_sha256": "d067807060427c6100fbf5c39d8686b2a14ed89e6cf79bae76d8da40dfc7062d", + "prompt_tokens": 83, + "task_id": 103, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "f4d5a2fdbe065c018862e148c2dba762e4d2992019792236edf2befe31c7c9f9", + "code_tokens": 34, + "prompt_token_ids_sha256": "70166cf7cfc37f3b3f45aa9dccf85b55b3e1abe4fc238661d224aa93620d795e", + "prompt_tokens": 185, + "task_id": 104, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "e641a23de5a23d2580b700a135318d4520bc54fce05b7496b8975803dc93d3c7", + "code_tokens": 11, + "prompt_token_ids_sha256": "41fdd0f7c571355d8524c3e2d212a16186c7458413d99249eaa7541b290d9002", + "prompt_tokens": 72, + "task_id": 105, + "total_tokens": 83 + }, + { + "code_token_ids_sha256": "d5de986de5810799534a50630a394311b1cb315566e44e464a6086dd705aed73", + "code_tokens": 31, + "prompt_token_ids_sha256": "5455224444b6ef5531536d5a7b626ee7b056748abaceddb18067eac6d4b75030", + "prompt_tokens": 154, + "task_id": 106, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "7f51ddda22b70adcf2c00962e05933ee9fe410af5ad541be6e5eedc01672371f", + "code_tokens": 118, + "prompt_token_ids_sha256": "ce39296e861c64d4db594bbb3c85a2f399a1fe32bbae70509728384c85447c2b", + "prompt_tokens": 85, + "task_id": 107, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "fdc8c884a12903eacbd78ca7a3db379117cc926d4c6e174e8707dce026aa46f3", + "code_tokens": 60, + "prompt_token_ids_sha256": "e82a7f107512a0c0f6f48feacbacc388f439efb17a777a1749624fa3764807d5", + "prompt_tokens": 485, + "task_id": 108, + "total_tokens": 545 + }, + { + "code_token_ids_sha256": "c0a1eee1cae5708feab0c624ac73ed90bcf2b874f00dc481296c1f8dedafabc1", + "code_tokens": 47, + "prompt_token_ids_sha256": "f2e758f96a12029f6c309def6b4110816b91a6fb851d0f2d62fed87993dfe510", + "prompt_tokens": 98, + "task_id": 109, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "537234c1c3b93d9026bf0f6d8fbdd14825607c4ac37bd97892cbe986c18f0c6b", + "code_tokens": 84, + "prompt_token_ids_sha256": "d92d4e72a0e83f30b7571e8b2bc17c19d5856bcbf4636b644e528bd2bb908e16", + "prompt_tokens": 288, + "task_id": 110, + "total_tokens": 372 + }, + { + "code_token_ids_sha256": "844ff75eea90279f76b626ad77496218862e5c5e3b07210594e6ff50074e7f6b", + "code_tokens": 27, + "prompt_token_ids_sha256": "36bf67e6d2eafae1ece26731ef973b60efc3ed5585085a2fad5661fc2d87e352", + "prompt_tokens": 248, + "task_id": 111, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "4fc3e9ccf26478cad737fde0881c6b125630fe3dbb0e51ec1c3ee7040f2368f9", + "code_tokens": 19, + "prompt_token_ids_sha256": "a65cf4432d2c0cbeae7bc4ddf1fb01dc68de9445edc3986bbbc7fc05ab6b2cb3", + "prompt_tokens": 72, + "task_id": 112, + "total_tokens": 91 + }, + { + "code_token_ids_sha256": "b7be982b00f7558a8d608825d18470483d2f75457a585d007c24411984154bc5", + "code_tokens": 114, + "prompt_token_ids_sha256": "4b92b23712dcc6d42c6fdcf9a49b1e7c78c6553d2d9ac1cb5f627917c4d0b9f9", + "prompt_tokens": 68, + "task_id": 113, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "2b076988f3d4b45bd5cbc5f0d7cd0e6d8283c4b08b048eb01e77acdce6cb6249", + "code_tokens": 40, + "prompt_token_ids_sha256": "7035d9971c839aab6000daf68b23c1d3e7deb21e9d960c03d30d37dbefaac9d6", + "prompt_tokens": 315, + "task_id": 114, + "total_tokens": 355 + }, + { + "code_token_ids_sha256": "b584f117c73c95b9cd42ccf48e88ea1a8db8224fa02b83266f187a2a4ed1cb70", + "code_tokens": 25, + "prompt_token_ids_sha256": "faef1334581a007b0ef1bcc6ee88fa48b5aefa4c8e670f2ca7e36af2904d81e4", + "prompt_tokens": 76, + "task_id": 115, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "88d80fa2fea0885de933125b857030d0d22258c74b09eb6f0e3a84da65270771", + "code_tokens": 23, + "prompt_token_ids_sha256": "ddca73e02700cd5e2b66607dc1e465c34e925d7108c9a32a42b5ec5ff2cc3bb0", + "prompt_tokens": 86, + "task_id": 116, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "6e85810c50a9fad2e8ae3f05a7a9b7f6194dd87ac3caef5bf443351f7ec6c2c9", + "code_tokens": 77, + "prompt_token_ids_sha256": "796779c06f14f120a1b57af498bb27993619bae06d105bd12ad58d16c565e01d", + "prompt_tokens": 297, + "task_id": 117, + "total_tokens": 374 + }, + { + "code_token_ids_sha256": "3531f86c9ac2069b2df809f5732d106e2cc3e71426ac197caed56fe392c56c22", + "code_tokens": 19, + "prompt_token_ids_sha256": "c1a7932e793602d3b349ae7b17d04d26e9e4925ee890fdab1a38cd2d1c52182f", + "prompt_tokens": 105, + "task_id": 118, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "6f8d7850d23a4da7f271209b269389dcedec6d9fd4ab92d477ec47475f5cdd04", + "code_tokens": 37, + "prompt_token_ids_sha256": "9f8e317ad291c50692bff5613a5d21da55642d0918f82e035c77dd06f40485d5", + "prompt_tokens": 117, + "task_id": 119, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "486629e5c3247ecfa794941e6e7709150137e281177c870fb82b4d9d498a9873", + "code_tokens": 33, + "prompt_token_ids_sha256": "1642a22406b3b950ffa6b5336deea64c49aad64e64c0849ddfabc71839562457", + "prompt_tokens": 143, + "task_id": 120, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "e75c71e50c510cb86fcc27dab32cfc6ee56a2601332ed76d932856675be26709", + "code_tokens": 95, + "prompt_token_ids_sha256": "947619a5a3a5429bd35bf00bafc999280cdf8237f67bf6039dad75bfd7c67877", + "prompt_tokens": 152, + "task_id": 121, + "total_tokens": 247 + }, + { + "code_token_ids_sha256": "ce64ba080056dca5433f061cd5c4df7b7aa779d5ab2b90108b85e6da9ddec4ba", + "code_tokens": 125, + "prompt_token_ids_sha256": "7d6dd1378053d3bc03116ae54c545a3de9c653d438de641751ccb577a4258b71", + "prompt_tokens": 77, + "task_id": 122, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "2978ed2e33a03e1113adc01a4b3f23279974a15bfcf912230b83924a29878342", + "code_tokens": 168, + "prompt_token_ids_sha256": "629117cb305bed6ef535b6aa11dd84f7d4e0ec829779fd9123bf6011d765cd51", + "prompt_tokens": 85, + "task_id": 123, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "a96a0ac08047d37f440ef003cfd47cf2a7067319c8d164d194dd77d0f0fd8502", + "code_tokens": 31, + "prompt_token_ids_sha256": "3842943b68e2181aed711d53a4a9b52eafb41e6c3ac70b39e85da2b8f742a1aa", + "prompt_tokens": 122, + "task_id": 124, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "a7426c7fc130b39623116d16c8d81abc82654c0d6949c0254b77b7da11049b1c", + "code_tokens": 83, + "prompt_token_ids_sha256": "d3ed852055f44583d5b690a874f79df3276e70f322ea0ba8fd01f139b39a9cbe", + "prompt_tokens": 125, + "task_id": 125, + "total_tokens": 208 + }, + { + "code_token_ids_sha256": "5eacd7bdb126bc851fc0ff66212aae8746fd9e11cbbfa9c347b9e086d5ff71cf", + "code_tokens": 50, + "prompt_token_ids_sha256": "71cc29143428760a041cdc13f587d0412a2c1bc02d6af3d73f131d770569a8d5", + "prompt_tokens": 83, + "task_id": 126, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "17953e34e753459fe8d60c700786a0550b98683f74c45cfc5036fdb32ccde99b", + "code_tokens": 69, + "prompt_token_ids_sha256": "3e023bda4119a2d0b3c7a57f0b12a0a6b2db8dae34b52ddaf85530d1cca71c89", + "prompt_tokens": 78, + "task_id": 127, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "f44c28c45126a5cf2ef99ba8923852d534d9b0bd0d5871248418278075f649aa", + "code_tokens": 50, + "prompt_token_ids_sha256": "1da633fd8226e8a9b8587b3763a47b92b46d5dcd9ccf37b79d48636f345b5dc8", + "prompt_tokens": 94, + "task_id": 128, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "9eb624dcf6e38543bcdb0bd182f88cdca66325bcaa4987a3c9e068fd7dae9717", + "code_tokens": 161, + "prompt_token_ids_sha256": "34a97210da86858a0a4a8c003b35a3fa3027fb03ae5d9cac88ddf554e3f16be2", + "prompt_tokens": 164, + "task_id": 129, + "total_tokens": 325 + }, + { + "code_token_ids_sha256": "6616386c5498b49da69f9d54e3485fec331b0d5425d6bee96169bef3f56b8f1b", + "code_tokens": 53, + "prompt_token_ids_sha256": "f258a2e044ba914ac20e34fa8536b247ab956cea825cfe198aeff7a42f66ea28", + "prompt_tokens": 198, + "task_id": 130, + "total_tokens": 251 + }, + { + "code_token_ids_sha256": "d4e6aed3f3b86d9ca64e3278498a2de74aaee170b0883492e83c3d5cb3633eaf", + "code_tokens": 97, + "prompt_token_ids_sha256": "a50dbed60cb8d3b9e69ccd88ccf4e6a6c2cb7d9d71c38a4e38c90aa08df74f7e", + "prompt_tokens": 80, + "task_id": 131, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "e5f570605a252bee0f1cbdeece0da75a13dc82f24b8adc9be1d064263e531335", + "code_tokens": 22, + "prompt_token_ids_sha256": "6f88f1b009611531120ea9174c27408e78d13322183bdd69c81bf5646bcc1e51", + "prompt_tokens": 116, + "task_id": 132, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "baf9073a4b9fbfdc65b87e7371db10321be91977995767760afff173d67e793c", + "code_tokens": 34, + "prompt_token_ids_sha256": "36e2cf063de71c662037eded7c9df41cd1f420f94fff71589ed5f0b76995b80b", + "prompt_tokens": 171, + "task_id": 133, + "total_tokens": 205 + }, + { + "code_token_ids_sha256": "9d9e590898ed5a766797862829e6cba044bd1aaa799b8c1edc21ea255bb4f8ce", + "code_tokens": 81, + "prompt_token_ids_sha256": "1d3724d3f7244584fc7de535c78ddb427b52120b1298a1fb2c4af13a0adbfac0", + "prompt_tokens": 109, + "task_id": 134, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "4bdbc0eeaccc3c56d81c60adbbbd38d4932b90ff750dad9694e8fdf92d1593a9", + "code_tokens": 17, + "prompt_token_ids_sha256": "11396a5bfbd18abcc16c0e2cc3d20b2951bfa91d79c6a7b2ff32519f334f8d3b", + "prompt_tokens": 75, + "task_id": 135, + "total_tokens": 92 + }, + { + "code_token_ids_sha256": "549f9725c32870d7d856ed10693d09a2d0edb4e142576b931ed9735991ec3cf3", + "code_tokens": 181, + "prompt_token_ids_sha256": "9cf4c5577b2c6e75845ebd7fa39c040b6116449ce7f51cd395a12c19139e92f1", + "prompt_tokens": 84, + "task_id": 136, + "total_tokens": 265 + }, + { + "code_token_ids_sha256": "689692118b17be82fa5ac184f217fa647bac166b92ccc198908c049630658e8e", + "code_tokens": 63, + "prompt_token_ids_sha256": "01b16977e551bea64428614c7e832a6f09ee954a0ea9668ebf6f92fbd0a5dcc3", + "prompt_tokens": 174, + "task_id": 137, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "564decc4a34680188fd13c41921d427063e53178df07ff97547126d766544b91", + "code_tokens": 35, + "prompt_token_ids_sha256": "599ab7b99b633de851604ee9859dd8db064b3285e5ab57d70db7b763c0768395", + "prompt_tokens": 98, + "task_id": 138, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "6b7ba9a3013e10b420864b6fda5d58e18a147c0de70a7dab445c530a4f9e4ac9", + "code_tokens": 25, + "prompt_token_ids_sha256": "39f109a083b5acd82fe44a37ff87160de184591734de89d65eaa0c0abeafb713", + "prompt_tokens": 110, + "task_id": 139, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "0df01c94e7cca81c701578f8c7a447b21a3393b75bf763f89d1b341f70548f09", + "code_tokens": 60, + "prompt_token_ids_sha256": "53a43e508589bf4900b64b780bb292e9aca2e8e40ef1279998724859435ce43e", + "prompt_tokens": 190, + "task_id": 140, + "total_tokens": 250 + }, + { + "code_token_ids_sha256": "7da1371e719db577e72e2f0412b53df3c07794a33349ef19c3df9e0e43679c19", + "code_tokens": 86, + "prompt_token_ids_sha256": "3b3431f0fac5d6ce559519662f2b8b37e0e81fb13086cbe25acb9fe5f8602541", + "prompt_tokens": 178, + "task_id": 141, + "total_tokens": 264 + }, + { + "code_token_ids_sha256": "8a5b2f87c5d29af793c4c8420770bcbb62c56cd2e6d94ce5fe02cdf58d378c65", + "code_tokens": 40, + "prompt_token_ids_sha256": "23a6793533e9e3da40682d6f9bd0ef2daf1bd20cc768bccc1ba007ead287530b", + "prompt_tokens": 204, + "task_id": 142, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "b6cbc21701f6e745d7bd860870f9153c711dab6787293f07e8f1ea21a99fead1", + "code_tokens": 27, + "prompt_token_ids_sha256": "071cf1d4ae96c4b61343247eb8371377a3f225bffbb680f98bb359b059c9cb60", + "prompt_tokens": 131, + "task_id": 143, + "total_tokens": 158 + }, + { + "code_token_ids_sha256": "7ee1df7ec87fdd3f7fedba92300d4a80378df9448a0c345585a91eaf15295f17", + "code_tokens": 52, + "prompt_token_ids_sha256": "74ef2fe9d92387280006c68daa96bdc2d24604a1bfb0c6ea0e1d10ce4bb4f508", + "prompt_tokens": 122, + "task_id": 144, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "b468182b74c335010da580a00f4e9985d0809aa0c7ab13b0abd1a5365a28d260", + "code_tokens": 71, + "prompt_token_ids_sha256": "2a382e37b54a56b0bf1df0714336d8757f3e8c295f46a06ca7da1059f8805075", + "prompt_tokens": 107, + "task_id": 145, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "a022ac9b6a93fff76bff6d958b408917d3c86b02f3780818fa45b088226013a9", + "code_tokens": 25, + "prompt_token_ids_sha256": "366019695da8006bb80746fbf3114b871d3de6b8d4369a09946a5aefe7f4d774", + "prompt_tokens": 72, + "task_id": 146, + "total_tokens": 97 + }, + { + "code_token_ids_sha256": "0ed86f0b0e9722b3597b676d45edcd554b1416116dfba80e4856cad582f1a567", + "code_tokens": 103, + "prompt_token_ids_sha256": "da966e29602938b5c2fe16d433f8dcce5227bc034cf04fd06eaabe0cda3acfb2", + "prompt_tokens": 173, + "task_id": 147, + "total_tokens": 276 + }, + { + "code_token_ids_sha256": "ed73b08d5d02ac3f7b5d079d0f7c9767f9d9523d74abb4653fdc1605961a0f2d", + "code_tokens": 116, + "prompt_token_ids_sha256": "8a5a31ee72ca5430b1f05cb4f02421140235bc904b593ac21fd072e2ff3195e9", + "prompt_tokens": 83, + "task_id": 148, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "e57a1827f10e52ab5d9ffd8421e738082d3e36f9e7d9c5ec88df2a0c606c43ae", + "code_tokens": 107, + "prompt_token_ids_sha256": "cae2283487303ae3601903c225d73ad0532d1621fa743a9be40af256e59002b2", + "prompt_tokens": 159, + "task_id": 149, + "total_tokens": 266 + }, + { + "code_token_ids_sha256": "904cbf900ef883a906bba7b85c7e3d29a705da833bc96da0d284df6b0fe8600a", + "code_tokens": 54, + "prompt_token_ids_sha256": "fac22133bdd4ede2bfbc830f401e2e6db582a6fb225231944ec04ac154fa56fa", + "prompt_tokens": 90, + "task_id": 150, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "c7efbfa77840fb4d43b5f22060598be59661b1349f0855820492e75050896340", + "code_tokens": 46, + "prompt_token_ids_sha256": "f249b2550919b736d8a495cff99d28f4d4388d9a51cfbca7f247af2fb68e2dd7", + "prompt_tokens": 88, + "task_id": 151, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "22574600d8fd3baf49f33aa435b31c24a878bc0b08f0ac127f5806985eb5506f", + "code_tokens": 170, + "prompt_token_ids_sha256": "25db7506463ce95eb3fdf9d2daaff54040c8cbf1ac4a2173b864ab882204aaff", + "prompt_tokens": 186, + "task_id": 152, + "total_tokens": 356 + }, + { + "code_token_ids_sha256": "5d016cdd76e1476006301187f4f79082abbf80e89302f44dbadbc81003c4c4f0", + "code_tokens": 45, + "prompt_token_ids_sha256": "682738107e75c17c5e1a0eb1701172e53c0a6f072ebe2c8e234f51a2c37448b8", + "prompt_tokens": 135, + "task_id": 153, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "074504890d25da67e4aea86af259b1ecc221ff949a23ef41a251e4a7e91f96ad", + "code_tokens": 24, + "prompt_token_ids_sha256": "2c9764e9478599482677e38c19dc89beafde096786c93720c12e2786c8142d7d", + "prompt_tokens": 192, + "task_id": 154, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "915c9cf6de4fbf781498970a9d39af9e1ed07bc46d94ee5e1c6c15c9600f5221", + "code_tokens": 78, + "prompt_token_ids_sha256": "9aa9714bf9b4afa4f5e7b13de01ff54046140af029d5299038a632fa0571843e", + "prompt_tokens": 81, + "task_id": 155, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "09b9f34b3d9d44cd86269b6dfcb39d167c10d98e661d9c2af57fa854c0af4dc5", + "code_tokens": 33, + "prompt_token_ids_sha256": "37472796e556fae703e468a8068594040200a3cb26d13e058146f384af6aeaac", + "prompt_tokens": 182, + "task_id": 156, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "12efef85c970e63715ca0d24fe58a7a32ca02f774cd6aa43452fad5b93960758", + "code_tokens": 32, + "prompt_token_ids_sha256": "f9cb5f32a410792999280c775f85d906da92f24ae4bba3aabef9625045d99101", + "prompt_tokens": 229, + "task_id": 157, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "e11293027cc3d8733b6030d5625dae88db0ae1ac95c35328356d315fd34fe74b", + "code_tokens": 77, + "prompt_token_ids_sha256": "52836ad4ff79706c605ba331260bbe8fd142b6f94715c1377050dcbfd0d918ec", + "prompt_tokens": 113, + "task_id": 158, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "8d52e4ee16b599d3d77b9b66fa0a9182b33aa6496afa0dbb73ca1bbb10a2a143", + "code_tokens": 222, + "prompt_token_ids_sha256": "5354dce594727993a6ef03a6fc63efa08a054eb51e740dec615de86f305dfb56", + "prompt_tokens": 78, + "task_id": 159, + "total_tokens": 300 + }, + { + "code_token_ids_sha256": "a8eeb602e626a96416653ac2c4bc28bfa8c2b0a0881b08244cc1d7e78d1220be", + "code_tokens": 86, + "prompt_token_ids_sha256": "44f0871200f30071ddd05325a570e4f2ec5d0798ee4f885f729275d9d10c1849", + "prompt_tokens": 115, + "task_id": 160, + "total_tokens": 201 + }, + { + "code_token_ids_sha256": "7b65e676e5ab78d91af7cc5193f0e94f882ff05c2d8404d27acee96c145d90cc", + "code_tokens": 31, + "prompt_token_ids_sha256": "a1c36d905417d4776d2841dd20513511e19b2d226870869f415156fc4274667c", + "prompt_tokens": 224, + "task_id": 161, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "3124835db76bd7740908fb541d9fdbe2f6508112354ce3de32d4d15475af6de7", + "code_tokens": 34, + "prompt_token_ids_sha256": "16b57728c2fec8a0ec830129c4f2561c5eaa668ca1d77b5df9817c56db134991", + "prompt_tokens": 85, + "task_id": 162, + "total_tokens": 119 + }, + { + "code_token_ids_sha256": "2827091fe4281af4257f6f005c682e424937e0bad7ab417039aa510b1cdb3562", + "code_tokens": 38, + "prompt_token_ids_sha256": "d9fd25640d0e1c577e97e043f2f06bd1de1c06908c6661ccbd6608076408560e", + "prompt_tokens": 122, + "task_id": 163, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "0551a9ee47a087934926b90c67ecc5da61adecfdfc7cdca82189f456d93cd1c1", + "code_tokens": 93, + "prompt_token_ids_sha256": "0d95cbbcbd368aee1958a03c0e9aa8be159a6f040c88eecc93889175468704e1", + "prompt_tokens": 80, + "task_id": 164, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "4e613d573efe4dd3b2067ef7ba4da3b38cb9b8afdc3248e53ca4b0b29391cead", + "code_tokens": 70, + "prompt_token_ids_sha256": "5d9bf90c4f0b571844b77905850a35ad967f44905df5c987b68d7a2eb301545a", + "prompt_tokens": 91, + "task_id": 165, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "4272afe984164f919ce49f43f429e81b14ee48de8fbe978f6163a875261bcb91", + "code_tokens": 67, + "prompt_token_ids_sha256": "8849c0a85df756844d09323db8fdc60e246db719534357b8cdac26365ff999fb", + "prompt_tokens": 108, + "task_id": 166, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "85ea611135278d13118510c1cf313da6445dc8a5b38067e0091900035a9b6fe1", + "code_tokens": 66, + "prompt_token_ids_sha256": "928b71e5f5b590f5c885ad47112df5529634161f57daf398436bb4f461e51d89", + "prompt_tokens": 88, + "task_id": 167, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "0bf09a28758f3b3f7588f778f864ff265f1af16fc345ec99d154966a6136f585", + "code_tokens": 34, + "prompt_token_ids_sha256": "a01f02e7b5c05d0290cc55067c96e2c12b94795d2e61b038bcab2343b24208f5", + "prompt_tokens": 101, + "task_id": 168, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "df2d40e8f7715bc1cf0789e9f679a2cf785dff867d3ed3fbb21f4372ed6329c5", + "code_tokens": 64, + "prompt_token_ids_sha256": "2bf0ee661ccdb5d66cc20a89abf784823938070c144a4c72f7ce60234b605bca", + "prompt_tokens": 74, + "task_id": 169, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "7d69269381fe4f6824d1ebee9776b7d9d80f9ea5a0edf47a4ba58229af014cf8", + "code_tokens": 56, + "prompt_token_ids_sha256": "f7076f337c4691b85ba0d5ccb591fae7cc7392a717a72a7236c17d88fc2b10d7", + "prompt_tokens": 168, + "task_id": 170, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "73f55c33613a8a3d22f686272239a28b232d5f8735e278febd6df99162461981", + "code_tokens": 21, + "prompt_token_ids_sha256": "caf28bc058105e85f000dc2e31bbbc4f9f94d104e04b15de748ea7d177db8a24", + "prompt_tokens": 73, + "task_id": 171, + "total_tokens": 94 + }, + { + "code_token_ids_sha256": "85bf02c7e7aa3707b61999f1e8614db2e114211940282fed3aac1ccc403720ac", + "code_tokens": 60, + "prompt_token_ids_sha256": "ff7c369139f6405164386d8e2e73fda41879467391139991aa718aedaae99a3c", + "prompt_tokens": 125, + "task_id": 172, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "7e47ac58e15d67f48299fe39ce222db542baf856bba9f681516893762764d549", + "code_tokens": 29, + "prompt_token_ids_sha256": "22e999d411ebf2f19af4a88ef7d26888d1fd50722dfcc59829d922c7e98c6af1", + "prompt_tokens": 132, + "task_id": 173, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "1768391d2807cc702db439d00c532ebaede6d9c9e1ec7a83b11b78ebb9360f23", + "code_tokens": 34, + "prompt_token_ids_sha256": "8cb028ca4686b5fc37475841c42ef74c2e3e5da51666811ea2abdfa006ed7533", + "prompt_tokens": 232, + "task_id": 174, + "total_tokens": 266 + }, + { + "code_token_ids_sha256": "7f6e29484730f27049fd3cfd2b1ec5d46dafadf49b442a1bf30da8121ea0be45", + "code_tokens": 88, + "prompt_token_ids_sha256": "7e9a5c014fd78c4a1ed5540b2f45381d60c13a3bb096ec1c05508140d3a44c25", + "prompt_tokens": 75, + "task_id": 175, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "50873befcdf0de466218c1eec911012f8d68596b30806959e92cf699765d52df", + "code_tokens": 17, + "prompt_token_ids_sha256": "6476fc372890f2f90c93627b2bdb3a98d8758e64b47fbe8381aabc4ac8f5cbed", + "prompt_tokens": 83, + "task_id": 176, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "c28a73f62c4623227184363272dad3836e36f22c90f786cded9894cbb2c43918", + "code_tokens": 35, + "prompt_token_ids_sha256": "dc54c20c626d0213c025df696ecf99c190aef0b8e5d082d051e413eeeaad5129", + "prompt_tokens": 87, + "task_id": 177, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "7a6ac6c910fd91ed658a40b0d0e2f88d3f4d2953c02bac4cb4ef0f7740afe25f", + "code_tokens": 48, + "prompt_token_ids_sha256": "fe59b7588a3c4e4a70b2ddc76778b24ae7ff8849aa4d679a111b65fd5273f4f5", + "prompt_tokens": 85, + "task_id": 178, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "090e997d41882dc3f0b2038b28e6c8d8a9a741c72d9c3cf8e757b4c718d2c7c8", + "code_tokens": 130, + "prompt_token_ids_sha256": "f52906afbdcce4ba75b01631cbcbb14b00dff5be4348d6befef63c509670774b", + "prompt_tokens": 80, + "task_id": 179, + "total_tokens": 210 + }, + { + "code_token_ids_sha256": "0a058af7fdde81295fe95505497e4a38e5ea57e1213326d2ae905d1959574821", + "code_tokens": 65, + "prompt_token_ids_sha256": "67daa8192d32751e6b4f25bbb449175207289f97aab6ef07e5649b804550f8b1", + "prompt_tokens": 162, + "task_id": 180, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "5208c3e810cc7749d5f3fde59a6201f00d34c5582dd2aa6076fc2601b729e4f3", + "code_tokens": 145, + "prompt_token_ids_sha256": "678207325db6664c299354b69b846989498a0360b04ad07e8a2a906dbf69ddfd", + "prompt_tokens": 114, + "task_id": 181, + "total_tokens": 259 + }, + { + "code_token_ids_sha256": "06bff9841ccf3b25fb98673c7c463c9c2a3ec6846313a7c282712ee7a227eab3", + "code_tokens": 80, + "prompt_token_ids_sha256": "c57c1a76612ddd83673f2608fe8888fb965bce261d20d71019e0a31888815500", + "prompt_tokens": 199, + "task_id": 182, + "total_tokens": 279 + }, + { + "code_token_ids_sha256": "a763bd0cc44e5d0c45350cceb01e816020c8a0f6fce2a9b6d4faef68e0232787", + "code_tokens": 70, + "prompt_token_ids_sha256": "2729fd6304e4fbce6c880536aeb13cd5b928bf8935ddb074e4f7831e841e6bc7", + "prompt_tokens": 134, + "task_id": 183, + "total_tokens": 204 + }, + { + "code_token_ids_sha256": "5faac4bcf032efc76d6737bb0af0d7572a045ead033e45c2fba0fc83696c8746", + "code_tokens": 25, + "prompt_token_ids_sha256": "15a335d8d84d797e5679b5034777a58ef034d895d41493485f796611d344ace0", + "prompt_tokens": 109, + "task_id": 184, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "d14e4db73704bd9a7c36255132d7af191e96f58c4522f83928d9bf245a73c319", + "code_tokens": 50, + "prompt_token_ids_sha256": "25263fd4d6b54c025a503f6a6a0e6038d9f55511c2425e22f90ee71032b3ef4f", + "prompt_tokens": 122, + "task_id": 185, + "total_tokens": 172 + }, + { + "code_token_ids_sha256": "c66c82c44f89b9eed4e16607051cbf859c65280e9cc902c8e7bf49830f98ea56", + "code_tokens": 48, + "prompt_token_ids_sha256": "6efdb3ae209ee8322c25430ab2f18bed11d64ae575728945497b8aa39f54cd91", + "prompt_tokens": 111, + "task_id": 186, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "ae341e0408c6fd63275a4d81d320bca3bba2b92caf0149da510ea8c693d2ab5f", + "code_tokens": 104, + "prompt_token_ids_sha256": "fb1dfb9b3ee30d5e8b81ad1fb09199a919c11793a38e75fa06667a56477e630c", + "prompt_tokens": 121, + "task_id": 187, + "total_tokens": 225 + }, + { + "code_token_ids_sha256": "d139db20f2cfb66831d25173ad82898b84bc546f661ff849066d774b22cdcbd2", + "code_tokens": 68, + "prompt_token_ids_sha256": "d839a7fee352db62223ae664372142609319762829eee2ea72698cf9b4827f26", + "prompt_tokens": 80, + "task_id": 188, + "total_tokens": 148 + }, + { + "code_token_ids_sha256": "a5e4823dc708d80e6c310565a812d8af835269eaa92220eb286f667ad6322ad1", + "code_tokens": 145, + "prompt_token_ids_sha256": "4d87cb11a678f889c1766dcc1492eb6a7e9e74814e07bd2d3e11ee2e089052f0", + "prompt_tokens": 109, + "task_id": 189, + "total_tokens": 254 + }, + { + "code_token_ids_sha256": "5d6aa72826025fa3b03ed5fb9dca68d9cbf36cd784e753fbaa58e150d16745b1", + "code_tokens": 41, + "prompt_token_ids_sha256": "9f98dc121e182da76da9abfa4f087c1e10189e406b50f35b8b916469cccde28a", + "prompt_tokens": 105, + "task_id": 190, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "60d6e8b118a73813a82e328770401fdfc3d8c54de0cbf544289865c7c65bfefc", + "code_tokens": 53, + "prompt_token_ids_sha256": "b59a12ce39649bf7b7e727d20914c95e0f0cc46392e92daeeaec1bf558adcca5", + "prompt_tokens": 71, + "task_id": 191, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "ed97dcd06f5692c66e65a9eaa1dc6609db8589a3bb4bd9a3e903e6c4d628026a", + "code_tokens": 58, + "prompt_token_ids_sha256": "5ce7c5026ff360de3446160e5d6e5114fd1a7f828c83d89acb16fab0b70fb040", + "prompt_tokens": 74, + "task_id": 192, + "total_tokens": 132 + }, + { + "code_token_ids_sha256": "b0d5c23ea5188663e3ce069d380a3ff7370562999b6a49c2b4bc68481d53d10a", + "code_tokens": 24, + "prompt_token_ids_sha256": "13a2bede5c228292c472dee015a863e023e3afe4e24dc952acded14541580956", + "prompt_tokens": 191, + "task_id": 193, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "9a9b4ba27395b31f7f121f958a004e16c3fe02857372c6796fbe2a9dfd5c7433", + "code_tokens": 89, + "prompt_token_ids_sha256": "3c4d527dcf5228766e1a01af841a25057d709fe185fdd0a55e3e99e66d9cc408", + "prompt_tokens": 84, + "task_id": 194, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "b8269674bcf1f7c7c5c7c1449ea1fbc997b100f338734fefd665b4016fc86fe0", + "code_tokens": 101, + "prompt_token_ids_sha256": "c7c64aec93785460801b7e4c75ff155c50b74004fa9566ec9ccb95d35adbe545", + "prompt_tokens": 114, + "task_id": 195, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "07610d933ee70dae6bbf3c30154d4fd7e987ea88593696836a205759e7e98bd7", + "code_tokens": 33, + "prompt_token_ids_sha256": "90fd977da03056837c645c68ef7f2d89acf8b4c6a058e1a5f027f71ba7a4af30", + "prompt_tokens": 232, + "task_id": 196, + "total_tokens": 265 + }, + { + "code_token_ids_sha256": "a35ef97ab6eacd1bd835f2bfd423febbb7a9f78f1a5cf0a49234b9e151aac646", + "code_tokens": 50, + "prompt_token_ids_sha256": "ec124878a477c9ff740dc702354fa802305205576d0a7b7b5e94444cd1a502e3", + "prompt_tokens": 234, + "task_id": 197, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "0d567a2e28d792776fbd484a0ecb3a83b51fd2e10eb476523e7146a10b9aae96", + "code_tokens": 58, + "prompt_token_ids_sha256": "1bb457ccd1edd997976dec26b4cf4326d63604fb2b8c6bcc83a68ea458deed22", + "prompt_tokens": 122, + "task_id": 198, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "afa5d4703c4163f3ef1fc78769320bc8fee51e3f62f22f7f72177f23a62658ee", + "code_tokens": 62, + "prompt_token_ids_sha256": "c9d979e6bc3c062fd44b9ebf8aa0c805f4853ede33ce5167a53bbb690fd63db2", + "prompt_tokens": 92, + "task_id": 199, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "f2c3817b940baeb158cb474b534ee4743a1d0dc55aa9dc36cf52ba5ceb99eade", + "code_tokens": 42, + "prompt_token_ids_sha256": "828b3078cf641ebec87f86d0d196467e90ca49c2ef73c612d0c06d730a4206e8", + "prompt_tokens": 164, + "task_id": 200, + "total_tokens": 206 + }, + { + "code_token_ids_sha256": "cbf6c3afb4270ad87cf4239319d648167be413e25e69cbdca943d47d196029ff", + "code_tokens": 15, + "prompt_token_ids_sha256": "166f51a6361aacf262c919b17aa855bcaba1600be2f7800c65a0f36798ea05c4", + "prompt_tokens": 84, + "task_id": 201, + "total_tokens": 99 + }, + { + "code_token_ids_sha256": "986fa8cdc9d20da4c5cd62aa1a6ae3040012059e11d5277509b5af120020b774", + "code_tokens": 57, + "prompt_token_ids_sha256": "3a723c5fc4c6b48ee08f5a1195883e36ba06499bd382097ac5d672ec30689d2f", + "prompt_tokens": 69, + "task_id": 202, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "a113f943d3232fa433abf1fabf98be2d5a2f4c10486eaeda5b8414ff4d4eb2d5", + "code_tokens": 58, + "prompt_token_ids_sha256": "c8ca01ae05cca614e76aeb2c07f6cb574e5cd05b5d0767a828b6c7c09bd8ce22", + "prompt_tokens": 83, + "task_id": 203, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "cac2cfbfa6ebdca0db76a33007956f9b5f063d928cd4ed3317d4d3a23b6a231b", + "code_tokens": 45, + "prompt_token_ids_sha256": "b007b77e856e8ec5a5dc817b6e6280d4c9865bb2a2957fff05aa6388b5b9c3c8", + "prompt_tokens": 81, + "task_id": 204, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "cc665b15da9247e02d581103c60cce6bf1a0d29f2313ad531cad01744953354e", + "code_tokens": 32, + "prompt_token_ids_sha256": "e0b2d98e36365d9e77988f1aa14ba0d0319e0490372b2c226efec8b8890f2dac", + "prompt_tokens": 178, + "task_id": 205, + "total_tokens": 210 + }, + { + "code_token_ids_sha256": "09f19e1011b3ba62968a600964e1ec39a478b6fa36b2822791ce18e23d6e46ac", + "code_tokens": 39, + "prompt_token_ids_sha256": "1a8a0b74b650280ae43ec6d1d6cd5f43162215adb528aaa3caa0a25b9229b36d", + "prompt_tokens": 180, + "task_id": 206, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "bc6c3c51d82fc42a20e78c3fe1cd20e42e13905ff0bc1292e944667347244ef7", + "code_tokens": 136, + "prompt_token_ids_sha256": "2a40fa8bca66ae8fe488fda69e028cb3e7666c503c726a8266bb7a2774cff4e4", + "prompt_tokens": 102, + "task_id": 207, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "5401e9f353039fec9dd145b1d35afb150f4a0ad999549940222bd5f2119cfe98", + "code_tokens": 50, + "prompt_token_ids_sha256": "f18b57064600a5c68db29a2e18f7a7903256b62d1618f66696a029ab9e34aeb0", + "prompt_tokens": 86, + "task_id": 208, + "total_tokens": 136 + }, + { + "code_token_ids_sha256": "f58e9f2920ecd6fb3bf990b6cce6bd973174df3639dcd3a1e13dac8c5b7abec3", + "code_tokens": 34, + "prompt_token_ids_sha256": "89f590174b5a08b3bd04c2c612ea3da5746f635341c30b1125649f6c5dbf7f3b", + "prompt_tokens": 239, + "task_id": 209, + "total_tokens": 273 + }, + { + "code_token_ids_sha256": "97e4bbe3e436885e42cea753db4ad3f757b124cd7a2790d61005dbb1938f102c", + "code_tokens": 44, + "prompt_token_ids_sha256": "3b7461f32e0894b1fc598e3c65f5051bf455cfad1ab33fdc0cecd8fb2fb36a71", + "prompt_tokens": 113, + "task_id": 210, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "1db96b10534bb35bbca5c04eccc6feaaba0e34895fb8a6b7a081d299efa05635", + "code_tokens": 36, + "prompt_token_ids_sha256": "c15f789b4af27b11c1765fb70fecba72d879d7d2cda5e5488ce0b5006a41ae90", + "prompt_tokens": 72, + "task_id": 211, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "85a7f5857728b8a9c8af70514bf95752bc7beac5db9bbcdd5a259e3012fae730", + "code_tokens": 46, + "prompt_token_ids_sha256": "cae640bd293c193a0fd9869e1e49672c86b2a241cc1ed72adc4203e28f12edab", + "prompt_tokens": 87, + "task_id": 212, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "b1e83acf8b4cb425d70474c10abd7abc7c692ddd2a75c5c91e1d9e13ccc952a4", + "code_tokens": 48, + "prompt_token_ids_sha256": "a154fdbf54e4c26d9f0d8dafdd0ac4be1b327c8ffef4219b0c5c55f35c8df6a6", + "prompt_tokens": 192, + "task_id": 213, + "total_tokens": 240 + }, + { + "code_token_ids_sha256": "d0ba927eaf1737b41df05fc52b9b3a4ef44a74cec5e25e0729a4ffb45d01ecf7", + "code_tokens": 25, + "prompt_token_ids_sha256": "9e147c3bd288a523679c1d7e19f2828747cfa1573b6c67dbdd4e685df0f758a7", + "prompt_tokens": 114, + "task_id": 214, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "73e98d63c1c318dd63070fdbd353dc67def23b822fa11a4c5a09246966d71b47", + "code_tokens": 69, + "prompt_token_ids_sha256": "1d242f9989d3bacca8f2c7e506ab17e66045c0157061ef8e203762125b9ddb0b", + "prompt_tokens": 206, + "task_id": 215, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "b0ea084ce7bc7a71e7562cc8818a1ef31df55c4c85d5ff4c1b45e879b8f11413", + "code_tokens": 60, + "prompt_token_ids_sha256": "7f7bcf3ad6d41ecf4c626a880e7dff039afd9f42875d1a179fd559ac9ac833ee", + "prompt_tokens": 225, + "task_id": 216, + "total_tokens": 285 + }, + { + "code_token_ids_sha256": "de38e6e3a1764ad0177678550f1fd618b97616ac9c946db2dd80e6ce0e443f8e", + "code_tokens": 49, + "prompt_token_ids_sha256": "0c4dbb8b9d5dac3999183466b790f91740f4a0d24b354a0d6504fe3a74a12cc0", + "prompt_tokens": 80, + "task_id": 217, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "380ac12b1136a14b989f221ef4b04b7e189ec9416fb38ffbcd9be5e231a699ec", + "code_tokens": 44, + "prompt_token_ids_sha256": "1755292276bfe69746eac43c0e45c9086ad2df8b1e4e661b31d1245444898ff8", + "prompt_tokens": 85, + "task_id": 218, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "b717d54e1e569b50b32a9c61973d1d0adaf3444fd4507fb1b82db46568edd048", + "code_tokens": 81, + "prompt_token_ids_sha256": "a7414041095409c8e76e81c314fd86ddb19c4d47a94e756cbe74f4779ffe95d7", + "prompt_tokens": 180, + "task_id": 219, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "e1cec3b3ab423a8236cf9ae3f84a8e8a99b28e46c0b2c161059cc27a0d7b1ca1", + "code_tokens": 26, + "prompt_token_ids_sha256": "32383c5724e2d52280d67a1cbe310ebfab8a1e48661219e710e63b760f1849a3", + "prompt_tokens": 117, + "task_id": 220, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "d6ef8f4bc995bef359a1f6e722cb2a4e8f3974ae43c2b1ab2caa4f6aaf4fe25f", + "code_tokens": 32, + "prompt_token_ids_sha256": "97b5bb2b4625a0f77115248f16d9fc73abcdd88524ba92f9f8909c4f0600aeb6", + "prompt_tokens": 106, + "task_id": 221, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "475b43c3a4ceae032db29ae908dfc68ab08592448f6551138dc6f4a15bcdd573", + "code_tokens": 48, + "prompt_token_ids_sha256": "3237cf65c2cef968d5496bb3bc23252c780a7346b63803bed67fcd08d0e62708", + "prompt_tokens": 107, + "task_id": 222, + "total_tokens": 155 + }, + { + "code_token_ids_sha256": "3f24a5c0ef8ff166e016efda816d6bad63db4aced7957556dfc99eae29936cb3", + "code_tokens": 180, + "prompt_token_ids_sha256": "c675676eac006f650130a88fb8f2ad8c1ff536907c760823c093da6c43476e64", + "prompt_tokens": 140, + "task_id": 223, + "total_tokens": 320 + }, + { + "code_token_ids_sha256": "6e0cac86970b2cfe3f2fd3a2b0503b300b800c7b0803f20d8a2c631bab70aac7", + "code_tokens": 38, + "prompt_token_ids_sha256": "2936d60cb56c915430d4fa8e8728399005717f74cefb368817b81fce053f6471", + "prompt_tokens": 75, + "task_id": 224, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "34866b482398acc658f1baabddb841aa878563ea84a1c346c9bcabb653d7c11e", + "code_tokens": 90, + "prompt_token_ids_sha256": "f43bfc2879a8c32552b645dfffdfb06039bc65526f531d77def66eab53bc3392", + "prompt_tokens": 104, + "task_id": 225, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "abef974f6c39a8bd010f6d91d51cd54681b8f24ad10b71e6b7bad0d2b47defd1", + "code_tokens": 44, + "prompt_token_ids_sha256": "80a8c2f0dc4c11bccc662c0acd8279eb726049115abfe656bff402853f9a3f99", + "prompt_tokens": 80, + "task_id": 226, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "a783cac4b4229e50d41aa6c2736ad86dc6dc7265ab2360acd670210710bc53ee", + "code_tokens": 60, + "prompt_token_ids_sha256": "9d910fec15aa7d8bbd5e2e999bc5d02739ed0566f3752fb1edf12febe966271c", + "prompt_tokens": 86, + "task_id": 227, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "7e9387500fcc0957326cb4b4b1fb7ae5e2ba69ededfe2e4d689c33ffd4d97297", + "code_tokens": 69, + "prompt_token_ids_sha256": "db9a2071137b0f53d5c3b034ca1b081ea6d01ad43d92cc64d4ff698f6eea7793", + "prompt_tokens": 113, + "task_id": 228, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "47d4610c97a7b06c80b016c00ef196a037f8e834cecb2cb1cdf36b43b2363889", + "code_tokens": 71, + "prompt_token_ids_sha256": "96cff410463e11af70d48e4e1afbcbe76259e3cd4f2e0a443802b9b430a6124a", + "prompt_tokens": 230, + "task_id": 229, + "total_tokens": 301 + }, + { + "code_token_ids_sha256": "56b0b38e212ea05906be72fc78321530aaa48bf22be2218793f2eec1cdb1c514", + "code_tokens": 22, + "prompt_token_ids_sha256": "08dfeeb31c07a37b0a9d4e2f135cae30268fe6737ecefcb45e5dc8ec1cc0ca77", + "prompt_tokens": 87, + "task_id": 230, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "daebb9823bf70d74f6d681811d409bc5b7303194eaea763ec6f34db093026429", + "code_tokens": 215, + "prompt_token_ids_sha256": "7de30941067ec90af881026ebc10070653f2ff08c1a5ceb45eca13b4ada9f3fe", + "prompt_tokens": 126, + "task_id": 231, + "total_tokens": 341 + }, + { + "code_token_ids_sha256": "23cab822a328cb65b24da1fab6182bf4c848627cde6286b08b663fdbd5009b02", + "code_tokens": 26, + "prompt_token_ids_sha256": "e97752b024aa100e74d2d8c7c82ca70b20c70c1a6df1d2db6c73d392be54afe7", + "prompt_tokens": 231, + "task_id": 232, + "total_tokens": 257 + }, + { + "code_token_ids_sha256": "e2699c013f0b2c0bd2551a58d20fbbac0755b99dda5f2f35fbcfe1c2269ce88a", + "code_tokens": 33, + "prompt_token_ids_sha256": "9b0907adbfcbee6031d3e78fac17a06b6bdf25e8aee730028be0aff318aef97d", + "prompt_tokens": 134, + "task_id": 233, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "234ce61593f272c0ad5b711f5b306fa39ef751806a8ec3ac329b250aeaae9798", + "code_tokens": 18, + "prompt_token_ids_sha256": "d36eeec6ca1b857694096c551e87e09ad8cc4a27feaf1ccdbb9846e09c7c6c84", + "prompt_tokens": 64, + "task_id": 234, + "total_tokens": 82 + }, + { + "code_token_ids_sha256": "b239ca6968d8fd4f359e614a9b75b8862359576af2134d513977b84a00762b5d", + "code_tokens": 71, + "prompt_token_ids_sha256": "a97a589800360c271204da8bec930d10e65242c6e89ea8a3e949ac80a8c406d9", + "prompt_tokens": 82, + "task_id": 235, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "919ddf651a96238dec7924969f9d25d1ca34f678d0b4a7b17dfc643a9f51b612", + "code_tokens": 107, + "prompt_token_ids_sha256": "704c4a6bfecf81453265645c5a616eaf2355712476c514b5fcb526844000dc85", + "prompt_tokens": 92, + "task_id": 236, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "70bf0fc5a8c4f0d80de09a2b20b3b08c1d09ed2cd0703833d694b8b2b70dcd01", + "code_tokens": 40, + "prompt_token_ids_sha256": "c241057a9df2d59a5f69779f90bd1fee25c2441e343d9fcc54af34ca366d36a8", + "prompt_tokens": 264, + "task_id": 237, + "total_tokens": 304 + }, + { + "code_token_ids_sha256": "e0492e08873486c15f4e3066b114632317cc53c5b468ef2f2ab3c9a4a226aff0", + "code_tokens": 32, + "prompt_token_ids_sha256": "ff41fa7faa9f882631ae534c66c9bcf56ccfae1478dcb5b760b44dae839baa93", + "prompt_tokens": 82, + "task_id": 238, + "total_tokens": 114 + }, + { + "code_token_ids_sha256": "c6b3742bd8af5c6c71c801afaca3db9f92499e2ef09a9511b1b75ad08beb30de", + "code_tokens": 130, + "prompt_token_ids_sha256": "fcfc99f4ac0d6157901c3774fe7a325bd13fac5d727993ef907529a13bce4d1e", + "prompt_tokens": 116, + "task_id": 239, + "total_tokens": 246 + }, + { + "code_token_ids_sha256": "7af0a4ba16719b2c864581c31847c9604d24544087a50d7e1e7b0c2e22fe2828", + "code_tokens": 27, + "prompt_token_ids_sha256": "b19ca4896dd3a514c7f727ff412231acb29b95eb166de3d46a0f3da78a834c95", + "prompt_tokens": 163, + "task_id": 240, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "fc047b378c71f5c99427c52283791d0cb32ec001b132a9d4007994138b0a7be5", + "code_tokens": 42, + "prompt_token_ids_sha256": "76db572975115261a3a9f2f06c39f6afb71874632768cd83c27c9f3ab9bb8362", + "prompt_tokens": 284, + "task_id": 241, + "total_tokens": 326 + }, + { + "code_token_ids_sha256": "b00483be45fda7685c229083002af43a510fda97db098481e26a1d05f03c102a", + "code_tokens": 30, + "prompt_token_ids_sha256": "1d570c8d5e1f38ca2a29cf35d7e89bed6137ec83758e3f799765570f48f026a9", + "prompt_tokens": 66, + "task_id": 242, + "total_tokens": 96 + }, + { + "code_token_ids_sha256": "6609e9e0397f45000d3ddd967f5e3d9649f6f39c32e61a67649c0147a965cb60", + "code_tokens": 58, + "prompt_token_ids_sha256": "d43164ecaeb94871845f204bfcf960f73c8d78f55258c7058e011d58fcea907c", + "prompt_tokens": 198, + "task_id": 243, + "total_tokens": 256 + }, + { + "code_token_ids_sha256": "0f94ade8be0cbf360e4ef6a0134348c684efb856929e9c0be8908d4d6e3a63de", + "code_tokens": 34, + "prompt_token_ids_sha256": "470663206e8838228cdbcda6e1c7d2c6f24fcb6c351d8df4f6bfca568a81cc7f", + "prompt_tokens": 84, + "task_id": 244, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "e7f52dac737ecc657e2b3412fdfbc25cf85cac0a1e8b261f60a0d0e69d1324db", + "code_tokens": 208, + "prompt_token_ids_sha256": "defc82e72463ea2128edb4c26f8140eca796bb92e31e63237943d57f72cbe5ad", + "prompt_tokens": 170, + "task_id": 245, + "total_tokens": 378 + }, + { + "code_token_ids_sha256": "f92e626be53b19e58897729f0d5f6391ed5d7ed8764e922744c50d804fbe6ae6", + "code_tokens": 83, + "prompt_token_ids_sha256": "d2d77241adab8f2fe90e29472cb0bbad0370ef5c3a751facfe6279df200270e7", + "prompt_tokens": 114, + "task_id": 246, + "total_tokens": 197 + }, + { + "code_token_ids_sha256": "b2b98146b214dbf56e5dbbd0d0b0b69445068eeedb7ce636406cad869ca63494", + "code_tokens": 170, + "prompt_token_ids_sha256": "67cd6617d7aabdfeb1cd90c808346b314a49ad844a3065ae38b3812b38e1aa12", + "prompt_tokens": 89, + "task_id": 247, + "total_tokens": 259 + }, + { + "code_token_ids_sha256": "b0efb73be82ea96bc295fb098efaaa64f4604104d2eb2fa6466049b697d25f1b", + "code_tokens": 40, + "prompt_token_ids_sha256": "d82e13b99a99dbaeff6fc4f3671df3649a2c5e27f71cac10d710ea23f955852d", + "prompt_tokens": 119, + "task_id": 248, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "6f3d8c15aba88e12b5a695298a46adb02cfe5fa52a38744c086d94533732cb7a", + "code_tokens": 31, + "prompt_token_ids_sha256": "cb019137b599c0764c85ac4a45db9b81f0bfc053d490d32759ed3b396995eece", + "prompt_tokens": 186, + "task_id": 249, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "4a19fef1a12b3295a7eaa4b140c5297601d771b52002326e995d312ec4194ef2", + "code_tokens": 42, + "prompt_token_ids_sha256": "b364a3b0893356eb88ae8c53d6172b531654db8b9544797d0eeebc288f457fe0", + "prompt_tokens": 192, + "task_id": 250, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "41294bfe28efa11c4d3f4c32a6d0c0ea9a535e863a24248649134f8d9e548866", + "code_tokens": 27, + "prompt_token_ids_sha256": "bf6a8f33258bb94b7d1bba11f4c5d402084436a0ada5f3ea3eb283e2f86c834e", + "prompt_tokens": 127, + "task_id": 251, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "a1b0ef320f8c2c9e345fc6653ea52923d116ac1fffcb6b9d8e9273e056319599", + "code_tokens": 25, + "prompt_token_ids_sha256": "c894d9fa0c4c7d5ed922c52cb280125a3fb6791316b47371d947a9c5d178f2c4", + "prompt_tokens": 87, + "task_id": 252, + "total_tokens": 112 + }, + { + "code_token_ids_sha256": "e03acec21469f3c26439a2f82a1790468c9847336c554eddc04dea71d92e28ce", + "code_tokens": 40, + "prompt_token_ids_sha256": "7ccb43e8bacc6cd4c3a7e290e4685f873511b1cb03ad7013244db39404d93dec", + "prompt_tokens": 90, + "task_id": 253, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "5d54cb7aabfff45142e1c29acade9d71645f668cac51c2bcacf9da45a861abe7", + "code_tokens": 24, + "prompt_token_ids_sha256": "c17505b4a47c275546d0b1b18f4dd67b332c13b2bfdcd1cf412e7e75531ce426", + "prompt_tokens": 91, + "task_id": 254, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "e21afff0456159d4262195bece5c9e4af1858a95e25a48d9cbd8578fd7732480", + "code_tokens": 28, + "prompt_token_ids_sha256": "f8110b71f6d5c29a0f443e12c2fa34a7fcc64e61db9d85282279eb69f5299eb0", + "prompt_tokens": 230, + "task_id": 255, + "total_tokens": 258 + }, + { + "code_token_ids_sha256": "f973d8a03763420c5c08b409394ac80a1af0318b6b86f76d52e45ce5c5b6d9d6", + "code_tokens": 71, + "prompt_token_ids_sha256": "f31cde895412eae78889b8044eaaba99e3b0e8daefdc98f6845e82a5042d2756", + "prompt_tokens": 85, + "task_id": 256, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "2725f10ead9d860ca79dc40983c106aa18fed0e7b27919fbbe35acea7e3bca2c", + "code_tokens": 24, + "prompt_token_ids_sha256": "07e69d33d9aec68aa92a26ea0b962087382af56ebb79993ebab794c80288e778", + "prompt_tokens": 92, + "task_id": 257, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "3effc8deb7a150990a7b662cc91b00d53d60e2edf5c725ed1f1f245dbf0ade6e", + "code_tokens": 34, + "prompt_token_ids_sha256": "070b9f90e81c7309639f4ed8511197af41f20a2b3038b0b48718f5a109dc9993", + "prompt_tokens": 117, + "task_id": 258, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "da71ba9c5a6967c53868f4c9d6ef04b75b0a616fb5f2be5de3b75a8e49c50965", + "code_tokens": 64, + "prompt_token_ids_sha256": "50850da5980ea377bbb9c2c21a52941f8bc5eb01a7fa0c40c722fd2ae8d622f1", + "prompt_tokens": 281, + "task_id": 259, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "96630a5b55281558319792591d52b56d94541991787db888df2c994e46952239", + "code_tokens": 45, + "prompt_token_ids_sha256": "0b43ed39bd0a3c3b5d851459c9ad6190b04c50e727d7ad846e553f8fd99675bd", + "prompt_tokens": 79, + "task_id": 260, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "d40216bd02e20f150a228e735808481c5a17b5b3d731c0aa0553ac5426ef6bef", + "code_tokens": 48, + "prompt_token_ids_sha256": "1616d93398e765b85316ef9472886a1bac55a28379134a4f1855467f0350e7c4", + "prompt_tokens": 166, + "task_id": 261, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "2f7d8943c08ced9925e6b2b1c8d1223c8f0a6be0e8f075d61f4219bfed7dc3cd", + "code_tokens": 21, + "prompt_token_ids_sha256": "bdbdb49bbd46bce804ca213c65e444eeb1b28e44be7d5a8894af93f8a2d77c0f", + "prompt_tokens": 172, + "task_id": 262, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "84bd93ada6cb58b27eef46d3091526f68b3a53ba43f937558e873f02510bda72", + "code_tokens": 24, + "prompt_token_ids_sha256": "47a204156ce7270042607d0dade003b3b663b13f44c975cf5f98746d8007f2e7", + "prompt_tokens": 231, + "task_id": 263, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "d954c386cf791272f5bbf03524ca839e2c1698f357a25009fb404894b82f5e6c", + "code_tokens": 63, + "prompt_token_ids_sha256": "8e349bd15a44a1fbf6290180021c785c108b13c677b7b7b02626956f2a6a70ba", + "prompt_tokens": 71, + "task_id": 264, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "9cc89430cd08d19568cb051134953d518177f7247cdfd411abf70a58c6782728", + "code_tokens": 22, + "prompt_token_ids_sha256": "71abf479c06e19f0ed55e611f6fd7d9bc98acc48c3219c498366479911a1b6fb", + "prompt_tokens": 245, + "task_id": 265, + "total_tokens": 267 + }, + { + "code_token_ids_sha256": "b1816656c377bc4dd231013507a739fb29d562e8d63e389754e5e455bf17dfc0", + "code_tokens": 25, + "prompt_token_ids_sha256": "5097c110bed3b41d206bff1d4c4494e7e9dfc7adff0d00f369ea42f32cf6de6d", + "prompt_tokens": 76, + "task_id": 266, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "38953e7e0535306f280b002dcf250cb4ebf49634a3fce514623740236e207a3f", + "code_tokens": 21, + "prompt_token_ids_sha256": "318c479d5082d705c1563a01084f9f6cad1c56bf2b11db659378c7b0d4d87eeb", + "prompt_tokens": 79, + "task_id": 267, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "9eabc3b55391abdfaae730c6c17e4302cfdef3f8bfa8ef4127408842ad727043", + "code_tokens": 24, + "prompt_token_ids_sha256": "1bfe78bcd9bfaac89dfcfcec8c7fb5fc87cda1974a249b8b32b451f53183e6b6", + "prompt_tokens": 75, + "task_id": 268, + "total_tokens": 99 + }, + { + "code_token_ids_sha256": "fe95755c6cf002bb60952eba720d0d3fdb70c62efaa8fd4da0f8b89f423eeadb", + "code_tokens": 15, + "prompt_token_ids_sha256": "3c9254f418fc2802aa83a5f44e98d2d0742732ab6396b32ed0946bba62af1074", + "prompt_tokens": 65, + "task_id": 269, + "total_tokens": 80 + }, + { + "code_token_ids_sha256": "91e51e3c14ae4e993ae9e80c9212a7ccb2c3d2497011a1bb2056a92bcf5614d1", + "code_tokens": 59, + "prompt_token_ids_sha256": "0b679a3e505920dd147cc97430389671330273d140185e29d11c3f240fe44a36", + "prompt_tokens": 150, + "task_id": 270, + "total_tokens": 209 + }, + { + "code_token_ids_sha256": "b09036b5ab2fc96b5034376f248313484c321b48a06c4efa7d24537aeecb6a94", + "code_tokens": 54, + "prompt_token_ids_sha256": "4329b4a7e61193f2d9c342decf6ace1f15099fa0002be93c6a91c35536669740", + "prompt_tokens": 90, + "task_id": 271, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "2aed1363928fcfcca52a42ff95d22452b4fe0d6c9d2bdc7d3712cdf17d534dc4", + "code_tokens": 28, + "prompt_token_ids_sha256": "513675d6ccea89515e3dc8d8d092abbe7600eb19baf544adf500fbe824b3ddd2", + "prompt_tokens": 196, + "task_id": 272, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "6caf76ae44bca4b49763423e9e780dc5b565a65d8a3918b74b066e4c52b0f7a8", + "code_tokens": 46, + "prompt_token_ids_sha256": "b518c9b93bf1b2036412fb49d02855558f4a4718ae6290689b63686fc21edc82", + "prompt_tokens": 156, + "task_id": 273, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "47235192c831a9f8a91bb3b7e7f8cc352d64e378047c0b9784e67b7d8855f7bf", + "code_tokens": 27, + "prompt_token_ids_sha256": "5d4a69e25d8afab56faa8d83fa3eff421369920e5c4992d420c134051759757a", + "prompt_tokens": 86, + "task_id": 274, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "e8fccc08833967537afd1cec5ef1fdd45433e4af21caf9b3b2712b1c4870c03a", + "code_tokens": 97, + "prompt_token_ids_sha256": "2a80fa2b7ef1b12b2d80fda36bdcf63e67f5371e9a07a07c7eafed9c175e2019", + "prompt_tokens": 98, + "task_id": 275, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "ad29d0034bf8bfec19f00f9679116b46a62c96ebe0db659d82a032c4ecbc96b3", + "code_tokens": 24, + "prompt_token_ids_sha256": "89b9a6078ddbe833058b6f08f4323d02f510df634ecee0df416664c6258b2c1f", + "prompt_tokens": 123, + "task_id": 276, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "c514e8c1e1e022fc9864b19490e1deeeedaccd4cc78c84c402fcf36666a43baa", + "code_tokens": 30, + "prompt_token_ids_sha256": "5bc3c4037eb48633f8d08d540e26f510979355c1604e1e6e210e78bf10fcb68d", + "prompt_tokens": 245, + "task_id": 277, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "7b3e2c319f7328bd79fd7ed78e92e2d63cc39b34cd278147b1a678956be958ca", + "code_tokens": 38, + "prompt_token_ids_sha256": "cba934f062512e8c749bc155ab7b91700379f8ee1a091b23f47d27c4b6315c2b", + "prompt_tokens": 129, + "task_id": 278, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "8f390c0ad7e639b9dc62e44e40611ed07b27b81770702abd702fc23647049531", + "code_tokens": 21, + "prompt_token_ids_sha256": "20a77f262b7bab1e4342522f7581d7182a7f67937c2a5be81fc6f775cac7188b", + "prompt_tokens": 79, + "task_id": 279, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "f1b252af9269d57e57d1f72914184c91e7a8e8475c5f5c7723d545b1b9cb0c60", + "code_tokens": 65, + "prompt_token_ids_sha256": "b9654b8a48ad76824aeff8aaa31b8a84e1d792c108b524fa37e6c54bf0f9aa51", + "prompt_tokens": 178, + "task_id": 280, + "total_tokens": 243 + }, + { + "code_token_ids_sha256": "b6ff50c8503f5e49e0ab0d9c4febe7129689b9ee6387db743e3da2fa9e9d8e3d", + "code_tokens": 27, + "prompt_token_ids_sha256": "8468d1a6d0a79462dc2c37a002b240166b6d66dfe729b7aba14a56eea4a95248", + "prompt_tokens": 89, + "task_id": 281, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "588cb17cedf2755433004dbca17599d039d6e7f58c2173e038af268f78509683", + "code_tokens": 35, + "prompt_token_ids_sha256": "8abb5e76d813fed7e8c4394b380be2dfedf8bbfee8b7ee603baa892445ee6086", + "prompt_tokens": 110, + "task_id": 282, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "cfdf96907a4091d1f4a915809bf2882b632d3a72ff172e1a4f5da54294a11344", + "code_tokens": 75, + "prompt_token_ids_sha256": "f14a398c952be6b17a78984e101c9317fb0bfb02666c0ab5d121783ee0241591", + "prompt_tokens": 81, + "task_id": 283, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "2926f1c15fa71693cbbf7bd38bd4ba2e83eece458686470dba233aeebeaf57d7", + "code_tokens": 25, + "prompt_token_ids_sha256": "5c508c6de618883a83acdaf9a22435153676b0c310a4b322873f543e2f1febad", + "prompt_tokens": 100, + "task_id": 284, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "746a2dbfe3efa38185119e338352fe1faa9ed16003bfdbf3be91b2327486e6b5", + "code_tokens": 52, + "prompt_token_ids_sha256": "9ec060ed2c1fc11e6747c012bdf26386f4acc81f51248c895d8487ea33860d8a", + "prompt_tokens": 90, + "task_id": 285, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "a1f66ab85fb840ca70ddc44516828eddedf6a33115f10603f7372539607e73ee", + "code_tokens": 110, + "prompt_token_ids_sha256": "4d47e477fa56fbd6b6bfe2f971e3774dee99eafc3406b7dfda1189a2e8a617a7", + "prompt_tokens": 140, + "task_id": 286, + "total_tokens": 250 + }, + { + "code_token_ids_sha256": "06f0c99cd7dd703f7010d077408d81d7cd057a1ed6864108c96a9d3ebb7c94ef", + "code_tokens": 25, + "prompt_token_ids_sha256": "427903e46acca3e807d897f55d4ce4c3fa579b68b5c0ac23a11688ce05635d79", + "prompt_tokens": 80, + "task_id": 287, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "97abc37effd3462f77fc7a58397566b409cd4925e01b7cdc5a79e5ee812ddcb4", + "code_tokens": 56, + "prompt_token_ids_sha256": "d187d21ec8b25cc0aace1b38540057dec0f47cff29c6da713f62b8bd173c46e4", + "prompt_tokens": 127, + "task_id": 288, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "f94c23be8c3554d475069ab8f9be2b1e005f01d687a48c704ac62433c15b277c", + "code_tokens": 108, + "prompt_token_ids_sha256": "89af3ddff31e2d47969db57f24414a07deb950246557ce9d3126c919564b217d", + "prompt_tokens": 78, + "task_id": 289, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "b79cfeb9c49a151ee97aa2149703fed78cc0575e032bc1eca0b7140fe2d3d1fc", + "code_tokens": 47, + "prompt_token_ids_sha256": "9fd0997cef74ff3268e06292dcce6eacf7e99d4db0140df4bbc96b8fcc017329", + "prompt_tokens": 168, + "task_id": 290, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "6853b99b4dd6e9724213cbba9355d8ff619faa10880e2cb161ec1fdf09d8e51a", + "code_tokens": 102, + "prompt_token_ids_sha256": "c3cd9e976d4b1561bf833582c4ffd48296cabd0cf4dd237a4786ffa3db36befa", + "prompt_tokens": 114, + "task_id": 291, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "c080f5c882ac0dafd33b5e616fdbd906e79616be24efee9e8aae4a8cddf72624", + "code_tokens": 18, + "prompt_token_ids_sha256": "e2338e5cf8db36b32cd15cab268093b33b71027d9beb5ffe2eba3849ad53a999", + "prompt_tokens": 72, + "task_id": 292, + "total_tokens": 90 + }, + { + "code_token_ids_sha256": "b5b5be6ddea216f279c083ef842f9440e69f360cd9efbcba6e31222db4dd96f0", + "code_tokens": 28, + "prompt_token_ids_sha256": "86c27a9382b680f4b8e447ee4adebced0c10b2dabf1df63885165a04bea20202", + "prompt_tokens": 109, + "task_id": 293, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "fad0b2751211997e72184c62c1c822c7143c8953a8f2e062a0e49de446296a4c", + "code_tokens": 30, + "prompt_token_ids_sha256": "ecc4f8bf5bdd1ab4662692637fbe432b57d975506f07ebf217e07ca0dc0a3698", + "prompt_tokens": 114, + "task_id": 294, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "65eca9fb1b20e18d85062d37e1b35054673ffd0d7a7d42770d23b26aeae1f053", + "code_tokens": 48, + "prompt_token_ids_sha256": "64f48236577683da6309cc93f026f0d2ee92abf852b99bbebb22550611d8e848", + "prompt_tokens": 67, + "task_id": 295, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "8978a881872d24acb6fdd186430b3e206ca05b309e34753b4f7c7be3f85e2ac9", + "code_tokens": 59, + "prompt_token_ids_sha256": "ac1e4c70e4bc67a75b798550b41dcad0fe11a6990bd99aa6f039d06d61640d00", + "prompt_tokens": 101, + "task_id": 296, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "1026b4fb24043cc45c030ce6538d7390b65b820328dd62f00b35d4fb55b5a357", + "code_tokens": 102, + "prompt_token_ids_sha256": "77ac12db03bb4820e60544a2185c7fc8f8a861beffea0048a389d5836a679e5a", + "prompt_tokens": 310, + "task_id": 297, + "total_tokens": 412 + }, + { + "code_token_ids_sha256": "dfde2bbc6dd0e3d54048dac2a3eb6d63fb52d169f49fa6486d84513a85b80db6", + "code_tokens": 36, + "prompt_token_ids_sha256": "7319d50eee6865906e83e7c34c6c5e923d0502f4e9b8ffe86b2d636e8e480450", + "prompt_tokens": 264, + "task_id": 298, + "total_tokens": 300 + }, + { + "code_token_ids_sha256": "4a0f8b076df3656c1094d5a860bf5429371ebc1135dee12b5dd65b53acab4999", + "code_tokens": 51, + "prompt_token_ids_sha256": "3d28a716588cd5dd7eb0a23dbac2656439c96765030bbd49f6343e28ff082667", + "prompt_tokens": 210, + "task_id": 299, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "42187e9ba89f02ea4102218c0a1129f9fd982205b64e168fe97003d0c0a4673e", + "code_tokens": 62, + "prompt_token_ids_sha256": "f1ce69ac3e1182db673580a0437de2cccf6c0676769c704d8a84553fd9d18980", + "prompt_tokens": 98, + "task_id": 300, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "a9eb78bfcb93a2b81acec90b022773f83751e87af9d406f3614ab9f8f6867eb9", + "code_tokens": 39, + "prompt_token_ids_sha256": "e79860f59c0fc229bde5cd6d735d78e721b47f66178b7fe48d3dc6775eb79203", + "prompt_tokens": 103, + "task_id": 301, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "dcc15915869e48632b410e3481efe8eea98b769e99c0ba5c2af9c8e2161079e2", + "code_tokens": 76, + "prompt_token_ids_sha256": "7a2b4031414e743d46b883ed9c6014f1b0cc8674ba7e7e6f2e04eea51ba465f1", + "prompt_tokens": 83, + "task_id": 302, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "e27d043a13c36f65456c24b44d1575430be23646560d7c4a46614ca26e4ba030", + "code_tokens": 58, + "prompt_token_ids_sha256": "64cfdcc7cebf04420fed8451d23f94b3205ce5ecaccc072fb706869e791d18bb", + "prompt_tokens": 87, + "task_id": 303, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "eb20e601e18e47ea2f6e8345e527ca1834f294869abc8631865c10b3c2c96f3e", + "code_tokens": 93, + "prompt_token_ids_sha256": "ad69659912f31954e3140491b812b9be230e0ae612de417bcd666907334f535a", + "prompt_tokens": 135, + "task_id": 304, + "total_tokens": 228 + }, + { + "code_token_ids_sha256": "7f67bed1e023a23a0c496eddadf28d15a227103f438380d758737961428c2622", + "code_tokens": 45, + "prompt_token_ids_sha256": "81a0671a96af76f9ed286a90babe4067a4149f59905d2ca9c9e7a4b8521a7783", + "prompt_tokens": 109, + "task_id": 305, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "d6c68aa53bb8fd603fa3634acbc0058e53d0c116f2aa3848599b4f4bf0705a4f", + "code_tokens": 209, + "prompt_token_ids_sha256": "b3fbffd479e6f05feea4f344c65a51d1bd5fb2a6444c74d188e1dfd44609daab", + "prompt_tokens": 203, + "task_id": 306, + "total_tokens": 412 + }, + { + "code_token_ids_sha256": "5d3884471cdf672b49a9277eb767b583613ac938bc551800c62269219ba5d366", + "code_tokens": 46, + "prompt_token_ids_sha256": "e66e42622484ddae81143d1297770a69d50bc2d5a551a1c788778b9918474622", + "prompt_tokens": 146, + "task_id": 307, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "84d93d4b2d9c6400a32663c64ba64287f20ec777b9e2d6dc3fb866247c1ffbe6", + "code_tokens": 39, + "prompt_token_ids_sha256": "aa363b33e6ca8361ae3755fdbb469f829e78da6b25ae2b1064590e5cf23e90dd", + "prompt_tokens": 216, + "task_id": 308, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "aa5e67d8b6b32bb0cbf1c14c616b9ea524f190ab68618c46c64d0bef86e183f6", + "code_tokens": 25, + "prompt_token_ids_sha256": "687b6a2c12d4d4f45c839724d83b52e4b23a4e8b0f7ce062ca625063f55e0af1", + "prompt_tokens": 73, + "task_id": 309, + "total_tokens": 98 + }, + { + "code_token_ids_sha256": "25b30fbfead29dc756a712d9f2dcd0e4f5c2d1b5dcc32fbca57220cf9aa11a02", + "code_tokens": 28, + "prompt_token_ids_sha256": "0e420dbe94e95d4120b464cd452aae16a656331e769e5fdb289365665f305d9a", + "prompt_tokens": 124, + "task_id": 310, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "dfa4466dc980a8b41cdd9e05aca523499cff6fd38091bb1f6ca57cd40a15926e", + "code_tokens": 84, + "prompt_token_ids_sha256": "2d2f26e88e5b0550b5616de2a138d2bea32941081e19d7225a98baadb102e526", + "prompt_tokens": 83, + "task_id": 311, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "36754a97c72657d362c7da8971dfcdc9be0506253722aca0e632b06d9797a4ec", + "code_tokens": 34, + "prompt_token_ids_sha256": "a17b77da8d63b5e50d9837f5d1e332dd75447c7f021988d4f5788ffb3f6ceafe", + "prompt_tokens": 125, + "task_id": 312, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "666ee679d8acb5c4e083b3f1ac820f038dda37c3607609e6202cc97fe5874d57", + "code_tokens": 28, + "prompt_token_ids_sha256": "4c0ead4c7f8d8002e968db01da6a5d045eb72181091cc87a99156608b58a35f8", + "prompt_tokens": 89, + "task_id": 313, + "total_tokens": 117 + }, + { + "code_token_ids_sha256": "02c7888073976cd4491aa604a5407b13259e392e9681ca20bbdbf7df79c12878", + "code_tokens": 92, + "prompt_token_ids_sha256": "e8bdde83fbd0c6a9904c10947615efb962adcf512005271a060503ac759127be", + "prompt_tokens": 191, + "task_id": 314, + "total_tokens": 283 + }, + { + "code_token_ids_sha256": "83745feb4abbaac4f8b590bf009e44e0edc0f271206c567a174f1509f47b045a", + "code_tokens": 185, + "prompt_token_ids_sha256": "a89533c3f924be90d2d70ab9dea5de9a0264c2ed27cfc0f4462ec66dc67f127e", + "prompt_tokens": 88, + "task_id": 315, + "total_tokens": 273 + }, + { + "code_token_ids_sha256": "fc5b7a84f664f65a3b3fd433c56585ca32e29fe3a9479b640268ac9a70e593e8", + "code_tokens": 106, + "prompt_token_ids_sha256": "c7cfe87e58c3c90dda74d473355149fe8560f20c5de9ad90dadb40df69d27d52", + "prompt_tokens": 172, + "task_id": 316, + "total_tokens": 278 + }, + { + "code_token_ids_sha256": "fef63ed01fb8b56d2513f4fa4cbf7599ceaaaada850412ee9a9ffdf613c551db", + "code_tokens": 64, + "prompt_token_ids_sha256": "af97d343f8cf28d5c3b4c0a8f3156549408c958db01053de97489dea4e3c8bc4", + "prompt_tokens": 157, + "task_id": 317, + "total_tokens": 221 + }, + { + "code_token_ids_sha256": "2ad6d2b631d0acd52eb70ab455aee976c677f8fe5b3472c30ca1e0125f7f4561", + "code_tokens": 74, + "prompt_token_ids_sha256": "240974cb8469a3c11dcd617e1336a7ca9837ad1b69e608387c4e6bdaa07dc9a2", + "prompt_tokens": 76, + "task_id": 318, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "439efb016a802bab932ba127d1b9898cb25447273aacf9d7854d4a4a88fcc1c7", + "code_tokens": 27, + "prompt_token_ids_sha256": "796c6a121489ad5dd03b6eb067e9b7306af65b56fb5f7a8c6a667786a64435d2", + "prompt_tokens": 93, + "task_id": 319, + "total_tokens": 120 + }, + { + "code_token_ids_sha256": "542cfca1f68ce5c50b476187734ebb079b738bd17e25fcd67e6d867483ce4ac1", + "code_tokens": 72, + "prompt_token_ids_sha256": "9da7b83626d3bd03d1bdee055fc9bd85dc9ca10161aa73eaaac09ed2ac925c1d", + "prompt_tokens": 92, + "task_id": 320, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "684298cf3b7f594f38aa104bb86af3f81c44a66a507ad3252b96c732b64d7a1a", + "code_tokens": 62, + "prompt_token_ids_sha256": "307828c8068fe7a163939bae5d68a7e78bb2214bb10cb6ed82d1d14774a1c8bd", + "prompt_tokens": 133, + "task_id": 321, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "adec23ca1ff43fbf9d538cbec0ac6269ca09df88b9ca4371ea91ce08d0633f8a", + "code_tokens": 42, + "prompt_token_ids_sha256": "cf3a242e67aa9da2b470094c5f07d48193a7557ff4345df8ad1a0a4efe8b4e6f", + "prompt_tokens": 159, + "task_id": 322, + "total_tokens": 201 + }, + { + "code_token_ids_sha256": "95def8f8f6465df6d016cf09313c75900ba1a4f9b961c91f1e8c37212fc22405", + "code_tokens": 238, + "prompt_token_ids_sha256": "aba65add39e289dca1702fc09538b901b369690d642dda5f71f277f77567e1fe", + "prompt_tokens": 218, + "task_id": 323, + "total_tokens": 456 + }, + { + "code_token_ids_sha256": "e5eaf8ebe19523eb9c832ce9de098a6c41bfcc3b97e2073bd1be41e8ddbe3f15", + "code_tokens": 68, + "prompt_token_ids_sha256": "13f2f78aa587c939079908afe723ba0e4f6349cb63d46ef2e766b062a1c21a52", + "prompt_tokens": 135, + "task_id": 324, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "237c4e3d2c492303a223b365ad9984c8de705c68b07627bfd627540e8fc22c72", + "code_tokens": 83, + "prompt_token_ids_sha256": "b240fe55c97be864428bb39b276e534433c935efc8433b2ff5117c20f4ed53e3", + "prompt_tokens": 82, + "task_id": 325, + "total_tokens": 165 + }, + { + "code_token_ids_sha256": "1b3b66b3526a6ed9a5fdf1bd928f6fcab8122a463bd5d8842c4a8efede1afb2e", + "code_tokens": 64, + "prompt_token_ids_sha256": "9837018d7884b3624653c4be99fbfdaf42aa7f59db76be8c7fbef0a3a97ed71d", + "prompt_tokens": 136, + "task_id": 326, + "total_tokens": 200 + }, + { + "code_token_ids_sha256": "a6ee182fb5aec5b345a0f84203aa19242379da06374ffb380207846114cfd59e", + "code_tokens": 37, + "prompt_token_ids_sha256": "de96a97e59d9e3dc514281adf58e3e9896027a359121670cdb7596696c408760", + "prompt_tokens": 92, + "task_id": 327, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "d36287e90864aede6387e932fb00af6f38669203a7f6e52b7c2a81e0683c62a0", + "code_tokens": 27, + "prompt_token_ids_sha256": "0d4b41844c9b984e75895dc82cc92efc95fa9aa0362cd695705ef3a4233c831c", + "prompt_tokens": 248, + "task_id": 328, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "4ab5054fda608eb376d8819c9964efef36226f0899819098cbd9387ed23442db", + "code_tokens": 40, + "prompt_token_ids_sha256": "8227558c488e7aed1a7519dba055feaa226d07a59aac318c6f1ade0215229a6a", + "prompt_tokens": 90, + "task_id": 329, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "c41d02d348a7ad940e0ad03d79f15addff318c700acc0e2d7379de1c2377d239", + "code_tokens": 28, + "prompt_token_ids_sha256": "12005e71469be646cbe6dcfe348b673610912d18a81d274b2ee26c52e21edc3c", + "prompt_tokens": 126, + "task_id": 330, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "a4624bb218646fcfd3c303c260e1a719df503c265e6be59ea78b84ffa8d4c3e6", + "code_tokens": 59, + "prompt_token_ids_sha256": "34145f19a93eae3dc43521238cf0a6fcaf43cb62122c88c049e57f142b41bb51", + "prompt_tokens": 72, + "task_id": 331, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "ae0be1b3d5e9f49bc63c8c861d567ac4bdfe59d887ce253f82311cb5c71ed7d0", + "code_tokens": 57, + "prompt_token_ids_sha256": "d126aa076e0d9c4bfac8bfc79cce798361a907002995e5927f8334e1dbcda186", + "prompt_tokens": 167, + "task_id": 332, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "a9d36fb726b68cf942fd27a69edcd102c1df262876fc5522b64a6a6abee39d1e", + "code_tokens": 25, + "prompt_token_ids_sha256": "28738dac11e6712ee33f8d0f696e442d0da2c7f796606bfee4f57f5e841e8d10", + "prompt_tokens": 243, + "task_id": 333, + "total_tokens": 268 + }, + { + "code_token_ids_sha256": "1ec8296c47c8db2cfd1e43ad66ac5a05f70342fb26dd5921d31bf0ae4a2cdf1c", + "code_tokens": 48, + "prompt_token_ids_sha256": "8ef4b0816e51b242cc4bc18256b1f4a2712ef1c9299e5154cdf5b38928f2b108", + "prompt_tokens": 87, + "task_id": 334, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "a6fcf8367b3af5435373f2a8436dee690fd02d2ce5510769ddd01b5a04de953d", + "code_tokens": 35, + "prompt_token_ids_sha256": "e07d4705244bf0f22eb1fc7f52b714968e36de3e135e11e443e9c1c12175e787", + "prompt_tokens": 76, + "task_id": 335, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "3923556bc032c62f4aad3fff59d5c97c4b76de8c64f8f3d43c6dc64844255bce", + "code_tokens": 30, + "prompt_token_ids_sha256": "f5a0968516deb7420174666ede3d9de7656d3db61348dae38ab850ad9cbf636f", + "prompt_tokens": 71, + "task_id": 336, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "4db6017fb9028f1f2cb6580ecd9f34451e8ba63ca28e3ff59aec1b65b1547b72", + "code_tokens": 49, + "prompt_token_ids_sha256": "cb294e234e1cad4fff82c901e5cb73668045dccbf6485c9b05fbcfaebf6ffd8d", + "prompt_tokens": 90, + "task_id": 337, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "195731a0e50925a1a36e4a9b76797be22c7c65295b4cf49846cf817b239648ed", + "code_tokens": 98, + "prompt_token_ids_sha256": "62c0f739f67dfea77ae8673fda4493a25765d0944c8e369b338d0ca89a36cdd2", + "prompt_tokens": 96, + "task_id": 338, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "b99aa0daef6b2ba1da1a8d674baaf3326de8a2455d4c5b1bd732b375b5f79db3", + "code_tokens": 25, + "prompt_token_ids_sha256": "ea04fd88fce45e58b962a96abaabc74a679750e2c7e3fd63c2ad4ce5f292d333", + "prompt_tokens": 84, + "task_id": 339, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "fe51167b0ef159f2a786a606adabdd71931108b57c62ba177c89f49e413b91c4", + "code_tokens": 27, + "prompt_token_ids_sha256": "865df8c803ea8290d1178eb87dfe013283689d68f0bddb900ae48cfb3b36730b", + "prompt_tokens": 123, + "task_id": 340, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "b2e3951d0aab5cff48bc57a76e5a4a89bfe2d01435cdcc912fb01363b03d8a71", + "code_tokens": 20, + "prompt_token_ids_sha256": "ae2055c2aa4931b7f3461617699a7857c25cf751aa3988de37b00e50618e2eeb", + "prompt_tokens": 166, + "task_id": 341, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "772292d3062160610af811760cf6d077cf6adc0bbce644f454fd8ea9294cd423", + "code_tokens": 253, + "prompt_token_ids_sha256": "3f51bdcc447f45f9992ed54063f7d16571eca840e0f9e0e02b0cc3fc2096dd17", + "prompt_tokens": 229, + "task_id": 342, + "total_tokens": 482 + }, + { + "code_token_ids_sha256": "d403a7fce254bc4bf218da9a7dd6ab7de4ee664fd0717e51c9345a405b11d895", + "code_tokens": 55, + "prompt_token_ids_sha256": "59770ab0702278ee31c5404c9e827613c7917989dd9fcbbf255a8478f71fd08f", + "prompt_tokens": 83, + "task_id": 343, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "5e22cd3bf953dac4281c49d46b144de384dd88f3ff252160d72dbaa77fb56212", + "code_tokens": 31, + "prompt_token_ids_sha256": "6c7e985112b7077508c41e7d7990c6fe9b7aba12e427524f60b2b51e3d47d625", + "prompt_tokens": 91, + "task_id": 344, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "85458fabfa2d02df2b17e2d8f77c76fe25526c53405c4bd264da299911115bd3", + "code_tokens": 35, + "prompt_token_ids_sha256": "a17a839ef439ff4d6032c1a52811e3e378f68a224d76abc4eb26304f4048828c", + "prompt_tokens": 202, + "task_id": 345, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "8027fea79b9c972a4c64c4200de07e72ad24201ba579893b20a6d196c3b0fe54", + "code_tokens": 61, + "prompt_token_ids_sha256": "e5f48f605cde6d1973e16ab0510b7ce618a17f3da3ebc34e43db243d5b5507a7", + "prompt_tokens": 78, + "task_id": 346, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "4887ebb7c60d260cbb7169dc6623273eaa26675f15a1f0b97447c79263dcb8a5", + "code_tokens": 55, + "prompt_token_ids_sha256": "af79f03ea1b0517a37c0ed24f0e2c268eae1d459a3799aa0116da32d48ef7c1f", + "prompt_tokens": 80, + "task_id": 347, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "eaa4d51ac2b55e2fc4f725c701ad72ca98be7b0965b40ed8dec975251c21f856", + "code_tokens": 108, + "prompt_token_ids_sha256": "079c7fc451d5a33262031d5406184c50ecda6e589df58e913880e479a628867d", + "prompt_tokens": 82, + "task_id": 348, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "8a67942a8e59c2013873c38d12f0ce7490ac3809879c4da6179a57b862137e33", + "code_tokens": 58, + "prompt_token_ids_sha256": "80476620d9e6967c7bf4c32d054e3d2b7a9844309d7e6fca43e716725fe9263f", + "prompt_tokens": 86, + "task_id": 349, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "bcbf036c2860a234687649593b58524908e26787d0b54a1235b9aa7a53af8755", + "code_tokens": 91, + "prompt_token_ids_sha256": "2d6b9a4064109b1d82ee1f975abaada11fa260295f1c5350de6f51b77e8eacef", + "prompt_tokens": 78, + "task_id": 350, + "total_tokens": 169 + }, + { + "code_token_ids_sha256": "37b76b73d4f8cf80f82b1ab401ef3331cda9c0a96d9fd4127cc5256158317d53", + "code_tokens": 105, + "prompt_token_ids_sha256": "2fd11edb9c88ddbbda344e067f8d3a5ce0c4f1eb2fab06f4505924c755e8ceb0", + "prompt_tokens": 122, + "task_id": 351, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "409eb7b208a25c1c956db1560d293308fcdd0f7f3bac5ee1f58dda50b2496f16", + "code_tokens": 49, + "prompt_token_ids_sha256": "1beeda2eda56ca2b0f11c90b3f3521bdfd1af81896ea7c322c555ddffb6a2d58", + "prompt_tokens": 74, + "task_id": 352, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "849ec81b32b900723b7d22938d62553d8ac843192ef81095142128275b7c7ccf", + "code_tokens": 27, + "prompt_token_ids_sha256": "63f726f1aa84d708fba7d4f593d77319d6ccbc60a3597c0769cbd1db7c9a5dc5", + "prompt_tokens": 215, + "task_id": 353, + "total_tokens": 242 + }, + { + "code_token_ids_sha256": "b39b4a088bec15454ddcde6a881c098445c1c600882883d5ed22afa0c30e6800", + "code_tokens": 25, + "prompt_token_ids_sha256": "b049919a7467ea0f9a0796293f96f2251d443b09fdc515d2b68b7620a118dba5", + "prompt_tokens": 80, + "task_id": 354, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "d12ddda1c75aa6530c093585eaba56437093955ddf54e0b72999d52e4ffa9b07", + "code_tokens": 98, + "prompt_token_ids_sha256": "90c1994a8fc58e591520f7e6462964fd4da554921ecffa151d9a822e8b254b7e", + "prompt_tokens": 76, + "task_id": 355, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "4192f3aa0c0f653e4b6fa7e8dff04d4dfb404f4d1f5eab62d1d5953bf5fc8436", + "code_tokens": 23, + "prompt_token_ids_sha256": "1f3293f45e76caff204c93c27fea127b70f0c403b3fc13f3ce59d17eea916a49", + "prompt_tokens": 80, + "task_id": 356, + "total_tokens": 103 + }, + { + "code_token_ids_sha256": "e342a039a59ae2a284ffec73aec713be55ec70c34c48482f31dc96dee96b0656", + "code_tokens": 31, + "prompt_token_ids_sha256": "48381f7680b85f05fe1b227042859abed4c36a54ca39d540baf42b7da50ab466", + "prompt_tokens": 159, + "task_id": 357, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "5f6a2becf03b29ee57a198a96695999222a125bd585670597d1fb0a1c8583100", + "code_tokens": 36, + "prompt_token_ids_sha256": "b52cb7b071fc2a3a10fb765bd98b5d0fc8b767516217e5b8cb7085e38c9ea538", + "prompt_tokens": 118, + "task_id": 358, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "7e1d068c421b0938ff74edc530fe03ae74a5726c51a559d562d6380487fa48e0", + "code_tokens": 38, + "prompt_token_ids_sha256": "e19f4f725e6d555f05f3baceff228b52102d2deb16ef88da734f8bb7b73e8f21", + "prompt_tokens": 95, + "task_id": 359, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "b590b4657d8e19f45898d0df03446f7a535458c4c3f7bc8bced6f99a63e1238e", + "code_tokens": 25, + "prompt_token_ids_sha256": "c939a734a51c162844442ecc0aa66f2d2bf9970a3522f8a3a741ac3d3c1dfcf3", + "prompt_tokens": 76, + "task_id": 360, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "c0f711fc0300901f3905b670a0926a951fb556f8aeb3d1164f8f3aa07cabb44f", + "code_tokens": 26, + "prompt_token_ids_sha256": "99cb9e7018eb72f99a83830ca8ccf076c051407e7ba5d117fef800253b0b4d2b", + "prompt_tokens": 151, + "task_id": 361, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "d60a68d3e668c5415d196f68641731edb152e635946797c7c44d6e4e4d3e8a78", + "code_tokens": 62, + "prompt_token_ids_sha256": "be51826c30c10296f46c23a70dffa137ec467bce1453f78450e9526bddd63cd0", + "prompt_tokens": 123, + "task_id": 362, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "b4f8a0abf190c1a2f8d35cd17bb49f1f870c6f444836e41cd20219920992e890", + "code_tokens": 36, + "prompt_token_ids_sha256": "cf90d2c76d0595dc3512f208d193803c6f301b975bd8827a9770bd2573e41088", + "prompt_tokens": 258, + "task_id": 363, + "total_tokens": 294 + }, + { + "code_token_ids_sha256": "58453ae606bc9ce5c337c0f07e782e2673bfc65e700e6e5819863be933684dca", + "code_tokens": 119, + "prompt_token_ids_sha256": "b7e18e4afe94e2e4863b7edda20a8864b9591dd66e85c28c633987e180891bef", + "prompt_tokens": 112, + "task_id": 364, + "total_tokens": 231 + }, + { + "code_token_ids_sha256": "67cb1fd5920f188c5fd01c06d589a49f68320fce89292112ab5f4261e90ed4e4", + "code_tokens": 37, + "prompt_token_ids_sha256": "4fd6ac3c056477c6ebe1bee821cefc104fd533fc9dc95c45a089c64dc7d09195", + "prompt_tokens": 91, + "task_id": 365, + "total_tokens": 128 + }, + { + "code_token_ids_sha256": "d70289a1a5ae1baf8c20695645d756c4e868adbf5bf518e2135b2bb9c79ae1c5", + "code_tokens": 28, + "prompt_token_ids_sha256": "d9b3ab6c8a69f328e3294941481ef3ba4756113d84aa5f17d98be7c376918673", + "prompt_tokens": 103, + "task_id": 366, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "6e1aa8245e6b8e95e80b3ac9a18aa6594bafc4c40ac35774be512f67341d7fab", + "code_tokens": 143, + "prompt_token_ids_sha256": "ebb2431803c47b0cffe98853bc4dd1f8c312ecb7d0f71261f2249c00e23978c2", + "prompt_tokens": 72, + "task_id": 367, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "d8d158424b151053bade117e973184b45387c6570ce2af072154b841d06812f1", + "code_tokens": 29, + "prompt_token_ids_sha256": "0e2b9053ce043909a02fbe28402c8c00e48feab4a61bb5df7ceb14359307a707", + "prompt_tokens": 151, + "task_id": 368, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "86183483115ad988cdf65549bd8728b25b143dd9723a8a4dcbfa66a4356db4d9", + "code_tokens": 28, + "prompt_token_ids_sha256": "3b87672bf846b184319a4a1a28ae46c10a83bf406a23f3c6364493fd3094105b", + "prompt_tokens": 97, + "task_id": 369, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "b3aff4b5af85e39dabcd3935e56590e32081c5209de5fb521c8fc7702f7ba37d", + "code_tokens": 30, + "prompt_token_ids_sha256": "f51d8a9ef6418e4f4880405555b1b9ab07339c9ad40d5637959b239406f70117", + "prompt_tokens": 214, + "task_id": 370, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "e8a9b44ce3169a31cee0b4b08fd9bd0dc8ed44b82ff1c9ffc4296b7dbcbcd158", + "code_tokens": 85, + "prompt_token_ids_sha256": "69ace21e983ccf1a0e111de8b9bcea8e236bc3dc3710a8d7e125aeffea1a464e", + "prompt_tokens": 149, + "task_id": 371, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "f0f8aa3a89f09fde6c7185ff59ad9d14ddf49a085a005ce33d4291b1a57b5eb0", + "code_tokens": 44, + "prompt_token_ids_sha256": "9a51429a91d3ffe7eb130d07416b7f1e22cc54c51ce0ce02d05fe9a3178697a8", + "prompt_tokens": 263, + "task_id": 372, + "total_tokens": 307 + }, + { + "code_token_ids_sha256": "d8ba53b35d25e3abf10d9452572786da58fa84a8bab2ea045745b58132a51274", + "code_tokens": 19, + "prompt_token_ids_sha256": "fe5559f7baa780dc21a2e19aa414d92698620a9d0a8d693bc660d57706308f5d", + "prompt_tokens": 88, + "task_id": 373, + "total_tokens": 107 + }, + { + "code_token_ids_sha256": "a3c99473b9f60d43d446a4febf4eb4d82b4bbdabf3a1defa7f95ee10993e5848", + "code_tokens": 113, + "prompt_token_ids_sha256": "97399775583cefbc3b171a45e0b4d44f3d9e041938b41076e1b39900a3c445cc", + "prompt_tokens": 184, + "task_id": 374, + "total_tokens": 297 + }, + { + "code_token_ids_sha256": "c5979aa0e544c11587849e6860a5071220cd317ee68eadad182ad7d11062a203", + "code_tokens": 40, + "prompt_token_ids_sha256": "2311cd9efff67690345257d1ac0f357e38358b1329cf31979732569431e78a65", + "prompt_tokens": 90, + "task_id": 375, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "6c171ab7064edc0e495c79632fa20b37246a95fa28939d2c7c5ee0162943749b", + "code_tokens": 51, + "prompt_token_ids_sha256": "25a75f74cd5b1f58c73116cdcfbadb32d111b76ad34c41c7874c2257b9c599cd", + "prompt_tokens": 264, + "task_id": 376, + "total_tokens": 315 + }, + { + "code_token_ids_sha256": "0c8bb9a549ab959e5b223367a766849f8cec584189d97e3de8a2d71f8eb9b458", + "code_tokens": 55, + "prompt_token_ids_sha256": "f0cb1985683eaef935e8df72e13177ac42dc81b9306a3b93008d3a5ab0980f5f", + "prompt_tokens": 83, + "task_id": 377, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "1546f3c22ff924af68166aa14ea6c0e120c5ba6b998c90ecb2d965660c3e4ab3", + "code_tokens": 27, + "prompt_token_ids_sha256": "3d9d7abbb5cc7bc0b9fa824ef1017f79328fedf442a2fa45cdc6586efa6266a4", + "prompt_tokens": 111, + "task_id": 378, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "6a449490bed762cbf3e575678b7f132526921932c3835c191992552a2290d567", + "code_tokens": 32, + "prompt_token_ids_sha256": "a67ad7fa988f34eb1ad5d84ca74c89caf8471d4dbf7b86f30488e85f25870117", + "prompt_tokens": 93, + "task_id": 379, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "ee33965f22260b2d78b0b53aa3b2e86af08436af821fab4168d322ddc8671669", + "code_tokens": 64, + "prompt_token_ids_sha256": "ad125dbaf236a6f5f4504883b91febb934b75bba086a3f1e9229f60ee899af98", + "prompt_tokens": 773, + "task_id": 380, + "total_tokens": 837 + }, + { + "code_token_ids_sha256": "a997c10a20feb163bc8417fd5047cd2b9123e2de16fbc92dbb368c970e15e742", + "code_tokens": 35, + "prompt_token_ids_sha256": "79e69aaf19f270e2cda24d9ec88f62cac70468283e533706a8c4452e325c6f00", + "prompt_tokens": 392, + "task_id": 381, + "total_tokens": 427 + }, + { + "code_token_ids_sha256": "91636985bd68c0f7ff3fcae63344db5df6a0c4a8218ecfb02365c570258107f5", + "code_tokens": 152, + "prompt_token_ids_sha256": "29414f1d81ebce83e477b2f6d826d6734f46d5e15538d89b6a76a9b5f6071c94", + "prompt_tokens": 134, + "task_id": 382, + "total_tokens": 286 + }, + { + "code_token_ids_sha256": "2721bee9bec79f6ce5df6a0c28a15959cb6023f97037ab4e99acf58151ec62ba", + "code_tokens": 77, + "prompt_token_ids_sha256": "3419608e93fa37260e63461b4de5f69d01ca9ec2b0f9bd78eff0628b21bb9912", + "prompt_tokens": 81, + "task_id": 383, + "total_tokens": 158 + }, + { + "code_token_ids_sha256": "13a3efacb82cd889da45b7478eae5f2ca4d174feb32ba12dbbbf40fa13db6453", + "code_tokens": 77, + "prompt_token_ids_sha256": "636f473deb0d61a7ea6e702cb2cf1ca7a36a7481b512bf6e73998e1f5a6dddb8", + "prompt_tokens": 120, + "task_id": 384, + "total_tokens": 197 + }, + { + "code_token_ids_sha256": "d357f87b48fd3e4a9bd796135bbca7bf794beed3cb97ffd1ac05a4da2a9649ed", + "code_tokens": 71, + "prompt_token_ids_sha256": "a21a1ff6c6bc2c5b1a6053750d02510dcc33eee85460830613816a316e853a70", + "prompt_tokens": 79, + "task_id": 385, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "a277cbacbd56588ff0a772a5d61247049d06c03e2314763ddaba31e927147c3e", + "code_tokens": 112, + "prompt_token_ids_sha256": "24421ccff6c27a3db93b968d11d199ac3efdcb1f06a7dd312782a6085643d51d", + "prompt_tokens": 79, + "task_id": 386, + "total_tokens": 191 + }, + { + "code_token_ids_sha256": "f20c1e8b64f4a96033c322562c89c98a04e62ee78bb26ff83cccd59010ae21e1", + "code_tokens": 111, + "prompt_token_ids_sha256": "a5d33b4b9aa47ffe16b205501824b41cdc34e74fd77c7e7ae1fab5eea3657a97", + "prompt_tokens": 81, + "task_id": 387, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "96f1a64a367da5d6d51d3b4c7ed18b3e759ba31fcf1f2aba9c5c5ab802e7cddf", + "code_tokens": 62, + "prompt_token_ids_sha256": "2b9a667eebccabfb88964f1dbcbd76e438d4380afa0b48254f6eca5ff276fb98", + "prompt_tokens": 94, + "task_id": 388, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "823cf284762f5d8b9a1e0976ded228cfe3b1eafe1a61981b11bbe7f4e6d93e9e", + "code_tokens": 52, + "prompt_token_ids_sha256": "7821d7626442ca7f51556deed0172ffa3479d8c659ec47a3fb245e7664de9a63", + "prompt_tokens": 73, + "task_id": 389, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "5995708c32f5f2bda176b47f64964db2e528e3317678b31f66c7d5811884ca80", + "code_tokens": 24, + "prompt_token_ids_sha256": "63452ed2d85f3ce950d4dd1d578f1a9595c4cceda7269bdb927c8f25460a33ec", + "prompt_tokens": 149, + "task_id": 390, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "5137448c57393dac981dccc8482c0dcd97475447febe63efc964e0cdcea681e5", + "code_tokens": 48, + "prompt_token_ids_sha256": "83226adfb67c6af10f9bfcf212d2549146bbf243660fc5b46e80aa4db071de15", + "prompt_tokens": 327, + "task_id": 391, + "total_tokens": 375 + }, + { + "code_token_ids_sha256": "00c6aa7c052f039b3d792aa3d67bbd54157bf58a3056dfd6bc535fda7a28a411", + "code_tokens": 98, + "prompt_token_ids_sha256": "609f3591653f389b0e4a3159bf33026be69a8118ca922a309ce85ea73d23763e", + "prompt_tokens": 110, + "task_id": 392, + "total_tokens": 208 + }, + { + "code_token_ids_sha256": "3a086203b5f49dde2e307a0850668251b632f5ee43ba3ca43cfd089309b2c214", + "code_tokens": 48, + "prompt_token_ids_sha256": "fee9d597cbb952369acf4e3ff703c3b4059f8cb40adbdacb2a9effcdd2524f00", + "prompt_tokens": 183, + "task_id": 393, + "total_tokens": 231 + }, + { + "code_token_ids_sha256": "ae5341be933240227103fccf61541b810f70494a9b72e7e2853fe1d8e26676da", + "code_tokens": 56, + "prompt_token_ids_sha256": "cbc06f073b4acd29088affc0ea445d1d692a87e16079c5d0ee9f6aa28dbf1d21", + "prompt_tokens": 105, + "task_id": 394, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "820bebaaf445f7fc51ea9c48160c0ae8122d1b85fa71a53b0e6e0fef7496827f", + "code_tokens": 88, + "prompt_token_ids_sha256": "a214eb582b6f2b3f02bedd6067719bda6dc0cebb3bc0c640105e92d678576493", + "prompt_tokens": 85, + "task_id": 395, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "dcd9b9d6a4cf24751a06d5c43190643ee4164ea66ad3648005a2198ffbd2192e", + "code_tokens": 50, + "prompt_token_ids_sha256": "b4baf34f404c7f98a67c84f0f92fb609d9c29807070c28e8833b6b4e4cf5eac8", + "prompt_tokens": 81, + "task_id": 396, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "38e92ee145d22efbe257e76681d376b619075b507566676215b9d443e48ae292", + "code_tokens": 85, + "prompt_token_ids_sha256": "23e5b7f2b1aa86fdbb8b18a57217f5a9beea2e5a94feaf93e64cd7897e72cf7d", + "prompt_tokens": 92, + "task_id": 397, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "a3ec172a9ef7af4465d2e8c942525dc0fa7b7c2e62ede8dfc63134f023e39881", + "code_tokens": 27, + "prompt_token_ids_sha256": "8d907a47a974f5038e17b82e6b85f3db8349877824415921b177ad43b48bb1d1", + "prompt_tokens": 106, + "task_id": 398, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "a463443fcf09dc2143a8162fd74437a5c92143918d1e3496add86386350642b2", + "code_tokens": 48, + "prompt_token_ids_sha256": "d030d5a8e225c7fff03dc744c28e2aa6ed93cf518e852f3724b7c5ac5338bc9b", + "prompt_tokens": 176, + "task_id": 399, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "635b31da3e01768501a431afc001f54bdd4488868846f981e10b839adc66d720", + "code_tokens": 29, + "prompt_token_ids_sha256": "08f44c2770d7f54758b56b1eab97e114ed62638057a2b77e4fb641860b323029", + "prompt_tokens": 141, + "task_id": 400, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "3f6c6775c481d8ceb03c00fa1317c8d8b020ef6708650b6e3300f3cdadc3ae21", + "code_tokens": 63, + "prompt_token_ids_sha256": "893bfcba55be3b84eade07e0ad6ac451b85bdad49dca5f4c332fea373f78f9fb", + "prompt_tokens": 301, + "task_id": 401, + "total_tokens": 364 + }, + { + "code_token_ids_sha256": "0c92d11ef9139385899aec5ade13ea97574e3fe239fb4db3372f4697e7258fb9", + "code_tokens": 89, + "prompt_token_ids_sha256": "bc83780af6a581cec8b1124dceb306ca7ffd2bfa3d0fdd0cbf9edb2f8dc2e4cd", + "prompt_tokens": 89, + "task_id": 402, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "0e0aa0b1f63d65caedd6c5e45e1025d488f8968d7b7264ab57a29e046b1567dc", + "code_tokens": 133, + "prompt_token_ids_sha256": "789ca9a1d8d3ade903c927a2bcc5c8a8aadedadf85e33087c21bb62dc2e0df5c", + "prompt_tokens": 84, + "task_id": 403, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "72951670868af7649ae3f2a74bd6bf5c594da848c7956478efdd5f41ded22b5a", + "code_tokens": 25, + "prompt_token_ids_sha256": "fc731cc60cfd558e3308578e15c928671266927085d57759e57b5c5a4385442f", + "prompt_tokens": 71, + "task_id": 404, + "total_tokens": 96 + }, + { + "code_token_ids_sha256": "05e15a4af4e552a1cc8507eec62c9143b727ab3c53242f4f4a1c45dc80d8a028", + "code_tokens": 33, + "prompt_token_ids_sha256": "5f1d56b9d7f447fd6742bb25755a943436e8a101358e9f57eafd28b4a3890ac5", + "prompt_tokens": 157, + "task_id": 405, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "ecf25ed16a67c4005a6f2014d3eb6d8f5f0dafda6e64ace740932aff4e781f58", + "code_tokens": 93, + "prompt_token_ids_sha256": "56b38d99040b8a0f30157cda0d354fb083e48dfc3225d9368c04a2f6561c87c5", + "prompt_tokens": 83, + "task_id": 406, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "85bc3e34a51b0aa26c0fc5e0577f21da5472bf3f7ab3f023020fefad5d3996bb", + "code_tokens": 101, + "prompt_token_ids_sha256": "f75cfc8c46f45026f5ca6dee05ef113f71c45bb7f841cbd84a1177fd8ac1171b", + "prompt_tokens": 82, + "task_id": 407, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "9af9ab89cdab6fa859c6d6deb39febdce6aa843334df3d56e9f92e66e3365798", + "code_tokens": 152, + "prompt_token_ids_sha256": "fc780bd6d7eaa5f0ab628af70c710d9a0f5b30feadd23f6edadcd506aa4e16d9", + "prompt_tokens": 175, + "task_id": 408, + "total_tokens": 327 + }, + { + "code_token_ids_sha256": "27223c423c82104b640a3f08713271d463e1d8e2826e0bf9d641b500d510fad7", + "code_tokens": 33, + "prompt_token_ids_sha256": "f5272d1ec45c21307eb4d608f68fc65495d6c4544db0a4ae88e5bbaf35e1098d", + "prompt_tokens": 141, + "task_id": 409, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "37cac7ee1fd6a681350286e2950045a7ec1ca99ec97f2254bb3bece496f4b27f", + "code_tokens": 29, + "prompt_token_ids_sha256": "a465352d496ae0904c46a37e71f7c9bec6f28ece2c9d6d01a5ce75f2b8f5486b", + "prompt_tokens": 114, + "task_id": 410, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "80a58b664a6737b7817f9743c415bd91bb8a62e8df036b2f672dfd977acec891", + "code_tokens": 27, + "prompt_token_ids_sha256": "455c91598f7d878c2b5291f659161a23cca2d86340998cf8f5fc4648f158d8ff", + "prompt_tokens": 89, + "task_id": 411, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "0eb4c63fbb3fd81390ae58369522b14d83b2202acb527e6219fd6ddc9fbcc0f0", + "code_tokens": 33, + "prompt_token_ids_sha256": "4e6efa1802fc6ab70d8a1f383317f80cccf5a0c516a07ff7fa7580152d2fcdbc", + "prompt_tokens": 94, + "task_id": 412, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "c3148bf9916eb8149b150d55413031a4685a8c637a8635e9b188e25c9de2fedd", + "code_tokens": 27, + "prompt_token_ids_sha256": "e1171b65f3bb9b15d7ba7665a8118ec1e07e92b80e9126ece5e716483b26c674", + "prompt_tokens": 276, + "task_id": 413, + "total_tokens": 303 + }, + { + "code_token_ids_sha256": "103aec98276f6c978ed29d7ac5a7dc1f733cecd11ca0a03658ede4bb1cbdd1fa", + "code_tokens": 84, + "prompt_token_ids_sha256": "e116ab609c5ddfbe8383e34983b06601394ee35c1fb0498656f8eecf8c80c057", + "prompt_tokens": 102, + "task_id": 414, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "c090474d9d006654617b82a4f3f2d0c409af47cce8c9996dbe06a7f54eac7891", + "code_tokens": 103, + "prompt_token_ids_sha256": "95a5857065e5c72114488d0a80e6867485cc081e3600ef4a5eb7d9ed67d81167", + "prompt_tokens": 113, + "task_id": 415, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "c8a5531a6cda286e76eebf4cee3cb9104a5802f6a53d5037bf85927b137a8544", + "code_tokens": 88, + "prompt_token_ids_sha256": "16c19665451af699add770a22010b2259e7f179f504df6fc37a1e891550257e5", + "prompt_tokens": 91, + "task_id": 416, + "total_tokens": 179 + }, + { + "code_token_ids_sha256": "0f7946f1f38a764c289e93792a71b7b70358a420c051a8c83cf23160546105fc", + "code_tokens": 62, + "prompt_token_ids_sha256": "dee40171cf9139bd1710229202ae8e0e6d9cb1a812778f83fa67582f6a19d092", + "prompt_tokens": 157, + "task_id": 417, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "0c1ba8822c69c64a4e08ca0cf8ff94c782e459381487468147ac0665b041445c", + "code_tokens": 24, + "prompt_token_ids_sha256": "0ae465189c114c74caf81117ae489c217340154704faeffeea9895f29d0fc6be", + "prompt_tokens": 123, + "task_id": 418, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "416e3921b01eba108651c264684419d4ee2e7fea140f7783497f26ebac861d08", + "code_tokens": 34, + "prompt_token_ids_sha256": "b57956297e903971c3efc944e7ac1b3c0eda9a09eca658c16c0f01263680b0b9", + "prompt_tokens": 165, + "task_id": 419, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "ed563720c8e7e36ba0f0b8c6f44571ef7b54bfc8c7bd855f6274d1a6f246ed50", + "code_tokens": 43, + "prompt_token_ids_sha256": "d06729ed3777e8cc8854102b75d22322d0067d741d582eb38de837011f169d9d", + "prompt_tokens": 80, + "task_id": 420, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "55577b06e8abe081cdb62e3465f0033df6b85181af64a73c8388477762e64ff3", + "code_tokens": 54, + "prompt_token_ids_sha256": "7c95a031313bcce14b2d5ac3c00c39b06b3e6ab457a44122230b59f238ba7012", + "prompt_tokens": 123, + "task_id": 421, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "f881546e4f9b82b445897f95fc6e368850dc78ade8ba249919b0563cb3b5e934", + "code_tokens": 50, + "prompt_token_ids_sha256": "6a7b892a2ee90cf8db3e76fee22f6b7255962bed50ea6f9f7b43388e5a968bcf", + "prompt_tokens": 87, + "task_id": 422, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "cc47ced335e30e39614015b42bde0138ad0567944b602c91bdd4bdaefc5fc819", + "code_tokens": 235, + "prompt_token_ids_sha256": "46462c425fc3fc0a3aa41c799209b15a953060dabaf55d652ba8650dea81f9e2", + "prompt_tokens": 135, + "task_id": 423, + "total_tokens": 370 + }, + { + "code_token_ids_sha256": "a5757499fc458ebdda35dc3a8178a161da3064fd99046ef7184344df7df021f1", + "code_tokens": 33, + "prompt_token_ids_sha256": "349fa06dd4da7471429c894160eeb63bfbe459fceb43e09160208fb3bda7ee4d", + "prompt_tokens": 122, + "task_id": 424, + "total_tokens": 155 + }, + { + "code_token_ids_sha256": "6fbadc54f386994f6f6bcb8c03eca1475102eb54bb2c6aa7d4851340df12dfc9", + "code_tokens": 45, + "prompt_token_ids_sha256": "6d4e5a36fca0233a5bf86f7b7cb365768a217bb896ae3617fbed4e699274c57d", + "prompt_tokens": 161, + "task_id": 425, + "total_tokens": 206 + }, + { + "code_token_ids_sha256": "762cc4ad066786ee50182f00db9718b023bc27f8340d326cc067deb452704faf", + "code_tokens": 28, + "prompt_token_ids_sha256": "53595738e4caa23fc00a9569e246549e8aa81cd1766491a309722cf8de68d0ca", + "prompt_tokens": 147, + "task_id": 426, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "2734a53246d3612708451eea0a031cd7d0d58a4f3aff48debb95d79e36de44ad", + "code_tokens": 49, + "prompt_token_ids_sha256": "174d245920464c5f9a88078932de3f46fbf6efa2fbf731e43f458fae47016a38", + "prompt_tokens": 137, + "task_id": 427, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "823cf9efc60c2504e0cf59f91a641f0369451e244bbe9a99cc4ac9ff6c6e24f9", + "code_tokens": 107, + "prompt_token_ids_sha256": "7900258b1a84b3bf0352e4dfea4279a688d642efe542901bb6391d90679ef825", + "prompt_tokens": 238, + "task_id": 428, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "872449a2dddd16b72f8fe54e471f2fcf093ebba60f2bcbc366186d16d4b8bec5", + "code_tokens": 48, + "prompt_token_ids_sha256": "3b3aeff1aa8fe22143fee0c988193a998385dda70d9f0ac1749b9fac789c4201", + "prompt_tokens": 172, + "task_id": 429, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "2374f55f2eb70a148b00bfc01d0812897d6ade093bfc4e588ab1e9a880fe01ce", + "code_tokens": 40, + "prompt_token_ids_sha256": "84bc19434955ffa4b052e8fc3d3853fdd940a2c9e36d15944cb20168c30bf4da", + "prompt_tokens": 91, + "task_id": 430, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "2d376f8643273c69a0a18dfa7c2fba4ceaac947436aa6db6a8dccea35abfb42a", + "code_tokens": 46, + "prompt_token_ids_sha256": "97379b69a13f7cee3cc09aae32f0362488e73c6f02905da70c9c54d1e929e21b", + "prompt_tokens": 117, + "task_id": 431, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "cbb91b3f27690b9f25fed6abfff80de8b90b6df81bad299845d10a560292f97e", + "code_tokens": 31, + "prompt_token_ids_sha256": "b0990fa3dee7262c2f6652f98676bfcffb9befd57c605e1b723f3519f2cf7003", + "prompt_tokens": 95, + "task_id": 432, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "42cb4bbaefa5777bcf36279cb38c9084b83119ca54757dc61074e817388ab345", + "code_tokens": 58, + "prompt_token_ids_sha256": "5630d7a71a1b2cf37832356fbce22ba73a221c1cf6c0fc07e744553d27570cd1", + "prompt_tokens": 159, + "task_id": 433, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "203c06f15dec2a207f1d4490d8863473b6a32fb785ffed8e1fb779009cc0ca51", + "code_tokens": 49, + "prompt_token_ids_sha256": "dd64caf7867c97ab6618234ab47be74de336ff7aae7f0cc25dd772b543fb6f73", + "prompt_tokens": 86, + "task_id": 434, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "73086fe83e31e9e9be41c31fcae3dcf4d9886bbaac5b054a991eff49f9ac80da", + "code_tokens": 18, + "prompt_token_ids_sha256": "604487311020ee2c6ebcd5fbcc4d454fdf17e3bb6f51000285e92aa799a714c8", + "prompt_tokens": 77, + "task_id": 435, + "total_tokens": 95 + }, + { + "code_token_ids_sha256": "992678a188681c0ad9ab8003f6c2a9a19a24ccc59d0a618697c5e57f15f28539", + "code_tokens": 28, + "prompt_token_ids_sha256": "35976196b41caa49eb3c1fe6521edfde2abe52a9c30ebdf85c20680cc275e75a", + "prompt_tokens": 95, + "task_id": 436, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "8a77573db293c0aad800c2558c798ca7befa168b39ba57890f1a5d0218344d70", + "code_tokens": 57, + "prompt_token_ids_sha256": "77fc6b28aebef71eaadb14c44f158dea12d03218c4b02d499f34a3ebd01fc5c0", + "prompt_tokens": 70, + "task_id": 437, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "2c59ea890e8aa414ccce863e7db3686c8c8fb343e3e5b1e8f1f5588ed69a5a42", + "code_tokens": 88, + "prompt_token_ids_sha256": "529fa7b7047729a6b4764494666af1c9d5690e0a44469efca9d3ef2ebbdded4e", + "prompt_tokens": 174, + "task_id": 438, + "total_tokens": 262 + }, + { + "code_token_ids_sha256": "c9f9fb41598c3ec7d6c7243369c85fb4f1f20411689dd83f1faa5a6feb830efc", + "code_tokens": 23, + "prompt_token_ids_sha256": "f11cde80a6d617aa4e4e5f6e8bfaaac83078ab078ee8075ed2a1566db498e62f", + "prompt_tokens": 115, + "task_id": 439, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "475d9244d478d6237f4c50c99c82d1aa174dd1a7722dcdeee60f20ee76ce76ba", + "code_tokens": 40, + "prompt_token_ids_sha256": "0386bb74c084796f760c80ae97e4aef8cc75c5a6eec6edfd99e29bd385678698", + "prompt_tokens": 121, + "task_id": 440, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "6c230a53f8428e29fa7a30eeaaf13b4db4dfa9542084230724bb284f46fe6e7c", + "code_tokens": 20, + "prompt_token_ids_sha256": "4bf8dce0b63b2ee0b897e2f2d0ed99c3749f28901430fb36da10374be2272469", + "prompt_tokens": 71, + "task_id": 441, + "total_tokens": 91 + }, + { + "code_token_ids_sha256": "0163f0925202b022d1ccedef4e2ad86400018880f999b03ca69f9b50c6f07f99", + "code_tokens": 63, + "prompt_token_ids_sha256": "b9635048aefd24aa680c37540e7bedd8b93ebaa0198735ed5820b2a19438a583", + "prompt_tokens": 175, + "task_id": 442, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "f4e79808580ac06e875cc7fce515cc676df790d0bec666fac4477d6d062dbeb8", + "code_tokens": 39, + "prompt_token_ids_sha256": "bd98aa978a2cae16d50459ef7c9f4d473ef469ac51613323ed6e7ff8b90ef119", + "prompt_tokens": 95, + "task_id": 443, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "8b1a557de1267d8bc76f94ed68d2d2e3bdeeacc31df451c037f2f48217077fa4", + "code_tokens": 50, + "prompt_token_ids_sha256": "25633bffefede34fa34c4a08c21dea8a3265a2d7c715bffbb02c0f586eb4e9a4", + "prompt_tokens": 313, + "task_id": 444, + "total_tokens": 363 + }, + { + "code_token_ids_sha256": "788797b88a26e6f6b554cf7954bcafbb5744f6e776b3950414c7d3a92a8af897", + "code_tokens": 63, + "prompt_token_ids_sha256": "637458d930e1dd6b32ee7492cc0d4cf6d3c908ce478b127d8b61004566d5015c", + "prompt_tokens": 306, + "task_id": 445, + "total_tokens": 369 + }, + { + "code_token_ids_sha256": "0957a08c823bad2b6e811a7a9eb67fbe51c4e9b826e7dac39828afeefbc90b89", + "code_tokens": 46, + "prompt_token_ids_sha256": "0aaa62d8023f33a89ec73023cd6ce8b035444dcbd1ae51f9edad7014b0128457", + "prompt_tokens": 147, + "task_id": 446, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "857e14e30669cbb5b6f43075c51fdea3884fdb15f329ca52306612558e461643", + "code_tokens": 25, + "prompt_token_ids_sha256": "fd18aaa72f1042a17f7c89ca4da38a2210c3db2d6a519175e4589317f471d472", + "prompt_tokens": 178, + "task_id": 447, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "1ff6fb4edb0097a02e9783deb02bfbef21abb4ab2977ac6e5d6aa7851d647062", + "code_tokens": 111, + "prompt_token_ids_sha256": "e53a7b44807a1933ae23881384ed18992f3ddc40c87369d811fe047f8cf6facd", + "prompt_tokens": 74, + "task_id": 448, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "89be437c743d039505a98ee91fb20263da5721338887dea61afdf18cb97e6bfe", + "code_tokens": 71, + "prompt_token_ids_sha256": "4503b1f6f2b6b10aaadf3c8d34d645e3b9cbd64d31b030972266c42b3686e344", + "prompt_tokens": 112, + "task_id": 449, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "414fa5518358eaa1791aae45d1a2fd000d31d4b78b6edfca6bb1fbe0d84ccd04", + "code_tokens": 28, + "prompt_token_ids_sha256": "4e4f96ebb780ea4b4fa6c88daed1d4e10ae72248fd516ce84995e408b4884845", + "prompt_tokens": 125, + "task_id": 450, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "6e6d233b1f10c4b46e88fafad4ce5874246ab25f58fb60780fbb504f8c3f9c08", + "code_tokens": 25, + "prompt_token_ids_sha256": "71ae000b6c83590c335c1975e71759d65dd054f209ef0e598b8de4df165fe978", + "prompt_tokens": 92, + "task_id": 451, + "total_tokens": 117 + }, + { + "code_token_ids_sha256": "34bf668847489a9d1069389f36fa47eb8b0ec2b7f4ac0089a05664e31eae56de", + "code_tokens": 40, + "prompt_token_ids_sha256": "883b5dc650318f91b94a26af76d82519873b0393da18042ba7ab559805b7a14e", + "prompt_tokens": 94, + "task_id": 452, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "d8c9646e51abc74deaac841b8d38b050cecaa19375af7636e36007e9231767a4", + "code_tokens": 177, + "prompt_token_ids_sha256": "e8e15f3b452d0d1613ed04b9a9c6ab024eedabc7a9aadc760f758830e2e38704", + "prompt_tokens": 78, + "task_id": 453, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "95d843b6be0b136cf9f7acdd434c4e1c85173f81005011b7ca98b0f829f2fee4", + "code_tokens": 51, + "prompt_token_ids_sha256": "25efb54301bb5216fccb11c79fab3e2ce86c8e7bafec2e6ea815f0d135d3d15c", + "prompt_tokens": 87, + "task_id": 454, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "47cebe9b5b9156daa250e0a62f2460e7a23a8db4508ce6fd31fbff5a350e95ef", + "code_tokens": 69, + "prompt_token_ids_sha256": "4c646f17ea739fe1e63b62114e9bb7012f87235825e5b80e13c9cba2977d08cf", + "prompt_tokens": 77, + "task_id": 455, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "92289c7b0cd3f62545c3b7f22ee6616a6a091dfa11811dc1a6db66a8e58aaf92", + "code_tokens": 26, + "prompt_token_ids_sha256": "9724b389e3d1d2d634d3e0d6cc8a51da7ae6bd68bdea108d0f25719b1fc37b43", + "prompt_tokens": 140, + "task_id": 456, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "edab4f21625e8e1d5f509083bd9cdb02dd53e4cab419ab1b285fb561c9ce3031", + "code_tokens": 24, + "prompt_token_ids_sha256": "46564286d737c960d48362b61c7f29dbfc27a67d81ea176ab38d33621ca8c32d", + "prompt_tokens": 111, + "task_id": 457, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "824975efc00afe503739a15468bce0baae64302a9781e033c304ff5ae9feb99a", + "code_tokens": 15, + "prompt_token_ids_sha256": "09aa8f2c1f932b831e1bfa1c8b89b882cab53300e940c8d889353d5706bb3687", + "prompt_tokens": 73, + "task_id": 458, + "total_tokens": 88 + }, + { + "code_token_ids_sha256": "73fd5fcf063f2bba2e83ffd1e378b53595949578dd3145be5b2901b53cd0ca66", + "code_tokens": 43, + "prompt_token_ids_sha256": "e19e81dc399814df8c1ba8e712a346d191f80f96cdcf6ec0c370f20d2d79f281", + "prompt_tokens": 129, + "task_id": 459, + "total_tokens": 172 + }, + { + "code_token_ids_sha256": "38fa159347900dedc1599c25acf807e48f74165b377709d3b50f9cbb8781d0ec", + "code_tokens": 18, + "prompt_token_ids_sha256": "c958ca500b117601d3ad6ce2c03aa257a056229aad1ab2a2f707b198b724a4e5", + "prompt_tokens": 120, + "task_id": 460, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "b72e9bfc3a743f38515d2b7298346d87c9635caccd734d6c20c0eb25c48fed53", + "code_tokens": 49, + "prompt_token_ids_sha256": "803ae460ed2a08cc0ef97b62e7e812dca864495782d539f9480b93dec5cacd14", + "prompt_tokens": 73, + "task_id": 461, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "7be49b4cefe908a8520da5b464a8bea62498e7ae5dbadc0262c679fe61d8ad31", + "code_tokens": 59, + "prompt_token_ids_sha256": "4faa9ab17b29609052377a74bae62d607d9234acd9c0148de02b0551ba8c3d21", + "prompt_tokens": 873, + "task_id": 462, + "total_tokens": 932 + }, + { + "code_token_ids_sha256": "799876ae1c56cf410b868e35accf00202233c51b14fabbf853a421727b0d5115", + "code_tokens": 221, + "prompt_token_ids_sha256": "d29d77055068c1db171cc086e8821cbad5a7bdc9c100b6a0a4bbb51d03698484", + "prompt_tokens": 126, + "task_id": 463, + "total_tokens": 347 + }, + { + "code_token_ids_sha256": "8d2276f4ac0a54f9422d1ce67552b63cb612459adf9549a4e60e2745ba279202", + "code_tokens": 25, + "prompt_token_ids_sha256": "1ffe4c05437be989b0d0568cae7027aa37acb450204176bd542f45bfa7a335ca", + "prompt_tokens": 174, + "task_id": 464, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "4584abb6e69723060b114fda9d0b2197352b5257196fcd00c4095f9ca830b5f7", + "code_tokens": 36, + "prompt_token_ids_sha256": "6220a0d8257135b27191d36796a2d52d4908759575030c2e8d2b2da488c6523f", + "prompt_tokens": 140, + "task_id": 465, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "990cebba5654db5efdc3d6cbf2ab1f99f3f345a6d520d75e0def916e67a809ef", + "code_tokens": 158, + "prompt_token_ids_sha256": "8a752480a526df351c61f7a73a61c5796b27c6c4ea6543d81bdcfa20984a44f9", + "prompt_tokens": 125, + "task_id": 466, + "total_tokens": 283 + }, + { + "code_token_ids_sha256": "a16b8839c438a368ffeb9444e4357e41cbe70e978d0bd33532080bf575c8c95d", + "code_tokens": 92, + "prompt_token_ids_sha256": "800b42f5b9770f9fab69ff9bdc55d938c71539830113d30102b7a86a3ee692ad", + "prompt_tokens": 79, + "task_id": 467, + "total_tokens": 171 + }, + { + "code_token_ids_sha256": "adfd4dcf6190ed87d08d6f2523f49bb530ae6194b0d47bea8cade40a9baed761", + "code_tokens": 102, + "prompt_token_ids_sha256": "1c9c8b3b494e38033a7c59950df3721490dfacb66921cb39e679e7779377717f", + "prompt_tokens": 167, + "task_id": 468, + "total_tokens": 269 + }, + { + "code_token_ids_sha256": "21d7c5cc64c3aaf577d4fcc5c90c4ff25c1b2f5cc8b098d8689bd897e0cdc4b1", + "code_tokens": 171, + "prompt_token_ids_sha256": "29f8be02ca6f6e9d828c2f0a31851f23a169e0dc57ac11a5e9c6726cb2ffffa3", + "prompt_tokens": 134, + "task_id": 469, + "total_tokens": 305 + }, + { + "code_token_ids_sha256": "df52258f1eb4c2a24531d4908358282888360e708a7176f5496db30854057e40", + "code_tokens": 40, + "prompt_token_ids_sha256": "94bf6f3ac13cd6a4db30ff107bfe0769c6e25aba8339715062b574ad6617d047", + "prompt_tokens": 155, + "task_id": 470, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "4a6293101f7ffda2f1fc0c84585a18c347e35d3c0da93168adca4b657dd27773", + "code_tokens": 48, + "prompt_token_ids_sha256": "06db1e0c8c8003557a916b0c2936c754d15063ef718a77bd1d7ad8c4b1ac16f6", + "prompt_tokens": 115, + "task_id": 471, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "596b353305959444bd94475c0733ffd4c034599b23738073e66a8015c0c37dd8", + "code_tokens": 24, + "prompt_token_ids_sha256": "3bc93469404b2e69233b7d24d719f3fad8213503018ba8ee08947aef9b0597bd", + "prompt_tokens": 92, + "task_id": 472, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "51bd7c0375cf44db345777e2902d9292a1517fa24adcc0455147e356c654e65f", + "code_tokens": 48, + "prompt_token_ids_sha256": "a186c4d63207e1a827c173b8883d3dcb8b57658a555d706226207a86299910d6", + "prompt_tokens": 261, + "task_id": 473, + "total_tokens": 309 + }, + { + "code_token_ids_sha256": "1b9b4f55c9726a28215353bdd18277e740a988fecf66ccbd11b9b39185f69d85", + "code_tokens": 25, + "prompt_token_ids_sha256": "d769f5f91345494bdf0da25da73d62005c3a6b34a7199a593c31f296be29ca22", + "prompt_tokens": 81, + "task_id": 474, + "total_tokens": 106 + }, + { + "code_token_ids_sha256": "f69b9853ef0b4193c96571de882730ec5a0269623c113e7939c726537a25298a", + "code_tokens": 30, + "prompt_token_ids_sha256": "0f4a8990ad88e764f821040b7d8162c6999c0200abc2fbddb8b38cf350f3438b", + "prompt_tokens": 192, + "task_id": 475, + "total_tokens": 222 + }, + { + "code_token_ids_sha256": "c3e0f2bdfeabfd30cc0750011ead9573751c2f7465b1a79e763ba96b173d96c3", + "code_tokens": 19, + "prompt_token_ids_sha256": "e6228be40e60a0a5327ca9260bbabc3005e98542c2d33707914cc1a28fb136c3", + "prompt_tokens": 89, + "task_id": 476, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "70fe9cd235e043be67292905a0e830fa2be9eebf2eb2d526e8e49f673b28ca6c", + "code_tokens": 12, + "prompt_token_ids_sha256": "0052cd659431cf7bb94fbf81f8e7f606ca2589201c207a7d92243d95c99fde3e", + "prompt_tokens": 77, + "task_id": 477, + "total_tokens": 89 + }, + { + "code_token_ids_sha256": "4f274f8f398e92267a50ee9cece5b8edc0266a24c93b26590491961a68e67b12", + "code_tokens": 37, + "prompt_token_ids_sha256": "38f21a40da258e41a3f88ad325cd39f90abb9c3cfdf77359b10bf48d2d0ae907", + "prompt_tokens": 78, + "task_id": 478, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "7d1f8fb8a6b69d8487ad188517a2519dd18b54972bf7ae178b00122472b27fdc", + "code_tokens": 33, + "prompt_token_ids_sha256": "45fe823074f6ac983ffe28bf3435d0cd9ac1b8733c6598da43ddc5f47829d5de", + "prompt_tokens": 78, + "task_id": 479, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "64000266374211d8ea3f7ae5c714108713917f57e4786d67a54ff38e30ac8262", + "code_tokens": 94, + "prompt_token_ids_sha256": "2f6ff6b642a4ead3906882ef0d30ae6192a427e821d9d16d00cda635060715e5", + "prompt_tokens": 86, + "task_id": 480, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "fce84b3cb604d6f5c3ab35ac907eb18696be5179704b2d729c67b3fe234ce69a", + "code_tokens": 90, + "prompt_token_ids_sha256": "767ec1c109fd90852cc10ece0f23de3d3cb5536bba41c146fcb65901cab3fa68", + "prompt_tokens": 149, + "task_id": 481, + "total_tokens": 239 + }, + { + "code_token_ids_sha256": "7cab131088a1530bdf64ad42763b2edc941329874297ae93e91ff50de3749d66", + "code_tokens": 47, + "prompt_token_ids_sha256": "da3e0af41366aaa6d451475e0a1977366cbc2b12ef764da5a550b47512154653", + "prompt_tokens": 86, + "task_id": 482, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "c45579954174b51a53a047fd77285672bd3def31192bc602ba9a3b69929db01e", + "code_tokens": 61, + "prompt_token_ids_sha256": "863989ac3beed98b638186a735d29ed7a3fa6fb5b9ed8030cf91014f9e025689", + "prompt_tokens": 93, + "task_id": 483, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "1abab0e6128672771933873952d5c3f249af74348f3d615ec0f6a1d1a3d075c4", + "code_tokens": 39, + "prompt_token_ids_sha256": "2183358c70f6319ed8463ab717d8b542aac94e5a5dc9488b5da8a6de9e38ad33", + "prompt_tokens": 193, + "task_id": 484, + "total_tokens": 232 + }, + { + "code_token_ids_sha256": "bd04eb67f3a68081d023ed7ee5fcc5058171674765be71389853a9a8fce5ca40", + "code_tokens": 152, + "prompt_token_ids_sha256": "72a1181ffe60ceecc6dd38f4db031d6ec54ea3777c64ccd276780b85dd2cd3bc", + "prompt_tokens": 139, + "task_id": 485, + "total_tokens": 291 + }, + { + "code_token_ids_sha256": "55114f55cb7d2c6aabb527a32a63e6393757f9562b5334e5279bc59fbfcd7171", + "code_tokens": 100, + "prompt_token_ids_sha256": "f26210376f5c7fcf085c0afde5cac8eba2ef44b32a584014830787f0943d1bf9", + "prompt_tokens": 147, + "task_id": 486, + "total_tokens": 247 + }, + { + "code_token_ids_sha256": "82db5c50df7c9f327daf79557117aa4a22220115f28237543b33f156a1398a8b", + "code_tokens": 89, + "prompt_token_ids_sha256": "f59c61f303b8f48562dbeb75ec214a978110ce058001e5f843df1c3663b1ff48", + "prompt_tokens": 172, + "task_id": 487, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "1a5367ba7807b275e6543221bcbd574e6ab905dcfe91955755589d28fc591017", + "code_tokens": 40, + "prompt_token_ids_sha256": "14c48b11c25ffe32d2b9aef310e2eebf713324ce76055db26e3c862274680892", + "prompt_tokens": 119, + "task_id": 488, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "35d7facc91dbdf87feab39224f21ae2cc0d6ac86629582c897295e132da151bd", + "code_tokens": 76, + "prompt_token_ids_sha256": "0f20d0b8f2623f1f553a05b87954d1375ad2ee36a4d08a606e49c7c824067041", + "prompt_tokens": 103, + "task_id": 489, + "total_tokens": 179 + }, + { + "code_token_ids_sha256": "77f881c9580e5eb6cbc686a3eb57d3f4b665ebb893e7461ace6ffe17a9fdb603", + "code_tokens": 56, + "prompt_token_ids_sha256": "5feca945d0ab02f193d1bb2813c43b06bd9dce2720354940e1fea96d078d81b8", + "prompt_tokens": 220, + "task_id": 490, + "total_tokens": 276 + }, + { + "code_token_ids_sha256": "ab00cbf4671cf9568f64ea18a0fce2473cbf7a87412287f38b4d7d99dd9c2cd7", + "code_tokens": 34, + "prompt_token_ids_sha256": "a1074586b7d5e8ba54eb74979360ddc0985dedcc4690ea0e96aa05c66a852eac", + "prompt_tokens": 79, + "task_id": 491, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "dc7f9d03e15ff91b0afaffa2c18f82ef619790057cf796079e2a86eb2351d9dc", + "code_tokens": 96, + "prompt_token_ids_sha256": "b642a1f2501144f1dabbbb1cb9949b300278345dcc62521dece305bae7ea475a", + "prompt_tokens": 123, + "task_id": 492, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "23c40de2385c008e5be4b0194e11dce1dee0f80f3c6776ff59b284bf95cc1c9a", + "code_tokens": 451, + "prompt_token_ids_sha256": "c2cfb3be95303df90702ee8a9bdc4219c2fe02b87565f5fdfd878ac9ef0c02cc", + "prompt_tokens": 3722, + "task_id": 493, + "total_tokens": 4173 + }, + { + "code_token_ids_sha256": "93c994f70d3bc6a1f67d304d12b5dca6d3daaf6b3697bfee0f89076bed3c6993", + "code_tokens": 37, + "prompt_token_ids_sha256": "6d667f7560e92609220e768b7bb31e345b653941a9429f2063c06568befcce7d", + "prompt_tokens": 133, + "task_id": 494, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "78e752283c2744cbf23e66d626b36293032b6b3b8f566201d02fb6ec96914f22", + "code_tokens": 42, + "prompt_token_ids_sha256": "21ee124724ffa301449ae66148543b3fa5c34b21402ed3f321e0d11f560b3118", + "prompt_tokens": 132, + "task_id": 495, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "f44d9f573a5beac095b5e1aa840edb8539a0103b71082ded5dfae64ad02a64cc", + "code_tokens": 33, + "prompt_token_ids_sha256": "5f41fcc745a14d5a878ffed67d413f03e42540aa3b16fe0d94d23c598a2b4dfd", + "prompt_tokens": 220, + "task_id": 496, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "8a397b991fa55ec79eadfb6488a15ea897df0899ae7c8981c443ad06a57a1ee3", + "code_tokens": 43, + "prompt_token_ids_sha256": "2da55871caff716f28954caf30fa2db2ea6b2daea63d26837c43d7bdced9723f", + "prompt_tokens": 127, + "task_id": 497, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "6661f7ed19e406f2317fec4f44f0ada57ec1f0b10ea366d79b058d1f71267bbf", + "code_tokens": 73, + "prompt_token_ids_sha256": "359c1d8b8fd2d67651a7101634dd0185fc4b9b07f14ce424359f2ffa35ec1aa8", + "prompt_tokens": 74, + "task_id": 498, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "61b550bacbf4f5ade681ce6d3316f754b9b8632fad3ee11c8a0ae1c07709ddcb", + "code_tokens": 15, + "prompt_token_ids_sha256": "34a2ce2710676122000b47fab50cf955787eacb5ce87982294b291ee7d4cb080", + "prompt_tokens": 67, + "task_id": 499, + "total_tokens": 82 + }, + { + "code_token_ids_sha256": "7e42ecfac3cb92bcd549e06a3a6a118309e62316020f39621ad785ec9f656ab9", + "code_tokens": 34, + "prompt_token_ids_sha256": "18b4ce9c80f0e050e7718669fac870f89183cb4d1457262daab964172973a2f1", + "prompt_tokens": 133, + "task_id": 500, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "272ae1e07db15913e36686e828b6c6bee67b4ad5c15f959b237fe08281c6b697", + "code_tokens": 136, + "prompt_token_ids_sha256": "355babcc0017d88140442a53287e969a4fb18f8392a3ad62f668cd964a3a2fba", + "prompt_tokens": 83, + "task_id": 501, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "38dcc2c5f3608e347aae5b8c529611a497f5a4da064f97f2ed8aca95ed2f26b3", + "code_tokens": 17, + "prompt_token_ids_sha256": "f71db5cdd1104bed62b243ec11f2cdf048ef71d25650ba50d817e40a7a9242db", + "prompt_tokens": 72, + "task_id": 502, + "total_tokens": 89 + }, + { + "code_token_ids_sha256": "23d54ad1d9f9835151238f004ac6ec1d6a7a07eaf534236358da20919ae977b2", + "code_tokens": 33, + "prompt_token_ids_sha256": "1658555e651da81eb5e2a379d1eca2367bab07e7e57488ad54e826002f58949f", + "prompt_tokens": 204, + "task_id": 503, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "322ea773ac4d585f26c9018703a7fbe22c2e5fb20d6d70f0ee78242612d9a81d", + "code_tokens": 38, + "prompt_token_ids_sha256": "d979c6b02c5170f9edba9f3e40f138ce375ef5684bf6969a554863aa2210af60", + "prompt_tokens": 80, + "task_id": 504, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "612e4915029db18a774d073a4c8c06326fca881421ec262f904b724410817339", + "code_tokens": 61, + "prompt_token_ids_sha256": "8b2d23fe7b7e6ee8e2f05e2cbb590a48185455c802e87da445286d65ddc45359", + "prompt_tokens": 226, + "task_id": 505, + "total_tokens": 287 + }, + { + "code_token_ids_sha256": "97bc8713661d8215d82e9e1620c4b1f1a8255335b7c1df84a4a7c2bc1646c3cc", + "code_tokens": 143, + "prompt_token_ids_sha256": "84073a618bed6d26ec08d9b924a5bdc328364505d0b4361b079a4f5d7862e991", + "prompt_tokens": 90, + "task_id": 506, + "total_tokens": 233 + }, + { + "code_token_ids_sha256": "f0267cf2f4af677e78c2c9534a374e5600bf4530170a92850abd69ab1c2311ca", + "code_tokens": 41, + "prompt_token_ids_sha256": "c0a21f6cb63c71a868553c0f4f9efd0ce28994217d86dcdce03adf06ab7ae3d5", + "prompt_tokens": 158, + "task_id": 507, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "54a0439163c990f4b8bd409e0011c43266ce73efff02004e5819acffdbfea07c", + "code_tokens": 67, + "prompt_token_ids_sha256": "33d0a6c82b049e809e4a32abed723f7488a3dba8f09207305e4f7bcd0f2ab995", + "prompt_tokens": 122, + "task_id": 508, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "c919d532de1cc40a854433d29cd0a02a68f75fb068cc721aa8e8679abd46ae50", + "code_tokens": 76, + "prompt_token_ids_sha256": "55426ecf1f35826eb1932ae647a05e1df389b4feb38e2e2666c5815a816cf95b", + "prompt_tokens": 77, + "task_id": 509, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "55c06b75b4e56aa5bc66ce0005c5346ff8bb309046d7cedbcda2a87e4ffed01b", + "code_tokens": 137, + "prompt_token_ids_sha256": "095088515de6040e32a3d4dc9e9e68dcf3dedcbe44fa4571650d81ef1205eb34", + "prompt_tokens": 115, + "task_id": 510, + "total_tokens": 252 + } + ], + "token_manifest_sha256": "199a8836489af9bd0af3fec027e85d57df356bd9919492b24015de51d143f525", + "tokenizer_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + } + }, + "schema_version": 1 +} diff --git a/evidence/mbpp-v02-confirmation.json b/evidence/mbpp-v02-confirmation.json new file mode 100644 index 0000000..7608cd7 --- /dev/null +++ b/evidence/mbpp-v02-confirmation.json @@ -0,0 +1,66974 @@ +{ + "artifact_kind": "recurquant_mbpp_teacher_forced_evaluation", + "canonical_evidence_sha256": "2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0", + "created_at_utc": "2026-07-22T13:33:56.510744+00:00", + "evidence": { + "candidates": { + "mse_selected_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 0.5702063694000244, + "macro_mean_kl": 0.5921501630544662, + "macro_top1_agreement": 0.7581791734695434, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 0.7392329616546631, + "macro_mean_kl": 0.7701967484951019, + "macro_top1_agreement": 0.6997111868858338, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 0.8596235785484314, + "macro_mean_kl": 0.869078378200531, + "macro_top1_agreement": 0.6703913822174072, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 1.0457885901927948, + "macro_mean_kl": 1.030025339603424, + "macro_top1_agreement": 0.6364823293685913, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 1.437837839126587, + "code_tokens": 98, + "cvar95_kl": 4.351431846618652, + "delta_nll": 0.7109821438789368, + "max_kl": 6.220755577087402, + "mean_kl": 0.8511053323745728, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.7448979616165161 + }, + { + "candidate_nll": 1.5275182723999023, + "code_tokens": 19, + "cvar95_kl": 2.3682641983032227, + "delta_nll": 0.3681051731109619, + "max_kl": 2.3682641983032227, + "mean_kl": 0.4774366319179535, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.7894737124443054 + }, + { + "candidate_nll": 1.8732476234436035, + "code_tokens": 39, + "cvar95_kl": 5.275177955627441, + "delta_nll": 0.8578983545303345, + "max_kl": 6.5553202629089355, + "mean_kl": 0.91459721326828, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.731768012046814, + "code_tokens": 24, + "cvar95_kl": 2.365508556365967, + "delta_nll": 0.2854783535003662, + "max_kl": 2.484398126602173, + "mean_kl": 0.74074387550354, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.661340355873108, + "code_tokens": 25, + "cvar95_kl": 1.4808666706085205, + "delta_nll": 0.2535820007324219, + "max_kl": 1.9732763767242432, + "mean_kl": 0.33196744322776794, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.5109857320785522, + "code_tokens": 55, + "cvar95_kl": 3.3475801944732666, + "delta_nll": 0.6153741478919983, + "max_kl": 4.112247467041016, + "mean_kl": 0.6234012842178345, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 1.6018259525299072, + "code_tokens": 16, + "cvar95_kl": 3.564999580383301, + "delta_nll": 0.7723853588104248, + "max_kl": 3.564999580383301, + "mean_kl": 0.6987718343734741, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.515711545944214, + "code_tokens": 199, + "cvar95_kl": 6.729245662689209, + "delta_nll": 1.2938181161880493, + "max_kl": 11.558697700500488, + "mean_kl": 1.3418058156967163, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.5929648280143738 + }, + { + "candidate_nll": 2.3652708530426025, + "code_tokens": 28, + "cvar95_kl": 2.6924314498901367, + "delta_nll": 0.7991927862167358, + "max_kl": 3.071814775466919, + "mean_kl": 0.6068517565727234, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5663589239120483, + "code_tokens": 87, + "cvar95_kl": 3.9407448768615723, + "delta_nll": 0.6933763027191162, + "max_kl": 5.059126377105713, + "mean_kl": 0.7755260467529297, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.7011494040489197 + }, + { + "candidate_nll": 1.7174787521362305, + "code_tokens": 35, + "cvar95_kl": 3.3629608154296875, + "delta_nll": 0.48451459407806396, + "max_kl": 3.828634738922119, + "mean_kl": 0.7272599339485168, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.6285714507102966 + }, + { + "candidate_nll": 1.5806151628494263, + "code_tokens": 62, + "cvar95_kl": 3.7029905319213867, + "delta_nll": 0.9413235783576965, + "max_kl": 4.377253532409668, + "mean_kl": 0.8842069506645203, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 2.112473249435425, + "code_tokens": 58, + "cvar95_kl": 5.068362712860107, + "delta_nll": 1.007415533065796, + "max_kl": 6.522302150726318, + "mean_kl": 0.976349949836731, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.6896551847457886 + }, + { + "candidate_nll": 2.333246946334839, + "code_tokens": 79, + "cvar95_kl": 6.9539594650268555, + "delta_nll": 1.1741987466812134, + "max_kl": 8.396726608276367, + "mean_kl": 1.4176898002624512, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.5316455960273743 + }, + { + "candidate_nll": 1.8327233791351318, + "code_tokens": 58, + "cvar95_kl": 4.431029796600342, + "delta_nll": 0.7396035194396973, + "max_kl": 5.23704719543457, + "mean_kl": 0.8934630155563354, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 2.3810391426086426, + "code_tokens": 48, + "cvar95_kl": 5.999946594238281, + "delta_nll": 1.218841314315796, + "max_kl": 6.224644660949707, + "mean_kl": 1.4035091400146484, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.5208333134651184 + }, + { + "candidate_nll": 1.9071476459503174, + "code_tokens": 37, + "cvar95_kl": 3.333885669708252, + "delta_nll": 0.7817151546478271, + "max_kl": 4.774967193603516, + "mean_kl": 0.6543330550193787, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.6756756901741028 + }, + { + "candidate_nll": 1.3782262802124023, + "code_tokens": 61, + "cvar95_kl": 3.308793544769287, + "delta_nll": 0.670819103717804, + "max_kl": 4.517922878265381, + "mean_kl": 0.679639995098114, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.6721311211585999 + }, + { + "candidate_nll": 1.6747642755508423, + "code_tokens": 77, + "cvar95_kl": 3.6212124824523926, + "delta_nll": 0.9050447344779968, + "max_kl": 4.1913862228393555, + "mean_kl": 0.8814529776573181, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 2.3024284839630127, + "code_tokens": 98, + "cvar95_kl": 6.912336826324463, + "delta_nll": 1.280725359916687, + "max_kl": 9.342297554016113, + "mean_kl": 1.227766513824463, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.6224489808082581 + }, + { + "candidate_nll": 1.940861701965332, + "code_tokens": 162, + "cvar95_kl": 5.761036396026611, + "delta_nll": 1.355710506439209, + "max_kl": 7.307144641876221, + "mean_kl": 1.4084795713424683, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.5370370149612427 + }, + { + "candidate_nll": 1.9787956476211548, + "code_tokens": 103, + "cvar95_kl": 4.169778347015381, + "delta_nll": 1.2343297004699707, + "max_kl": 4.880768775939941, + "mean_kl": 1.2213464975357056, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.6310679316520691 + }, + { + "candidate_nll": 2.2950422763824463, + "code_tokens": 73, + "cvar95_kl": 3.780155658721924, + "delta_nll": 0.976071834564209, + "max_kl": 5.071652412414551, + "mean_kl": 0.7210307717323303, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.7397260069847107 + }, + { + "candidate_nll": 2.143333673477173, + "code_tokens": 119, + "cvar95_kl": 4.810328960418701, + "delta_nll": 1.1603600978851318, + "max_kl": 6.720498561859131, + "mean_kl": 1.1632674932479858, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.5630252361297607 + }, + { + "candidate_nll": 1.3111584186553955, + "code_tokens": 17, + "cvar95_kl": 4.171998977661133, + "delta_nll": 0.1216350793838501, + "max_kl": 4.171998977661133, + "mean_kl": 0.7182435393333435, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 2.308159589767456, + "code_tokens": 56, + "cvar95_kl": 4.133174419403076, + "delta_nll": 0.8824578523635864, + "max_kl": 5.817321300506592, + "mean_kl": 0.8873200416564941, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.6607142686843872 + }, + { + "candidate_nll": 1.2595347166061401, + "code_tokens": 57, + "cvar95_kl": 5.3571457862854, + "delta_nll": 0.792174220085144, + "max_kl": 7.418487548828125, + "mean_kl": 0.7693098187446594, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.8070175647735596 + }, + { + "candidate_nll": 1.3133705854415894, + "code_tokens": 63, + "cvar95_kl": 3.237618923187256, + "delta_nll": 0.6806253790855408, + "max_kl": 3.6589512825012207, + "mean_kl": 0.6985809206962585, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 2.0654091835021973, + "code_tokens": 200, + "cvar95_kl": 5.61568546295166, + "delta_nll": 1.2274736166000366, + "max_kl": 8.750672340393066, + "mean_kl": 1.217779517173767, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.5799999833106995 + }, + { + "candidate_nll": 1.554159164428711, + "code_tokens": 30, + "cvar95_kl": 3.463693380355835, + "delta_nll": 0.46995651721954346, + "max_kl": 4.301401615142822, + "mean_kl": 0.6157476902008057, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.8970035314559937, + "code_tokens": 28, + "cvar95_kl": 5.645899295806885, + "delta_nll": 1.2806525230407715, + "max_kl": 5.692856788635254, + "mean_kl": 1.0834671258926392, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.7571412324905396, + "code_tokens": 26, + "cvar95_kl": 5.2432074546813965, + "delta_nll": 0.9861845970153809, + "max_kl": 6.532666206359863, + "mean_kl": 0.9541648626327515, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.5769230723381042 + }, + { + "candidate_nll": 1.5984139442443848, + "code_tokens": 54, + "cvar95_kl": 3.7895619869232178, + "delta_nll": 0.7654725909233093, + "max_kl": 4.494767189025879, + "mean_kl": 0.7355435490608215, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.6818748712539673, + "code_tokens": 47, + "cvar95_kl": 4.013880252838135, + "delta_nll": 0.7251387238502502, + "max_kl": 6.493446350097656, + "mean_kl": 0.7611171007156372, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.7021276354789734 + }, + { + "candidate_nll": 1.8551881313323975, + "code_tokens": 93, + "cvar95_kl": 5.879759788513184, + "delta_nll": 1.1206586360931396, + "max_kl": 9.46138858795166, + "mean_kl": 1.075543999671936, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.2261435985565186, + "code_tokens": 30, + "cvar95_kl": 2.757434129714966, + "delta_nll": 0.366769015789032, + "max_kl": 3.489267110824585, + "mean_kl": 0.4687761664390564, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.7666666507720947 + }, + { + "candidate_nll": 2.0647683143615723, + "code_tokens": 89, + "cvar95_kl": 5.481202125549316, + "delta_nll": 0.9962650537490845, + "max_kl": 7.952178955078125, + "mean_kl": 1.0418976545333862, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.6629213690757751 + }, + { + "candidate_nll": 1.628982424736023, + "code_tokens": 70, + "cvar95_kl": 3.3860714435577393, + "delta_nll": 0.6647331118583679, + "max_kl": 4.6697587966918945, + "mean_kl": 0.8595913648605347, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 2.0597481727600098, + "code_tokens": 26, + "cvar95_kl": 2.989271879196167, + "delta_nll": 0.4465210437774658, + "max_kl": 3.2768750190734863, + "mean_kl": 0.6326386332511902, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.7307692170143127 + }, + { + "candidate_nll": 2.046163558959961, + "code_tokens": 48, + "cvar95_kl": 3.5074586868286133, + "delta_nll": 1.088456630706787, + "max_kl": 4.420538425445557, + "mean_kl": 0.8607252240180969, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.6458333134651184 + }, + { + "candidate_nll": 1.3034740686416626, + "code_tokens": 29, + "cvar95_kl": 2.1741554737091064, + "delta_nll": 0.43432605266571045, + "max_kl": 2.288686513900757, + "mean_kl": 0.4486943185329437, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.8275862336158752 + }, + { + "candidate_nll": 1.6809512376785278, + "code_tokens": 17, + "cvar95_kl": 2.059248685836792, + "delta_nll": 0.5677881240844727, + "max_kl": 2.059248685836792, + "mean_kl": 0.5237215757369995, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.4593381881713867, + "code_tokens": 37, + "cvar95_kl": 2.116163730621338, + "delta_nll": 0.5391220450401306, + "max_kl": 2.314948081970215, + "mean_kl": 0.49860671162605286, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 1.5211553573608398, + "code_tokens": 121, + "cvar95_kl": 5.004239559173584, + "delta_nll": 1.009753704071045, + "max_kl": 6.1679911613464355, + "mean_kl": 0.9638730883598328, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.6611570119857788 + }, + { + "candidate_nll": 2.239880084991455, + "code_tokens": 30, + "cvar95_kl": 3.435619592666626, + "delta_nll": 0.841036319732666, + "max_kl": 4.433403491973877, + "mean_kl": 0.9730808734893799, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.5666666626930237 + }, + { + "candidate_nll": 2.0859503746032715, + "code_tokens": 74, + "cvar95_kl": 5.128108978271484, + "delta_nll": 1.1874786615371704, + "max_kl": 6.625497817993164, + "mean_kl": 1.0472651720046997, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 2.1283061504364014, + "code_tokens": 53, + "cvar95_kl": 4.611486911773682, + "delta_nll": 1.2938823699951172, + "max_kl": 5.384513854980469, + "mean_kl": 1.2723076343536377, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.6415094137191772 + }, + { + "candidate_nll": 2.0187103748321533, + "code_tokens": 21, + "cvar95_kl": 0.8156514167785645, + "delta_nll": 0.15931463241577148, + "max_kl": 0.8530668616294861, + "mean_kl": 0.2385498285293579, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.9047619104385376 + }, + { + "candidate_nll": 1.6533855199813843, + "code_tokens": 20, + "cvar95_kl": 1.4527795314788818, + "delta_nll": 0.29097187519073486, + "max_kl": 1.4527795314788818, + "mean_kl": 0.354911744594574, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.9091107845306396, + "code_tokens": 122, + "cvar95_kl": 4.473572731018066, + "delta_nll": 1.0901124477386475, + "max_kl": 5.267557144165039, + "mean_kl": 1.1033142805099487, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.6147540807723999 + }, + { + "candidate_nll": 1.7527860403060913, + "code_tokens": 94, + "cvar95_kl": 4.880742073059082, + "delta_nll": 0.9126462340354919, + "max_kl": 6.542252063751221, + "mean_kl": 0.8952356576919556, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.6808510422706604 + }, + { + "candidate_nll": 1.2744691371917725, + "code_tokens": 12, + "cvar95_kl": 0.8177033066749573, + "delta_nll": 0.20916759967803955, + "max_kl": 0.8177033066749573, + "mean_kl": 0.18918752670288086, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.141298532485962, + "code_tokens": 38, + "cvar95_kl": 4.041401386260986, + "delta_nll": 0.63489830493927, + "max_kl": 4.969882965087891, + "mean_kl": 0.7539941668510437, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 1.2633423805236816, + "code_tokens": 58, + "cvar95_kl": 1.7498207092285156, + "delta_nll": 0.35855525732040405, + "max_kl": 2.4351818561553955, + "mean_kl": 0.32653743028640747, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.8275862336158752 + }, + { + "candidate_nll": 1.620820164680481, + "code_tokens": 54, + "cvar95_kl": 5.895735263824463, + "delta_nll": 1.0245563983917236, + "max_kl": 8.724684715270996, + "mean_kl": 0.9615625739097595, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.5960285663604736, + "code_tokens": 40, + "cvar95_kl": 3.86961030960083, + "delta_nll": 0.664488673210144, + "max_kl": 4.1336798667907715, + "mean_kl": 0.739889919757843, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.7749999761581421 + }, + { + "candidate_nll": 1.4100948572158813, + "code_tokens": 115, + "cvar95_kl": 3.976778268814087, + "delta_nll": 0.8975489139556885, + "max_kl": 4.926577568054199, + "mean_kl": 0.849139392375946, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.7043478488922119 + }, + { + "candidate_nll": 1.330396056175232, + "code_tokens": 60, + "cvar95_kl": 5.183268070220947, + "delta_nll": 0.8057823777198792, + "max_kl": 5.726524353027344, + "mean_kl": 0.870845377445221, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.7446472644805908, + "code_tokens": 124, + "cvar95_kl": 4.777351379394531, + "delta_nll": 0.9746601581573486, + "max_kl": 7.386229515075684, + "mean_kl": 0.916567862033844, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.6048387289047241 + }, + { + "candidate_nll": 1.522768497467041, + "code_tokens": 91, + "cvar95_kl": 3.4123008251190186, + "delta_nll": 0.7520198225975037, + "max_kl": 4.66524076461792, + "mean_kl": 0.7261348366737366, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.7362637519836426 + }, + { + "candidate_nll": 2.352339029312134, + "code_tokens": 125, + "cvar95_kl": 6.275059700012207, + "delta_nll": 1.532421588897705, + "max_kl": 7.9018120765686035, + "mean_kl": 1.4746301174163818, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.527999997138977 + }, + { + "candidate_nll": 1.540420413017273, + "code_tokens": 26, + "cvar95_kl": 1.7736355066299438, + "delta_nll": 0.4045438766479492, + "max_kl": 1.8003995418548584, + "mean_kl": 0.4136735796928406, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 2.790148973464966, + "code_tokens": 24, + "cvar95_kl": 2.1047091484069824, + "delta_nll": 0.5497238636016846, + "max_kl": 2.5175137519836426, + "mean_kl": 0.5278720855712891, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.6732341051101685, + "code_tokens": 180, + "cvar95_kl": 3.640596628189087, + "delta_nll": 0.8190522789955139, + "max_kl": 6.573902606964111, + "mean_kl": 0.8366333842277527, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 2.4713895320892334, + "code_tokens": 40, + "cvar95_kl": 6.343910217285156, + "delta_nll": 1.096580982208252, + "max_kl": 7.595676898956299, + "mean_kl": 1.192630648612976, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.732940435409546, + "code_tokens": 72, + "cvar95_kl": 4.269186496734619, + "delta_nll": 0.7579435706138611, + "max_kl": 5.194479465484619, + "mean_kl": 0.8687319755554199, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 1.483136773109436, + "code_tokens": 20, + "cvar95_kl": 1.9894614219665527, + "delta_nll": -0.09443438053131104, + "max_kl": 1.9894614219665527, + "mean_kl": 0.39476698637008667, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5741811990737915, + "code_tokens": 76, + "cvar95_kl": 2.7298948764801025, + "delta_nll": 0.5835368633270264, + "max_kl": 4.470555782318115, + "mean_kl": 0.5563408136367798, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.8639322519302368, + "code_tokens": 42, + "cvar95_kl": 1.9470844268798828, + "delta_nll": 0.4921908378601074, + "max_kl": 3.8022232055664062, + "mean_kl": 0.4195159375667572, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 0.9814545512199402, + "code_tokens": 28, + "cvar95_kl": 2.935736656188965, + "delta_nll": 0.3982313871383667, + "max_kl": 4.827160358428955, + "mean_kl": 0.3858254551887512, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.871192216873169, + "code_tokens": 58, + "cvar95_kl": 4.539680004119873, + "delta_nll": 0.8377153873443604, + "max_kl": 4.981967449188232, + "mean_kl": 0.7963540554046631, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.3158167600631714, + "code_tokens": 25, + "cvar95_kl": 2.733332633972168, + "delta_nll": 0.4471673369407654, + "max_kl": 4.223751544952393, + "mean_kl": 0.43643128871917725, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.7155133485794067, + "code_tokens": 88, + "cvar95_kl": 3.522193193435669, + "delta_nll": 0.7505055069923401, + "max_kl": 3.939423084259033, + "mean_kl": 0.7995510101318359, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.6955205202102661, + "code_tokens": 42, + "cvar95_kl": 1.7402547597885132, + "delta_nll": 0.5680032968521118, + "max_kl": 2.30548095703125, + "mean_kl": 0.4710255563259125, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 1.5900367498397827, + "code_tokens": 24, + "cvar95_kl": 3.362295627593994, + "delta_nll": 0.7704859972000122, + "max_kl": 5.007962226867676, + "mean_kl": 0.5839657187461853, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 1.0337450504302979, + "code_tokens": 24, + "cvar95_kl": 1.7097326517105103, + "delta_nll": 0.34887367486953735, + "max_kl": 1.820784091949463, + "mean_kl": 0.42712411284446716, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.9986655712127686, + "code_tokens": 43, + "cvar95_kl": 3.490222215652466, + "delta_nll": 0.9314864873886108, + "max_kl": 4.165478706359863, + "mean_kl": 0.6279502511024475, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.7906976938247681 + }, + { + "candidate_nll": 1.4780582189559937, + "code_tokens": 24, + "cvar95_kl": 2.7348403930664062, + "delta_nll": 0.591838538646698, + "max_kl": 2.764106512069702, + "mean_kl": 0.62176513671875, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.7916666865348816 + }, + { + "candidate_nll": 1.3939669132232666, + "code_tokens": 14, + "cvar95_kl": 2.5942883491516113, + "delta_nll": 0.08134126663208008, + "max_kl": 2.5942883491516113, + "mean_kl": 0.34973660111427307, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.7820640802383423, + "code_tokens": 41, + "cvar95_kl": 4.869851589202881, + "delta_nll": 1.2910208702087402, + "max_kl": 5.4240617752075195, + "mean_kl": 1.1749684810638428, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.5853658318519592 + }, + { + "candidate_nll": 1.5149013996124268, + "code_tokens": 32, + "cvar95_kl": 2.0438811779022217, + "delta_nll": 0.5265614986419678, + "max_kl": 2.6868960857391357, + "mean_kl": 0.5388344526290894, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.728800892829895, + "code_tokens": 53, + "cvar95_kl": 5.352388858795166, + "delta_nll": 0.9671059846878052, + "max_kl": 6.601135730743408, + "mean_kl": 0.8195364475250244, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.7358490824699402 + }, + { + "candidate_nll": 1.0845825672149658, + "code_tokens": 51, + "cvar95_kl": 3.7059097290039062, + "delta_nll": 0.7062914371490479, + "max_kl": 4.448524475097656, + "mean_kl": 0.6864140629768372, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.7254902124404907 + }, + { + "candidate_nll": 2.463548183441162, + "code_tokens": 36, + "cvar95_kl": 3.073725700378418, + "delta_nll": 0.771392822265625, + "max_kl": 3.097757339477539, + "mean_kl": 0.7697619795799255, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 2.062177896499634, + "code_tokens": 24, + "cvar95_kl": 1.7952526807785034, + "delta_nll": 0.5014256238937378, + "max_kl": 2.6227877140045166, + "mean_kl": 0.3531194031238556, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.159846067428589, + "code_tokens": 39, + "cvar95_kl": 4.348177909851074, + "delta_nll": 0.8455116748809814, + "max_kl": 4.41887092590332, + "mean_kl": 1.273642659187317, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.5897436141967773 + }, + { + "candidate_nll": 1.7310853004455566, + "code_tokens": 84, + "cvar95_kl": 4.761188507080078, + "delta_nll": 1.032672643661499, + "max_kl": 6.709907531738281, + "mean_kl": 1.0695093870162964, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.6309523582458496 + }, + { + "candidate_nll": 1.6105939149856567, + "code_tokens": 32, + "cvar95_kl": 3.1254615783691406, + "delta_nll": 0.6878256797790527, + "max_kl": 3.208247661590576, + "mean_kl": 0.6040952801704407, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.71875 + }, + { + "candidate_nll": 1.84221613407135, + "code_tokens": 19, + "cvar95_kl": 3.2909109592437744, + "delta_nll": 0.5320525169372559, + "max_kl": 3.2909109592437744, + "mean_kl": 0.6661286354064941, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.5683282613754272, + "code_tokens": 50, + "cvar95_kl": 4.090367794036865, + "delta_nll": 0.7503514885902405, + "max_kl": 6.3829026222229, + "mean_kl": 0.8958125114440918, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.6200000047683716 + }, + { + "candidate_nll": 1.689907431602478, + "code_tokens": 71, + "cvar95_kl": 5.390604019165039, + "delta_nll": 1.190093755722046, + "max_kl": 5.7891316413879395, + "mean_kl": 1.200129747390747, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.6478873491287231 + }, + { + "candidate_nll": 2.016000270843506, + "code_tokens": 28, + "cvar95_kl": 3.39534592628479, + "delta_nll": 0.628031849861145, + "max_kl": 3.703434944152832, + "mean_kl": 0.6244677901268005, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.5485821962356567, + "code_tokens": 76, + "cvar95_kl": 3.133467674255371, + "delta_nll": 0.6793208122253418, + "max_kl": 4.241065502166748, + "mean_kl": 0.678875744342804, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 1.839333176612854, + "code_tokens": 34, + "cvar95_kl": 3.196134567260742, + "delta_nll": 0.8767479658126831, + "max_kl": 3.306941032409668, + "mean_kl": 0.624768078327179, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.469204902648926, + "code_tokens": 11, + "cvar95_kl": 1.7996630668640137, + "delta_nll": 0.32181310653686523, + "max_kl": 1.7996630668640137, + "mean_kl": 0.46649789810180664, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 2.114027500152588, + "code_tokens": 31, + "cvar95_kl": 2.774495840072632, + "delta_nll": 0.6977599859237671, + "max_kl": 2.913696765899658, + "mean_kl": 0.7990099191665649, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.568469524383545, + "code_tokens": 118, + "cvar95_kl": 3.1507060527801514, + "delta_nll": 0.7216932773590088, + "max_kl": 4.130584716796875, + "mean_kl": 0.7494707107543945, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.6610169410705566 + }, + { + "candidate_nll": 1.1507320404052734, + "code_tokens": 60, + "cvar95_kl": 3.0679829120635986, + "delta_nll": 0.375036358833313, + "max_kl": 3.3976972103118896, + "mean_kl": 0.5510272979736328, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.7833333611488342 + }, + { + "candidate_nll": 1.5297048091888428, + "code_tokens": 47, + "cvar95_kl": 3.4964656829833984, + "delta_nll": 0.6342548727989197, + "max_kl": 4.989348888397217, + "mean_kl": 0.8003712296485901, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.7234042286872864 + }, + { + "candidate_nll": 2.136024236679077, + "code_tokens": 84, + "cvar95_kl": 4.908811569213867, + "delta_nll": 0.984009861946106, + "max_kl": 8.32959270477295, + "mean_kl": 0.9612395167350769, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.6547619104385376 + }, + { + "candidate_nll": 1.7778691053390503, + "code_tokens": 27, + "cvar95_kl": 2.3641579151153564, + "delta_nll": 0.7838430404663086, + "max_kl": 2.7429707050323486, + "mean_kl": 0.5442574620246887, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 1.654711127281189, + "code_tokens": 19, + "cvar95_kl": 1.6343698501586914, + "delta_nll": 0.04280400276184082, + "max_kl": 1.6343698501586914, + "mean_kl": 0.44417592883110046, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.611034870147705, + "code_tokens": 114, + "cvar95_kl": 3.9537465572357178, + "delta_nll": 0.8380223512649536, + "max_kl": 4.154403209686279, + "mean_kl": 0.9484782218933105, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.466111183166504, + "code_tokens": 40, + "cvar95_kl": 4.551571369171143, + "delta_nll": 0.7382179498672485, + "max_kl": 5.785944938659668, + "mean_kl": 0.7726505398750305, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 2.170844078063965, + "code_tokens": 25, + "cvar95_kl": 1.830533742904663, + "delta_nll": 0.6851658821105957, + "max_kl": 1.9657474756240845, + "mean_kl": 0.47394439578056335, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.4679319858551025, + "code_tokens": 23, + "cvar95_kl": 2.166440725326538, + "delta_nll": 0.6972429156303406, + "max_kl": 2.2522311210632324, + "mean_kl": 0.5364603400230408, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.2561111450195312, + "code_tokens": 77, + "cvar95_kl": 5.846391677856445, + "delta_nll": 1.2397615909576416, + "max_kl": 6.894842147827148, + "mean_kl": 1.2531832456588745, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.8633546829223633, + "code_tokens": 19, + "cvar95_kl": 1.6350184679031372, + "delta_nll": 0.755689263343811, + "max_kl": 1.6350184679031372, + "mean_kl": 0.3480040431022644, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.3318679332733154, + "code_tokens": 37, + "cvar95_kl": 6.5147705078125, + "delta_nll": 1.3486242294311523, + "max_kl": 6.986086845397949, + "mean_kl": 1.5934216976165771, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.5135135054588318 + }, + { + "candidate_nll": 1.8219059705734253, + "code_tokens": 33, + "cvar95_kl": 2.950320243835449, + "delta_nll": 0.4387410879135132, + "max_kl": 3.499119281768799, + "mean_kl": 0.5838028788566589, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 1.571190595626831, + "code_tokens": 95, + "cvar95_kl": 5.023710250854492, + "delta_nll": 0.9113491773605347, + "max_kl": 7.053951263427734, + "mean_kl": 0.9139501452445984, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.7157894968986511 + }, + { + "candidate_nll": 2.039400815963745, + "code_tokens": 125, + "cvar95_kl": 5.159369468688965, + "delta_nll": 1.0262404680252075, + "max_kl": 7.109761714935303, + "mean_kl": 1.0650709867477417, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.6399999856948853 + }, + { + "candidate_nll": 1.925919532775879, + "code_tokens": 168, + "cvar95_kl": 5.99751091003418, + "delta_nll": 1.2128710746765137, + "max_kl": 7.38164758682251, + "mean_kl": 1.317311406135559, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.5357142686843872 + }, + { + "candidate_nll": 1.532333254814148, + "code_tokens": 31, + "cvar95_kl": 2.7582340240478516, + "delta_nll": 0.21693110466003418, + "max_kl": 3.589409351348877, + "mean_kl": 0.6136119961738586, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.8064516186714172 + }, + { + "candidate_nll": 1.9273406267166138, + "code_tokens": 83, + "cvar95_kl": 5.184304237365723, + "delta_nll": 1.1656339168548584, + "max_kl": 6.698734760284424, + "mean_kl": 1.0433247089385986, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.6746987700462341 + }, + { + "candidate_nll": 1.5729767084121704, + "code_tokens": 50, + "cvar95_kl": 4.1302056312561035, + "delta_nll": 0.8730342388153076, + "max_kl": 4.913782119750977, + "mean_kl": 0.8164222836494446, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.6399999856948853 + }, + { + "candidate_nll": 1.329505205154419, + "code_tokens": 69, + "cvar95_kl": 4.265841007232666, + "delta_nll": 0.9623232483863831, + "max_kl": 4.862369060516357, + "mean_kl": 0.9329202771186829, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.6811594367027283 + }, + { + "candidate_nll": 2.0954368114471436, + "code_tokens": 50, + "cvar95_kl": 5.004846572875977, + "delta_nll": 1.0105406045913696, + "max_kl": 6.332308292388916, + "mean_kl": 1.0917145013809204, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.081373453140259, + "code_tokens": 161, + "cvar95_kl": 5.520568370819092, + "delta_nll": 1.0775917768478394, + "max_kl": 7.0954766273498535, + "mean_kl": 1.0200989246368408, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.7018633484840393 + }, + { + "candidate_nll": 1.5112152099609375, + "code_tokens": 53, + "cvar95_kl": 4.294021129608154, + "delta_nll": 0.8554390668869019, + "max_kl": 5.576375961303711, + "mean_kl": 0.7886008620262146, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.6603773832321167 + }, + { + "candidate_nll": 1.3950626850128174, + "code_tokens": 97, + "cvar95_kl": 4.250563621520996, + "delta_nll": 0.9495346546173096, + "max_kl": 4.96079683303833, + "mean_kl": 0.8034611940383911, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.7525773048400879 + }, + { + "candidate_nll": 1.9310895204544067, + "code_tokens": 22, + "cvar95_kl": 2.032834053039551, + "delta_nll": 0.6172765493392944, + "max_kl": 2.534329891204834, + "mean_kl": 0.5720844864845276, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.5270134210586548, + "code_tokens": 34, + "cvar95_kl": 4.532155513763428, + "delta_nll": 0.6581093072891235, + "max_kl": 6.442066669464111, + "mean_kl": 0.7767821550369263, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.394924283027649, + "code_tokens": 81, + "cvar95_kl": 3.5088982582092285, + "delta_nll": 0.5362810492515564, + "max_kl": 5.037527561187744, + "mean_kl": 0.7119351625442505, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.654321014881134 + }, + { + "candidate_nll": 1.3312137126922607, + "code_tokens": 17, + "cvar95_kl": 1.5652077198028564, + "delta_nll": 0.06911039352416992, + "max_kl": 1.5652077198028564, + "mean_kl": 0.22962011396884918, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.8823529481887817 + }, + { + "candidate_nll": 1.594070315361023, + "code_tokens": 181, + "cvar95_kl": 4.279249668121338, + "delta_nll": 0.768812894821167, + "max_kl": 8.66480827331543, + "mean_kl": 0.8002496361732483, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.6850828528404236 + }, + { + "candidate_nll": 1.6760903596878052, + "code_tokens": 63, + "cvar95_kl": 3.077331066131592, + "delta_nll": 0.6956818699836731, + "max_kl": 4.401571273803711, + "mean_kl": 0.6958545446395874, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.7301587462425232 + }, + { + "candidate_nll": 0.9973301291465759, + "code_tokens": 35, + "cvar95_kl": 2.331942081451416, + "delta_nll": 0.30380702018737793, + "max_kl": 2.804950714111328, + "mean_kl": 0.41396552324295044, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.8285714387893677 + }, + { + "candidate_nll": 1.4302093982696533, + "code_tokens": 25, + "cvar95_kl": 2.184866189956665, + "delta_nll": 0.625421941280365, + "max_kl": 3.1138641834259033, + "mean_kl": 0.4299301505088806, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.9984405040740967, + "code_tokens": 60, + "cvar95_kl": 6.066128253936768, + "delta_nll": 0.9100750684738159, + "max_kl": 8.73543930053711, + "mean_kl": 0.956393837928772, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.013244867324829, + "code_tokens": 86, + "cvar95_kl": 4.19992733001709, + "delta_nll": 1.1015980243682861, + "max_kl": 6.164646148681641, + "mean_kl": 0.934320867061615, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.6976743936538696 + }, + { + "candidate_nll": 1.571995496749878, + "code_tokens": 40, + "cvar95_kl": 3.1403117179870605, + "delta_nll": 0.6458297371864319, + "max_kl": 3.3989503383636475, + "mean_kl": 0.6274427175521851, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.7250000238418579 + }, + { + "candidate_nll": 2.090998649597168, + "code_tokens": 27, + "cvar95_kl": 1.9174832105636597, + "delta_nll": 0.5274384021759033, + "max_kl": 1.9958009719848633, + "mean_kl": 0.47828900814056396, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.9251688718795776, + "code_tokens": 52, + "cvar95_kl": 5.209041595458984, + "delta_nll": 0.8355859518051147, + "max_kl": 5.464498519897461, + "mean_kl": 1.0447616577148438, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.6346153616905212 + }, + { + "candidate_nll": 1.6442748308181763, + "code_tokens": 71, + "cvar95_kl": 5.6382036209106445, + "delta_nll": 0.9024007320404053, + "max_kl": 7.8199968338012695, + "mean_kl": 0.9220471382141113, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.6901408433914185 + }, + { + "candidate_nll": 1.6607983112335205, + "code_tokens": 25, + "cvar95_kl": 4.604377269744873, + "delta_nll": 0.5589466094970703, + "max_kl": 6.188091278076172, + "mean_kl": 0.793627917766571, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.4450721740722656, + "code_tokens": 103, + "cvar95_kl": 3.9904944896698, + "delta_nll": 0.8505765199661255, + "max_kl": 4.764760494232178, + "mean_kl": 0.8784255385398865, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.6699029207229614 + }, + { + "candidate_nll": 1.8436771631240845, + "code_tokens": 116, + "cvar95_kl": 3.7996137142181396, + "delta_nll": 0.7861391305923462, + "max_kl": 6.018502712249756, + "mean_kl": 0.8208531737327576, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.6637930870056152 + }, + { + "candidate_nll": 1.7308849096298218, + "code_tokens": 107, + "cvar95_kl": 5.088382244110107, + "delta_nll": 1.0825386047363281, + "max_kl": 6.895065784454346, + "mean_kl": 1.054499864578247, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.5794392228126526 + }, + { + "candidate_nll": 1.5361276865005493, + "code_tokens": 54, + "cvar95_kl": 2.5602316856384277, + "delta_nll": 0.5930899381637573, + "max_kl": 3.058537006378174, + "mean_kl": 0.5787672996520996, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.7856906652450562, + "code_tokens": 46, + "cvar95_kl": 5.896358489990234, + "delta_nll": 1.1522982120513916, + "max_kl": 8.174508094787598, + "mean_kl": 1.1224322319030762, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.3115392923355103, + "code_tokens": 170, + "cvar95_kl": 5.511594772338867, + "delta_nll": 0.9823925495147705, + "max_kl": 8.774435043334961, + "mean_kl": 1.0058202743530273, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 2.3939850330352783, + "code_tokens": 45, + "cvar95_kl": 3.928727388381958, + "delta_nll": 0.9672927856445312, + "max_kl": 4.192455291748047, + "mean_kl": 1.0962470769882202, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 1.8299256563186646, + "code_tokens": 24, + "cvar95_kl": 2.0445573329925537, + "delta_nll": 0.42162251472473145, + "max_kl": 2.141908645629883, + "mean_kl": 0.40619850158691406, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.0432076454162598, + "code_tokens": 78, + "cvar95_kl": 3.181962013244629, + "delta_nll": 0.8812477588653564, + "max_kl": 3.667109251022339, + "mean_kl": 0.8838945031166077, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.5897436141967773 + }, + { + "candidate_nll": 1.4589956998825073, + "code_tokens": 33, + "cvar95_kl": 5.343605995178223, + "delta_nll": 0.8284386396408081, + "max_kl": 7.80642032623291, + "mean_kl": 0.7979444861412048, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 2.252589225769043, + "code_tokens": 32, + "cvar95_kl": 3.1679248809814453, + "delta_nll": 1.136919617652893, + "max_kl": 3.732572555541992, + "mean_kl": 1.0272454023361206, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 2.004063606262207, + "code_tokens": 77, + "cvar95_kl": 3.696166753768921, + "delta_nll": 1.0153138637542725, + "max_kl": 5.7238311767578125, + "mean_kl": 0.8602582216262817, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.649350643157959 + }, + { + "candidate_nll": 0.8638460040092468, + "code_tokens": 222, + "cvar95_kl": 2.752610445022583, + "delta_nll": 0.4415954053401947, + "max_kl": 4.615993976593018, + "mean_kl": 0.4772477149963379, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.815315306186676 + }, + { + "candidate_nll": 2.3223893642425537, + "code_tokens": 86, + "cvar95_kl": 4.934406757354736, + "delta_nll": 1.2737945318222046, + "max_kl": 6.604762077331543, + "mean_kl": 1.308000922203064, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.6395348906517029 + }, + { + "candidate_nll": 0.8021089434623718, + "code_tokens": 31, + "cvar95_kl": 2.7074379920959473, + "delta_nll": 0.4680594503879547, + "max_kl": 3.5851571559906006, + "mean_kl": 0.47544974088668823, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.7419354915618896 + }, + { + "candidate_nll": 1.103749394416809, + "code_tokens": 34, + "cvar95_kl": 3.2629623413085938, + "delta_nll": 0.5921493172645569, + "max_kl": 4.271722316741943, + "mean_kl": 0.5829216241836548, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.1370041370391846, + "code_tokens": 38, + "cvar95_kl": 3.715728998184204, + "delta_nll": 0.7495437860488892, + "max_kl": 4.9643402099609375, + "mean_kl": 0.724778950214386, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 2.0550122261047363, + "code_tokens": 93, + "cvar95_kl": 3.579421281814575, + "delta_nll": 1.0919675827026367, + "max_kl": 4.614491939544678, + "mean_kl": 0.9615810513496399, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.838106393814087, + "code_tokens": 70, + "cvar95_kl": 4.960568428039551, + "delta_nll": 1.0595006942749023, + "max_kl": 5.538738250732422, + "mean_kl": 1.0343478918075562, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.691083550453186, + "code_tokens": 67, + "cvar95_kl": 3.2460126876831055, + "delta_nll": 0.8805915117263794, + "max_kl": 3.755563735961914, + "mean_kl": 0.7991440296173096, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.6865671873092651 + }, + { + "candidate_nll": 2.3151888847351074, + "code_tokens": 66, + "cvar95_kl": 4.5163254737854, + "delta_nll": 1.0290615558624268, + "max_kl": 6.432679176330566, + "mean_kl": 0.9873060584068298, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.5757575631141663 + }, + { + "candidate_nll": 1.7566393613815308, + "code_tokens": 34, + "cvar95_kl": 3.181601047515869, + "delta_nll": 0.7443451881408691, + "max_kl": 3.2493536472320557, + "mean_kl": 0.6402506232261658, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.6176470518112183 + }, + { + "candidate_nll": 1.8556638956069946, + "code_tokens": 64, + "cvar95_kl": 3.943228244781494, + "delta_nll": 0.9456385374069214, + "max_kl": 4.87177848815918, + "mean_kl": 0.9277783036231995, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.609375 + }, + { + "candidate_nll": 2.1626455783843994, + "code_tokens": 56, + "cvar95_kl": 6.1824822425842285, + "delta_nll": 0.8522900342941284, + "max_kl": 7.676549911499023, + "mean_kl": 0.9068112373352051, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.6861999034881592, + "code_tokens": 21, + "cvar95_kl": 1.332353115081787, + "delta_nll": 0.5611175298690796, + "max_kl": 1.4406205415725708, + "mean_kl": 0.448677122592926, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.5580421686172485, + "code_tokens": 60, + "cvar95_kl": 4.17443323135376, + "delta_nll": 0.8927018046379089, + "max_kl": 5.071746826171875, + "mean_kl": 0.9052857160568237, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.0340769290924072, + "code_tokens": 29, + "cvar95_kl": 3.3199303150177, + "delta_nll": 0.6803009510040283, + "max_kl": 3.7341411113739014, + "mean_kl": 0.6923015117645264, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.3338102102279663, + "code_tokens": 34, + "cvar95_kl": 3.5734474658966064, + "delta_nll": 0.6117653250694275, + "max_kl": 4.77994441986084, + "mean_kl": 0.6395746469497681, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 2.3149831295013428, + "code_tokens": 88, + "cvar95_kl": 5.636975288391113, + "delta_nll": 1.3958756923675537, + "max_kl": 10.449828147888184, + "mean_kl": 1.3487963676452637, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.5568181872367859 + }, + { + "candidate_nll": 1.212145447731018, + "code_tokens": 17, + "cvar95_kl": 2.9401886463165283, + "delta_nll": 0.6689218282699585, + "max_kl": 2.9401886463165283, + "mean_kl": 0.5465613007545471, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.4588000774383545, + "code_tokens": 35, + "cvar95_kl": 1.3194866180419922, + "delta_nll": 0.5682529211044312, + "max_kl": 1.352709412574768, + "mean_kl": 0.5214791893959045, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.357059359550476, + "code_tokens": 48, + "cvar95_kl": 1.8637690544128418, + "delta_nll": 0.46728283166885376, + "max_kl": 1.9415417909622192, + "mean_kl": 0.45307672023773193, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.7291666865348816 + }, + { + "candidate_nll": 1.9790363311767578, + "code_tokens": 130, + "cvar95_kl": 4.627747535705566, + "delta_nll": 0.9887639284133911, + "max_kl": 6.140337944030762, + "mean_kl": 0.9725895524024963, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.607692301273346 + }, + { + "candidate_nll": 3.0495872497558594, + "code_tokens": 65, + "cvar95_kl": 7.1827921867370605, + "delta_nll": 1.9546477794647217, + "max_kl": 8.93212890625, + "mean_kl": 1.8015974760055542, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.4153846204280853 + }, + { + "candidate_nll": 2.235452175140381, + "code_tokens": 145, + "cvar95_kl": 5.031227111816406, + "delta_nll": 1.3604371547698975, + "max_kl": 6.25171422958374, + "mean_kl": 1.239142656326294, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.5586206912994385 + }, + { + "candidate_nll": 1.0810580253601074, + "code_tokens": 80, + "cvar95_kl": 4.0204315185546875, + "delta_nll": 0.36535871028900146, + "max_kl": 5.251855850219727, + "mean_kl": 0.5212334990501404, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.8374999761581421 + }, + { + "candidate_nll": 1.7141131162643433, + "code_tokens": 70, + "cvar95_kl": 5.562346935272217, + "delta_nll": 1.1319763660430908, + "max_kl": 7.40985107421875, + "mean_kl": 1.2132337093353271, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.9817008972167969, + "code_tokens": 25, + "cvar95_kl": 2.557723045349121, + "delta_nll": 0.4934118986129761, + "max_kl": 2.932297945022583, + "mean_kl": 0.5099629163742065, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 2.5396459102630615, + "code_tokens": 50, + "cvar95_kl": 6.171702861785889, + "delta_nll": 0.8852624893188477, + "max_kl": 7.636078357696533, + "mean_kl": 1.166253924369812, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.46000000834465027 + }, + { + "candidate_nll": 1.2731434106826782, + "code_tokens": 48, + "cvar95_kl": 2.3191959857940674, + "delta_nll": 0.47510045766830444, + "max_kl": 2.891127586364746, + "mean_kl": 0.4860738515853882, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.7708333134651184 + }, + { + "candidate_nll": 1.4226531982421875, + "code_tokens": 104, + "cvar95_kl": 6.255232334136963, + "delta_nll": 1.0785808563232422, + "max_kl": 8.623175621032715, + "mean_kl": 1.032798171043396, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.7211538553237915 + }, + { + "candidate_nll": 1.9666929244995117, + "code_tokens": 68, + "cvar95_kl": 3.412477493286133, + "delta_nll": 0.976698637008667, + "max_kl": 4.074804306030273, + "mean_kl": 0.8548634648323059, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 1.494521975517273, + "code_tokens": 145, + "cvar95_kl": 3.5522561073303223, + "delta_nll": 0.7396098375320435, + "max_kl": 4.686226844787598, + "mean_kl": 0.7456536889076233, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.7172414064407349 + }, + { + "candidate_nll": 1.237764596939087, + "code_tokens": 41, + "cvar95_kl": 2.923518180847168, + "delta_nll": 0.3928768038749695, + "max_kl": 4.6337385177612305, + "mean_kl": 0.5110464692115784, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.7804877758026123 + }, + { + "candidate_nll": 1.5545239448547363, + "code_tokens": 53, + "cvar95_kl": 3.6254470348358154, + "delta_nll": 0.7934016585350037, + "max_kl": 3.9424192905426025, + "mean_kl": 0.7189407348632812, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.7169811129570007 + }, + { + "candidate_nll": 1.6420966386795044, + "code_tokens": 58, + "cvar95_kl": 3.273249387741089, + "delta_nll": 0.8498668670654297, + "max_kl": 3.664853096008301, + "mean_kl": 0.7556498050689697, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.6896551847457886 + }, + { + "candidate_nll": 2.3292598724365234, + "code_tokens": 24, + "cvar95_kl": 4.473161220550537, + "delta_nll": 0.5971997976303101, + "max_kl": 4.638923168182373, + "mean_kl": 0.7550384998321533, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.207904100418091, + "code_tokens": 89, + "cvar95_kl": 5.26861572265625, + "delta_nll": 1.216050148010254, + "max_kl": 7.20202112197876, + "mean_kl": 1.2106013298034668, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.6067415475845337 + }, + { + "candidate_nll": 1.6184386014938354, + "code_tokens": 101, + "cvar95_kl": 5.860593795776367, + "delta_nll": 1.1333624124526978, + "max_kl": 8.293519973754883, + "mean_kl": 1.047418475151062, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.6732673048973083 + }, + { + "candidate_nll": 2.233825206756592, + "code_tokens": 33, + "cvar95_kl": 5.080671310424805, + "delta_nll": 0.776267409324646, + "max_kl": 5.697253704071045, + "mean_kl": 0.8108587265014648, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.7304071187973022, + "code_tokens": 50, + "cvar95_kl": 4.312148571014404, + "delta_nll": 0.683741569519043, + "max_kl": 5.763941287994385, + "mean_kl": 0.7180960178375244, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.2879302501678467, + "code_tokens": 58, + "cvar95_kl": 2.9492263793945312, + "delta_nll": 0.879177451133728, + "max_kl": 3.6061527729034424, + "mean_kl": 0.677055299282074, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.6034482717514038 + }, + { + "candidate_nll": 1.9441567659378052, + "code_tokens": 62, + "cvar95_kl": 4.397705554962158, + "delta_nll": 0.9649978280067444, + "max_kl": 5.614462375640869, + "mean_kl": 0.840583086013794, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.2633821964263916, + "code_tokens": 42, + "cvar95_kl": 2.704160690307617, + "delta_nll": 0.5716922283172607, + "max_kl": 3.2994444370269775, + "mean_kl": 0.6693840622901917, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8394999504089355, + "code_tokens": 15, + "cvar95_kl": 3.4482526779174805, + "delta_nll": 0.9147589802742004, + "max_kl": 3.4482526779174805, + "mean_kl": 0.7181757092475891, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.791417121887207, + "code_tokens": 57, + "cvar95_kl": 5.377686977386475, + "delta_nll": 1.0526349544525146, + "max_kl": 6.553092956542969, + "mean_kl": 0.9371963739395142, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.6315789222717285 + }, + { + "candidate_nll": 2.2950026988983154, + "code_tokens": 58, + "cvar95_kl": 4.248537063598633, + "delta_nll": 1.2514667510986328, + "max_kl": 4.306375980377197, + "mean_kl": 1.1180880069732666, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.5862069129943848 + }, + { + "candidate_nll": 1.9169104099273682, + "code_tokens": 45, + "cvar95_kl": 5.500059604644775, + "delta_nll": 1.0449392795562744, + "max_kl": 7.531994819641113, + "mean_kl": 1.091720700263977, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.644444465637207 + }, + { + "candidate_nll": 2.7144839763641357, + "code_tokens": 32, + "cvar95_kl": 3.588292360305786, + "delta_nll": 0.6852536201477051, + "max_kl": 5.384130001068115, + "mean_kl": 0.9199109077453613, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 2.0151829719543457, + "code_tokens": 39, + "cvar95_kl": 3.4913129806518555, + "delta_nll": 0.6945165395736694, + "max_kl": 3.6355254650115967, + "mean_kl": 0.7591186761856079, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.5903136730194092, + "code_tokens": 136, + "cvar95_kl": 4.787999629974365, + "delta_nll": 1.1795523166656494, + "max_kl": 5.622316837310791, + "mean_kl": 1.1059013605117798, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.595588207244873 + }, + { + "candidate_nll": 1.8570283651351929, + "code_tokens": 50, + "cvar95_kl": 2.906562089920044, + "delta_nll": 0.6362717151641846, + "max_kl": 3.2378296852111816, + "mean_kl": 0.6789836883544922, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.714455008506775, + "code_tokens": 34, + "cvar95_kl": 3.5784621238708496, + "delta_nll": 0.8485392928123474, + "max_kl": 3.6134321689605713, + "mean_kl": 0.8952341079711914, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.7352941036224365 + }, + { + "candidate_nll": 1.6534298658370972, + "code_tokens": 44, + "cvar95_kl": 4.005340576171875, + "delta_nll": 0.4746396541595459, + "max_kl": 8.045111656188965, + "mean_kl": 0.631790280342102, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.866131067276001, + "code_tokens": 36, + "cvar95_kl": 1.3476462364196777, + "delta_nll": 0.4327092170715332, + "max_kl": 1.6028351783752441, + "mean_kl": 0.44165897369384766, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 1.4961055517196655, + "code_tokens": 46, + "cvar95_kl": 3.403928756713867, + "delta_nll": 0.6929483413696289, + "max_kl": 4.585171699523926, + "mean_kl": 0.7065553069114685, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.760869562625885 + }, + { + "candidate_nll": 2.2846481800079346, + "code_tokens": 48, + "cvar95_kl": 4.934628009796143, + "delta_nll": 1.1043082475662231, + "max_kl": 6.056484222412109, + "mean_kl": 0.9168172478675842, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.6041666865348816 + }, + { + "candidate_nll": 1.6638339757919312, + "code_tokens": 25, + "cvar95_kl": 3.2705488204956055, + "delta_nll": 0.7684229016304016, + "max_kl": 4.139795780181885, + "mean_kl": 0.5923088192939758, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.394541025161743, + "code_tokens": 69, + "cvar95_kl": 6.7038679122924805, + "delta_nll": 0.8902620077133179, + "max_kl": 10.661364555358887, + "mean_kl": 1.2651560306549072, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.49275362491607666 + }, + { + "candidate_nll": 1.3562698364257812, + "code_tokens": 60, + "cvar95_kl": 2.6273107528686523, + "delta_nll": 0.3908121585845947, + "max_kl": 3.030283212661743, + "mean_kl": 0.5341432094573975, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.8166666626930237 + }, + { + "candidate_nll": 1.6419490575790405, + "code_tokens": 49, + "cvar95_kl": 4.407739162445068, + "delta_nll": 0.7274715900421143, + "max_kl": 6.119176387786865, + "mean_kl": 0.7175212502479553, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.6938775777816772 + }, + { + "candidate_nll": 1.965149998664856, + "code_tokens": 44, + "cvar95_kl": 1.6601191759109497, + "delta_nll": 0.2646217346191406, + "max_kl": 1.9823342561721802, + "mean_kl": 0.4484603703022003, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 2.2091293334960938, + "code_tokens": 81, + "cvar95_kl": 4.895567893981934, + "delta_nll": 1.0637151002883911, + "max_kl": 6.820925712585449, + "mean_kl": 1.0465245246887207, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.7598247528076172, + "code_tokens": 26, + "cvar95_kl": 2.982357978820801, + "delta_nll": 0.32308948040008545, + "max_kl": 3.1492247581481934, + "mean_kl": 0.6875041723251343, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.3608142137527466, + "code_tokens": 32, + "cvar95_kl": 3.4163453578948975, + "delta_nll": 0.4808931350708008, + "max_kl": 4.928884506225586, + "mean_kl": 0.6390304565429688, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.78125 + }, + { + "candidate_nll": 1.8975330591201782, + "code_tokens": 48, + "cvar95_kl": 3.5240275859832764, + "delta_nll": 0.9497911334037781, + "max_kl": 4.060540676116943, + "mean_kl": 0.7865393757820129, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.978298306465149, + "code_tokens": 180, + "cvar95_kl": 5.0296783447265625, + "delta_nll": 1.2789771556854248, + "max_kl": 7.069170951843262, + "mean_kl": 1.2029547691345215, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.5777778029441833 + }, + { + "candidate_nll": 1.5074663162231445, + "code_tokens": 38, + "cvar95_kl": 3.4338808059692383, + "delta_nll": 0.7857047319412231, + "max_kl": 3.6890652179718018, + "mean_kl": 0.7474092841148376, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.5789473652839661 + }, + { + "candidate_nll": 1.934070348739624, + "code_tokens": 90, + "cvar95_kl": 6.209603309631348, + "delta_nll": 1.2390086650848389, + "max_kl": 8.92114543914795, + "mean_kl": 1.2147440910339355, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.6333333253860474 + }, + { + "candidate_nll": 1.279183268547058, + "code_tokens": 44, + "cvar95_kl": 4.104951858520508, + "delta_nll": 0.7045531868934631, + "max_kl": 4.532039642333984, + "mean_kl": 0.8774381875991821, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.6818181872367859 + }, + { + "candidate_nll": 1.5062382221221924, + "code_tokens": 60, + "cvar95_kl": 4.598969459533691, + "delta_nll": 0.7623380422592163, + "max_kl": 7.09505558013916, + "mean_kl": 0.7661399841308594, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.893372654914856, + "code_tokens": 69, + "cvar95_kl": 4.813685894012451, + "delta_nll": 0.89057457447052, + "max_kl": 7.786147594451904, + "mean_kl": 0.776037335395813, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.687326192855835, + "code_tokens": 71, + "cvar95_kl": 5.39392614364624, + "delta_nll": 1.1697643995285034, + "max_kl": 6.970792293548584, + "mean_kl": 1.147890567779541, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.6197183132171631 + }, + { + "candidate_nll": 1.697794795036316, + "code_tokens": 22, + "cvar95_kl": 2.194110870361328, + "delta_nll": 0.45512163639068604, + "max_kl": 2.9233522415161133, + "mean_kl": 0.43486425280570984, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.3446471691131592, + "code_tokens": 215, + "cvar95_kl": 3.96205997467041, + "delta_nll": 0.7908676862716675, + "max_kl": 6.131277561187744, + "mean_kl": 0.8570724129676819, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.6558139324188232 + }, + { + "candidate_nll": 1.5275791883468628, + "code_tokens": 26, + "cvar95_kl": 1.6029101610183716, + "delta_nll": 0.42478954792022705, + "max_kl": 1.9135972261428833, + "mean_kl": 0.39831897616386414, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.8846153616905212 + }, + { + "candidate_nll": 1.753525972366333, + "code_tokens": 33, + "cvar95_kl": 4.487386226654053, + "delta_nll": 0.8852792978286743, + "max_kl": 5.0195112228393555, + "mean_kl": 0.8944844603538513, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.0879502296447754, + "code_tokens": 18, + "cvar95_kl": 0.6777887940406799, + "delta_nll": 0.15180212259292603, + "max_kl": 0.6777887940406799, + "mean_kl": 0.24614855647087097, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 2.2590761184692383, + "code_tokens": 71, + "cvar95_kl": 5.509476661682129, + "delta_nll": 0.9790143966674805, + "max_kl": 6.910741329193115, + "mean_kl": 1.1907845735549927, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.6056337952613831 + }, + { + "candidate_nll": 1.4496458768844604, + "code_tokens": 107, + "cvar95_kl": 3.9019651412963867, + "delta_nll": 0.6682444214820862, + "max_kl": 4.687696933746338, + "mean_kl": 0.6866754293441772, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.7196261882781982 + }, + { + "candidate_nll": 2.8429195880889893, + "code_tokens": 40, + "cvar95_kl": 5.153518199920654, + "delta_nll": 0.7878949642181396, + "max_kl": 5.5966010093688965, + "mean_kl": 0.9146377444267273, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 2.001621723175049, + "code_tokens": 32, + "cvar95_kl": 3.4253008365631104, + "delta_nll": 0.6373248100280762, + "max_kl": 3.5417118072509766, + "mean_kl": 0.805347740650177, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.71875 + }, + { + "candidate_nll": 1.629675030708313, + "code_tokens": 130, + "cvar95_kl": 4.378066062927246, + "delta_nll": 0.9304212927818298, + "max_kl": 7.064347267150879, + "mean_kl": 0.9283219575881958, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.6461538672447205 + }, + { + "candidate_nll": 2.3776659965515137, + "code_tokens": 27, + "cvar95_kl": 4.762880802154541, + "delta_nll": 0.9062442779541016, + "max_kl": 6.56791353225708, + "mean_kl": 0.8794907331466675, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.56169855594635, + "code_tokens": 42, + "cvar95_kl": 3.5952131748199463, + "delta_nll": 0.5887582302093506, + "max_kl": 5.95373010635376, + "mean_kl": 0.6388250589370728, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 1.854259729385376, + "code_tokens": 30, + "cvar95_kl": 4.0978007316589355, + "delta_nll": 1.1087465286254883, + "max_kl": 5.7968244552612305, + "mean_kl": 0.8492130041122437, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 2.81780743598938, + "code_tokens": 58, + "cvar95_kl": 5.652854919433594, + "delta_nll": 1.3985745906829834, + "max_kl": 6.680981636047363, + "mean_kl": 1.2162731885910034, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.517241358757019 + }, + { + "candidate_nll": 1.8249861001968384, + "code_tokens": 34, + "cvar95_kl": 3.721623182296753, + "delta_nll": 0.5947588682174683, + "max_kl": 3.7677230834960938, + "mean_kl": 0.6241449117660522, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 1.677505373954773, + "code_tokens": 208, + "cvar95_kl": 4.425317287445068, + "delta_nll": 0.9760428667068481, + "max_kl": 9.65798282623291, + "mean_kl": 0.9900054335594177, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.6394230723381042 + }, + { + "candidate_nll": 1.5022534132003784, + "code_tokens": 83, + "cvar95_kl": 2.515969753265381, + "delta_nll": 0.603347659111023, + "max_kl": 3.6074225902557373, + "mean_kl": 0.7021126747131348, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.6265060305595398 + }, + { + "candidate_nll": 1.918843388557434, + "code_tokens": 170, + "cvar95_kl": 6.531036376953125, + "delta_nll": 1.3962876796722412, + "max_kl": 10.504840850830078, + "mean_kl": 1.402720332145691, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.5588235259056091 + }, + { + "candidate_nll": 1.5509626865386963, + "code_tokens": 40, + "cvar95_kl": 6.642059803009033, + "delta_nll": 0.7695421576499939, + "max_kl": 7.107378959655762, + "mean_kl": 0.8668548464775085, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.995086669921875, + "code_tokens": 31, + "cvar95_kl": 4.016458511352539, + "delta_nll": 0.7483770847320557, + "max_kl": 5.105479717254639, + "mean_kl": 0.8075739741325378, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.523214340209961, + "code_tokens": 42, + "cvar95_kl": 3.702531099319458, + "delta_nll": 0.7284443378448486, + "max_kl": 6.634225845336914, + "mean_kl": 0.7620762586593628, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.6904761791229248 + }, + { + "candidate_nll": 2.3896312713623047, + "code_tokens": 27, + "cvar95_kl": 3.555924415588379, + "delta_nll": 0.6205652952194214, + "max_kl": 4.269290924072266, + "mean_kl": 0.6885077357292175, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 2.3434765338897705, + "code_tokens": 25, + "cvar95_kl": 2.072944164276123, + "delta_nll": 0.14366602897644043, + "max_kl": 2.18044376373291, + "mean_kl": 0.6416712403297424, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.2202625274658203, + "code_tokens": 40, + "cvar95_kl": 4.748190879821777, + "delta_nll": 0.6518760919570923, + "max_kl": 4.939370155334473, + "mean_kl": 0.638741672039032, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.204289197921753, + "code_tokens": 24, + "cvar95_kl": 2.964901924133301, + "delta_nll": 0.819934606552124, + "max_kl": 3.553415298461914, + "mean_kl": 0.6125619411468506, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.3745003938674927, + "code_tokens": 28, + "cvar95_kl": 3.0922820568084717, + "delta_nll": 0.4917554259300232, + "max_kl": 3.1954879760742188, + "mean_kl": 0.5234538316726685, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.25815749168396, + "code_tokens": 71, + "cvar95_kl": 4.260236740112305, + "delta_nll": 0.7792929410934448, + "max_kl": 5.760156154632568, + "mean_kl": 0.7824870347976685, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.7323943376541138 + }, + { + "candidate_nll": 1.0163840055465698, + "code_tokens": 24, + "cvar95_kl": 1.5815180540084839, + "delta_nll": 0.4870232343673706, + "max_kl": 2.1410579681396484, + "mean_kl": 0.335100382566452, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 2.3539557456970215, + "code_tokens": 34, + "cvar95_kl": 7.563270568847656, + "delta_nll": 1.0823746919631958, + "max_kl": 8.60350513458252, + "mean_kl": 1.3974666595458984, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 2.053457736968994, + "code_tokens": 64, + "cvar95_kl": 7.214577674865723, + "delta_nll": 0.9341638088226318, + "max_kl": 9.157510757446289, + "mean_kl": 1.2422655820846558, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.671875 + }, + { + "candidate_nll": 1.3067593574523926, + "code_tokens": 45, + "cvar95_kl": 3.69111704826355, + "delta_nll": 0.6001680493354797, + "max_kl": 4.184877872467041, + "mean_kl": 0.6168801188468933, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.6888889074325562 + }, + { + "candidate_nll": 2.2819278240203857, + "code_tokens": 48, + "cvar95_kl": 5.935068607330322, + "delta_nll": 1.0832237005233765, + "max_kl": 6.314632892608643, + "mean_kl": 1.0914994478225708, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.0842819213867188, + "code_tokens": 21, + "cvar95_kl": 1.64229416847229, + "delta_nll": 0.38408076763153076, + "max_kl": 2.218334674835205, + "mean_kl": 0.37952980399131775, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 1.6648787260055542, + "code_tokens": 24, + "cvar95_kl": 5.616354942321777, + "delta_nll": 0.9104645848274231, + "max_kl": 6.097921371459961, + "mean_kl": 1.0816696882247925, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.5383594036102295, + "code_tokens": 63, + "cvar95_kl": 5.217547416687012, + "delta_nll": 0.9664320945739746, + "max_kl": 8.618024826049805, + "mean_kl": 1.0535266399383545, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.4116727113723755, + "code_tokens": 22, + "cvar95_kl": 2.8223984241485596, + "delta_nll": 0.13291358947753906, + "max_kl": 4.028323650360107, + "mean_kl": 0.5219434499740601, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.5755139589309692, + "code_tokens": 25, + "cvar95_kl": 2.665889263153076, + "delta_nll": 0.44685161113739014, + "max_kl": 3.6610512733459473, + "mean_kl": 0.7221518754959106, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.4314682483673096, + "code_tokens": 21, + "cvar95_kl": 1.3364009857177734, + "delta_nll": 0.4711425304412842, + "max_kl": 1.5617436170578003, + "mean_kl": 0.4020133316516876, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.9894105195999146, + "code_tokens": 24, + "cvar95_kl": 2.6003944873809814, + "delta_nll": 0.523078441619873, + "max_kl": 3.1083710193634033, + "mean_kl": 0.5830727219581604, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.7916666865348816 + }, + { + "candidate_nll": 2.9030909538269043, + "code_tokens": 15, + "cvar95_kl": 6.340234756469727, + "delta_nll": 1.1096031665802002, + "max_kl": 6.340234756469727, + "mean_kl": 0.8005049228668213, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.6358495950698853, + "code_tokens": 59, + "cvar95_kl": 3.1014702320098877, + "delta_nll": 0.8549931645393372, + "max_kl": 3.5077762603759766, + "mean_kl": 0.7149861454963684, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.6610169410705566 + }, + { + "candidate_nll": 1.6072207689285278, + "code_tokens": 54, + "cvar95_kl": 2.798616647720337, + "delta_nll": 0.6731025576591492, + "max_kl": 2.838393449783325, + "mean_kl": 0.8256797194480896, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.5925925970077515 + }, + { + "candidate_nll": 2.4017252922058105, + "code_tokens": 28, + "cvar95_kl": 2.305636405944824, + "delta_nll": 0.4498661756515503, + "max_kl": 2.44950008392334, + "mean_kl": 0.5075047016143799, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.8928571343421936 + }, + { + "candidate_nll": 1.6199557781219482, + "code_tokens": 46, + "cvar95_kl": 3.8294360637664795, + "delta_nll": 0.5352298021316528, + "max_kl": 4.08341121673584, + "mean_kl": 0.826393723487854, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 1.8967223167419434, + "code_tokens": 27, + "cvar95_kl": 3.379631996154785, + "delta_nll": 0.38189995288848877, + "max_kl": 4.300095081329346, + "mean_kl": 0.6284482479095459, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.7777777910232544 + }, + { + "candidate_nll": 2.391467809677124, + "code_tokens": 97, + "cvar95_kl": 4.3888115882873535, + "delta_nll": 1.1340371370315552, + "max_kl": 5.161090850830078, + "mean_kl": 1.057094931602478, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.6082473993301392 + }, + { + "candidate_nll": 1.8346141576766968, + "code_tokens": 24, + "cvar95_kl": 2.866438388824463, + "delta_nll": 1.0323817729949951, + "max_kl": 3.2612571716308594, + "mean_kl": 0.6640434861183167, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.491055727005005, + "code_tokens": 30, + "cvar95_kl": 5.694753646850586, + "delta_nll": 1.2259070873260498, + "max_kl": 6.15928840637207, + "mean_kl": 1.0961772203445435, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.8979215621948242, + "code_tokens": 38, + "cvar95_kl": 3.3727941513061523, + "delta_nll": 0.4648517370223999, + "max_kl": 4.0166754722595215, + "mean_kl": 0.6539572477340698, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.7631579041481018 + }, + { + "candidate_nll": 1.8850573301315308, + "code_tokens": 21, + "cvar95_kl": 1.8053181171417236, + "delta_nll": 0.3972783088684082, + "max_kl": 1.8481935262680054, + "mean_kl": 0.5291749835014343, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.369319200515747, + "code_tokens": 65, + "cvar95_kl": 4.046999931335449, + "delta_nll": 0.8323574066162109, + "max_kl": 6.070259094238281, + "mean_kl": 0.7907069325447083, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.6307692527770996 + }, + { + "candidate_nll": 0.9778472781181335, + "code_tokens": 27, + "cvar95_kl": 1.5884100198745728, + "delta_nll": 0.19332098960876465, + "max_kl": 1.6270277500152588, + "mean_kl": 0.32111626863479614, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.8148148059844971 + }, + { + "candidate_nll": 1.4316010475158691, + "code_tokens": 35, + "cvar95_kl": 5.328105449676514, + "delta_nll": 0.7077849507331848, + "max_kl": 7.468367576599121, + "mean_kl": 0.8774805068969727, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.735164761543274, + "code_tokens": 75, + "cvar95_kl": 5.0317583084106445, + "delta_nll": 0.8106406331062317, + "max_kl": 6.880044460296631, + "mean_kl": 1.0299311876296997, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.6266666650772095 + }, + { + "candidate_nll": 2.3429009914398193, + "code_tokens": 25, + "cvar95_kl": 3.8272459506988525, + "delta_nll": 0.7011654376983643, + "max_kl": 4.6284260749816895, + "mean_kl": 1.0043768882751465, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.5777878761291504, + "code_tokens": 52, + "cvar95_kl": 3.069300889968872, + "delta_nll": 0.6537989377975464, + "max_kl": 4.233497142791748, + "mean_kl": 0.6401729583740234, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.9131393432617188, + "code_tokens": 110, + "cvar95_kl": 5.498794078826904, + "delta_nll": 1.3024792671203613, + "max_kl": 6.290473937988281, + "mean_kl": 1.3385440111160278, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.9648257493972778, + "code_tokens": 25, + "cvar95_kl": 3.739701509475708, + "delta_nll": 0.7320038080215454, + "max_kl": 3.987820625305176, + "mean_kl": 0.8204809427261353, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.6489132642745972, + "code_tokens": 56, + "cvar95_kl": 2.87494158744812, + "delta_nll": 0.6319272518157959, + "max_kl": 3.102954149246216, + "mean_kl": 0.7540364861488342, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.6785714030265808 + }, + { + "candidate_nll": 2.811880350112915, + "code_tokens": 108, + "cvar95_kl": 4.006493091583252, + "delta_nll": 1.2580320835113525, + "max_kl": 4.893801689147949, + "mean_kl": 1.044417381286621, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 1.940338134765625, + "code_tokens": 47, + "cvar95_kl": 4.2197370529174805, + "delta_nll": 0.5685982704162598, + "max_kl": 4.915261745452881, + "mean_kl": 0.8544566631317139, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.6595744490623474 + }, + { + "candidate_nll": 2.126844644546509, + "code_tokens": 102, + "cvar95_kl": 5.176737308502197, + "delta_nll": 1.1284360885620117, + "max_kl": 6.814855575561523, + "mean_kl": 1.206226110458374, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.5588235259056091 + }, + { + "candidate_nll": 2.018544912338257, + "code_tokens": 18, + "cvar95_kl": 1.7832688093185425, + "delta_nll": 0.39291930198669434, + "max_kl": 1.7832688093185425, + "mean_kl": 0.5559535026550293, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.039269208908081, + "code_tokens": 28, + "cvar95_kl": 4.464201927185059, + "delta_nll": 0.9396049976348877, + "max_kl": 6.379303455352783, + "mean_kl": 0.7926393151283264, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 2.211474895477295, + "code_tokens": 30, + "cvar95_kl": 2.1380836963653564, + "delta_nll": 0.6680740118026733, + "max_kl": 2.141587734222412, + "mean_kl": 0.5180180668830872, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.7666666507720947 + }, + { + "candidate_nll": 1.4775985479354858, + "code_tokens": 48, + "cvar95_kl": 4.732845783233643, + "delta_nll": 1.0024815797805786, + "max_kl": 6.091846466064453, + "mean_kl": 1.0739169120788574, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 1.3423503637313843, + "code_tokens": 59, + "cvar95_kl": 5.409399509429932, + "delta_nll": 0.8053353428840637, + "max_kl": 5.998922348022461, + "mean_kl": 0.8272143006324768, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.7796609997749329 + }, + { + "candidate_nll": 1.9327728748321533, + "code_tokens": 102, + "cvar95_kl": 3.5502431392669678, + "delta_nll": 0.8905789852142334, + "max_kl": 4.954209804534912, + "mean_kl": 0.8284209966659546, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 1.5817111730575562, + "code_tokens": 36, + "cvar95_kl": 2.9861900806427, + "delta_nll": 0.8413455486297607, + "max_kl": 3.0498547554016113, + "mean_kl": 0.7214430570602417, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.8784481287002563, + "code_tokens": 51, + "cvar95_kl": 4.3032145500183105, + "delta_nll": 0.9895166158676147, + "max_kl": 5.679299831390381, + "mean_kl": 0.912172794342041, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 2.485616445541382, + "code_tokens": 62, + "cvar95_kl": 5.717792987823486, + "delta_nll": 1.2399377822875977, + "max_kl": 6.415706157684326, + "mean_kl": 1.3577053546905518, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.8079123497009277, + "code_tokens": 39, + "cvar95_kl": 6.829513072967529, + "delta_nll": 1.2316303253173828, + "max_kl": 6.982604503631592, + "mean_kl": 1.3465116024017334, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.6153846383094788 + }, + { + "candidate_nll": 1.502866506576538, + "code_tokens": 76, + "cvar95_kl": 3.8723974227905273, + "delta_nll": 0.7631821632385254, + "max_kl": 4.565370082855225, + "mean_kl": 0.6854333877563477, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.6973684430122375 + }, + { + "candidate_nll": 2.170872926712036, + "code_tokens": 58, + "cvar95_kl": 3.5489377975463867, + "delta_nll": 0.6662218570709229, + "max_kl": 3.6588220596313477, + "mean_kl": 0.6807057857513428, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 2.3633079528808594, + "code_tokens": 93, + "cvar95_kl": 4.347151756286621, + "delta_nll": 1.2359126806259155, + "max_kl": 5.3135762214660645, + "mean_kl": 1.0022687911987305, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.5806451439857483 + }, + { + "candidate_nll": 2.041283369064331, + "code_tokens": 45, + "cvar95_kl": 4.600948333740234, + "delta_nll": 0.6860378980636597, + "max_kl": 8.34201431274414, + "mean_kl": 0.718752384185791, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.481890082359314, + "code_tokens": 209, + "cvar95_kl": 4.503603458404541, + "delta_nll": 0.9249146580696106, + "max_kl": 5.918145656585693, + "mean_kl": 1.007394552230835, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.6172248721122742 + }, + { + "candidate_nll": 1.7768257856369019, + "code_tokens": 46, + "cvar95_kl": 3.2651360034942627, + "delta_nll": 0.8083581328392029, + "max_kl": 4.387097358703613, + "mean_kl": 0.6620118021965027, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.804347813129425 + }, + { + "candidate_nll": 1.3745371103286743, + "code_tokens": 39, + "cvar95_kl": 4.754983425140381, + "delta_nll": 0.6537501215934753, + "max_kl": 6.58702278137207, + "mean_kl": 0.6235072016716003, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.3445266485214233, + "code_tokens": 25, + "cvar95_kl": 2.5755693912506104, + "delta_nll": 0.4028312563896179, + "max_kl": 4.240860939025879, + "mean_kl": 0.3520977795124054, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.8399999737739563 + }, + { + "candidate_nll": 1.7583348751068115, + "code_tokens": 28, + "cvar95_kl": 1.6424822807312012, + "delta_nll": 0.5831385850906372, + "max_kl": 1.7156944274902344, + "mean_kl": 0.5225332379341125, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 2.192363977432251, + "code_tokens": 84, + "cvar95_kl": 4.257025718688965, + "delta_nll": 0.9384413957595825, + "max_kl": 5.538780689239502, + "mean_kl": 0.8417872190475464, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.6309523582458496 + }, + { + "candidate_nll": 1.3636388778686523, + "code_tokens": 34, + "cvar95_kl": 2.069392442703247, + "delta_nll": 0.5513585805892944, + "max_kl": 2.3290133476257324, + "mean_kl": 0.4581373929977417, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 1.8156776428222656, + "code_tokens": 28, + "cvar95_kl": 2.298698663711548, + "delta_nll": 0.5606305599212646, + "max_kl": 2.455296277999878, + "mean_kl": 0.5367228388786316, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.195101261138916, + "code_tokens": 92, + "cvar95_kl": 5.323950290679932, + "delta_nll": 1.0962557792663574, + "max_kl": 7.809332370758057, + "mean_kl": 1.1034612655639648, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.5760869383811951 + }, + { + "candidate_nll": 1.924422264099121, + "code_tokens": 185, + "cvar95_kl": 4.962944030761719, + "delta_nll": 1.194307565689087, + "max_kl": 7.171605587005615, + "mean_kl": 1.1585057973861694, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 1.5151300430297852, + "code_tokens": 106, + "cvar95_kl": 4.892622470855713, + "delta_nll": 1.1621354818344116, + "max_kl": 7.02252197265625, + "mean_kl": 1.1589603424072266, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.6320754885673523 + }, + { + "candidate_nll": 2.93650221824646, + "code_tokens": 64, + "cvar95_kl": 5.444129943847656, + "delta_nll": 1.393947958946228, + "max_kl": 8.062878608703613, + "mean_kl": 1.1890745162963867, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.53125 + }, + { + "candidate_nll": 2.288447141647339, + "code_tokens": 74, + "cvar95_kl": 4.688990592956543, + "delta_nll": 1.1776630878448486, + "max_kl": 5.277153968811035, + "mean_kl": 1.247626781463623, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.5810810923576355 + }, + { + "candidate_nll": 1.7771239280700684, + "code_tokens": 27, + "cvar95_kl": 3.6366748809814453, + "delta_nll": 0.7897200584411621, + "max_kl": 4.716245651245117, + "mean_kl": 0.7655318975448608, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.2180447578430176, + "code_tokens": 72, + "cvar95_kl": 3.770374298095703, + "delta_nll": 0.5942068696022034, + "max_kl": 4.390496730804443, + "mean_kl": 0.7001088857650757, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.6805555820465088 + }, + { + "candidate_nll": 1.6118693351745605, + "code_tokens": 62, + "cvar95_kl": 3.7237703800201416, + "delta_nll": 0.6785669922828674, + "max_kl": 5.993428707122803, + "mean_kl": 0.7262714505195618, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.4764209985733032, + "code_tokens": 42, + "cvar95_kl": 3.5520236492156982, + "delta_nll": 0.7782546281814575, + "max_kl": 4.016918182373047, + "mean_kl": 0.8874324560165405, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.8903356790542603, + "code_tokens": 238, + "cvar95_kl": 4.626539707183838, + "delta_nll": 1.0030028820037842, + "max_kl": 6.333737373352051, + "mean_kl": 1.007502555847168, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.651260495185852 + }, + { + "candidate_nll": 1.5304481983184814, + "code_tokens": 68, + "cvar95_kl": 4.022569179534912, + "delta_nll": 0.6159364581108093, + "max_kl": 6.678524494171143, + "mean_kl": 0.651828944683075, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.779411792755127 + }, + { + "candidate_nll": 2.240602970123291, + "code_tokens": 83, + "cvar95_kl": 5.882298469543457, + "delta_nll": 1.256453037261963, + "max_kl": 6.749444961547852, + "mean_kl": 1.2417844533920288, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.5542168617248535 + }, + { + "candidate_nll": 1.977936029434204, + "code_tokens": 64, + "cvar95_kl": 4.066760540008545, + "delta_nll": 0.7141685485839844, + "max_kl": 4.803587436676025, + "mean_kl": 0.7168624997138977, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.1726034879684448, + "code_tokens": 37, + "cvar95_kl": 2.4078779220581055, + "delta_nll": 0.5158035755157471, + "max_kl": 2.8215465545654297, + "mean_kl": 0.4947506785392761, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 2.191648006439209, + "code_tokens": 27, + "cvar95_kl": 4.200430870056152, + "delta_nll": 0.6817208528518677, + "max_kl": 5.165392875671387, + "mean_kl": 0.920294463634491, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.5191147327423096, + "code_tokens": 40, + "cvar95_kl": 4.229637622833252, + "delta_nll": 0.6273983716964722, + "max_kl": 4.811721324920654, + "mean_kl": 0.6423074007034302, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.824999988079071 + }, + { + "candidate_nll": 1.6087592840194702, + "code_tokens": 28, + "cvar95_kl": 3.656810760498047, + "delta_nll": 0.610986590385437, + "max_kl": 4.832565784454346, + "mean_kl": 0.7691978812217712, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.6071428656578064 + }, + { + "candidate_nll": 2.077192544937134, + "code_tokens": 59, + "cvar95_kl": 4.8621978759765625, + "delta_nll": 1.2225629091262817, + "max_kl": 4.9956207275390625, + "mean_kl": 1.238348126411438, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.5254237055778503 + }, + { + "candidate_nll": 1.6513302326202393, + "code_tokens": 57, + "cvar95_kl": 4.111303806304932, + "delta_nll": 1.0813300609588623, + "max_kl": 4.830085277557373, + "mean_kl": 0.9468443989753723, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.323896646499634, + "code_tokens": 25, + "cvar95_kl": 3.709001064300537, + "delta_nll": 0.939956545829773, + "max_kl": 3.73199462890625, + "mean_kl": 0.8215607404708862, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.5705699920654297, + "code_tokens": 48, + "cvar95_kl": 3.790027379989624, + "delta_nll": 0.8234176635742188, + "max_kl": 4.488565921783447, + "mean_kl": 0.7136160731315613, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.2560110092163086, + "code_tokens": 35, + "cvar95_kl": 6.557774543762207, + "delta_nll": 1.3942351341247559, + "max_kl": 8.099321365356445, + "mean_kl": 1.0183093547821045, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.6285714507102966 + }, + { + "candidate_nll": 1.0428683757781982, + "code_tokens": 30, + "cvar95_kl": 2.4282617568969727, + "delta_nll": 0.3981611132621765, + "max_kl": 2.4878106117248535, + "mean_kl": 0.4819530248641968, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.6268430948257446, + "code_tokens": 49, + "cvar95_kl": 2.703545331954956, + "delta_nll": 0.5037655830383301, + "max_kl": 3.32241153717041, + "mean_kl": 0.533726155757904, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.8163265585899353 + }, + { + "candidate_nll": 2.1179404258728027, + "code_tokens": 98, + "cvar95_kl": 5.151853084564209, + "delta_nll": 1.101723551750183, + "max_kl": 6.265498161315918, + "mean_kl": 1.0643218755722046, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.5415500402450562, + "code_tokens": 25, + "cvar95_kl": 1.9616461992263794, + "delta_nll": 0.16074371337890625, + "max_kl": 1.9914664030075073, + "mean_kl": 0.5274837017059326, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.6792467832565308, + "code_tokens": 27, + "cvar95_kl": 6.6540727615356445, + "delta_nll": 0.9015235900878906, + "max_kl": 8.010356903076172, + "mean_kl": 0.9426111578941345, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.4321391582489014, + "code_tokens": 20, + "cvar95_kl": 2.757904291152954, + "delta_nll": 0.387054443359375, + "max_kl": 2.757904291152954, + "mean_kl": 0.540091872215271, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.9783357381820679, + "code_tokens": 253, + "cvar95_kl": 5.750490188598633, + "delta_nll": 1.28566575050354, + "max_kl": 8.462579727172852, + "mean_kl": 1.3112311363220215, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.573122501373291 + }, + { + "candidate_nll": 1.5513908863067627, + "code_tokens": 55, + "cvar95_kl": 5.037249565124512, + "delta_nll": 0.7222706079483032, + "max_kl": 6.185207366943359, + "mean_kl": 0.9286230802536011, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.6909090876579285 + }, + { + "candidate_nll": 1.7075077295303345, + "code_tokens": 31, + "cvar95_kl": 2.9380156993865967, + "delta_nll": 0.5010093450546265, + "max_kl": 3.006361484527588, + "mean_kl": 0.6303184032440186, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.4210706949234009, + "code_tokens": 35, + "cvar95_kl": 3.557878017425537, + "delta_nll": 0.7168245315551758, + "max_kl": 4.532816410064697, + "mean_kl": 0.6324841976165771, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.3909335136413574, + "code_tokens": 61, + "cvar95_kl": 4.305072784423828, + "delta_nll": 0.5880650281906128, + "max_kl": 5.565624237060547, + "mean_kl": 0.7529662847518921, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.7049180269241333 + }, + { + "candidate_nll": 2.104436159133911, + "code_tokens": 55, + "cvar95_kl": 6.045368194580078, + "delta_nll": 1.1497185230255127, + "max_kl": 7.894254207611084, + "mean_kl": 1.1388628482818604, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.5272727012634277 + }, + { + "candidate_nll": 2.455811023712158, + "code_tokens": 108, + "cvar95_kl": 4.015523433685303, + "delta_nll": 0.9488438367843628, + "max_kl": 4.621423244476318, + "mean_kl": 0.853942334651947, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.8685200214385986, + "code_tokens": 58, + "cvar95_kl": 3.192451238632202, + "delta_nll": 0.5812139511108398, + "max_kl": 4.082377910614014, + "mean_kl": 0.5818920135498047, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.7068965435028076 + }, + { + "candidate_nll": 2.2756502628326416, + "code_tokens": 91, + "cvar95_kl": 6.469282627105713, + "delta_nll": 1.448194980621338, + "max_kl": 10.094683647155762, + "mean_kl": 1.4185662269592285, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.5274725556373596 + }, + { + "candidate_nll": 1.6420491933822632, + "code_tokens": 105, + "cvar95_kl": 3.7321863174438477, + "delta_nll": 0.8783097267150879, + "max_kl": 4.274991512298584, + "mean_kl": 0.8876848220825195, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.6952381134033203 + }, + { + "candidate_nll": 1.6528997421264648, + "code_tokens": 49, + "cvar95_kl": 3.457401990890503, + "delta_nll": 1.0056471824645996, + "max_kl": 3.9171180725097656, + "mean_kl": 0.8698972463607788, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.6938775777816772 + }, + { + "candidate_nll": 2.1378047466278076, + "code_tokens": 27, + "cvar95_kl": 3.873096466064453, + "delta_nll": 0.6756417751312256, + "max_kl": 4.432293891906738, + "mean_kl": 0.8573782444000244, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.5541309118270874, + "code_tokens": 25, + "cvar95_kl": 2.7307331562042236, + "delta_nll": 0.5137542486190796, + "max_kl": 2.9405405521392822, + "mean_kl": 0.6988351941108704, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 2.2432618141174316, + "code_tokens": 98, + "cvar95_kl": 4.589789867401123, + "delta_nll": 0.9482073783874512, + "max_kl": 5.73980188369751, + "mean_kl": 1.151818871498108, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.6020408272743225 + }, + { + "candidate_nll": 1.3970036506652832, + "code_tokens": 23, + "cvar95_kl": 2.6582131385803223, + "delta_nll": 0.7472219467163086, + "max_kl": 3.1739344596862793, + "mean_kl": 0.6522051692008972, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.782608687877655 + }, + { + "candidate_nll": 3.051603317260742, + "code_tokens": 31, + "cvar95_kl": 5.87539529800415, + "delta_nll": 1.1862295866012573, + "max_kl": 7.6394362449646, + "mean_kl": 1.0920718908309937, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.3704065084457397, + "code_tokens": 36, + "cvar95_kl": 3.57853364944458, + "delta_nll": 0.622313380241394, + "max_kl": 4.717039108276367, + "mean_kl": 0.705575704574585, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.6944444179534912 + }, + { + "candidate_nll": 1.860507607460022, + "code_tokens": 38, + "cvar95_kl": 3.6113638877868652, + "delta_nll": 0.4268300533294678, + "max_kl": 3.7077741622924805, + "mean_kl": 0.7494192123413086, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 2.3780930042266846, + "code_tokens": 25, + "cvar95_kl": 1.8934836387634277, + "delta_nll": 0.772003173828125, + "max_kl": 2.6258270740509033, + "mean_kl": 0.47493094205856323, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 1.4012622833251953, + "code_tokens": 26, + "cvar95_kl": 3.1061973571777344, + "delta_nll": 0.521813154220581, + "max_kl": 3.6092917919158936, + "mean_kl": 0.6802133321762085, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.8871623277664185, + "code_tokens": 62, + "cvar95_kl": 5.499540328979492, + "delta_nll": 1.1447203159332275, + "max_kl": 6.343327522277832, + "mean_kl": 1.0124648809432983, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.725806474685669 + }, + { + "candidate_nll": 2.680555820465088, + "code_tokens": 36, + "cvar95_kl": 3.5386977195739746, + "delta_nll": 0.9420367479324341, + "max_kl": 3.6486566066741943, + "mean_kl": 0.900456964969635, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 2.139406204223633, + "code_tokens": 119, + "cvar95_kl": 4.925670146942139, + "delta_nll": 0.8088172674179077, + "max_kl": 6.419483661651611, + "mean_kl": 1.0452830791473389, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.6134454011917114 + }, + { + "candidate_nll": 0.7195303440093994, + "code_tokens": 37, + "cvar95_kl": 1.5333824157714844, + "delta_nll": 0.4068227708339691, + "max_kl": 1.5682716369628906, + "mean_kl": 0.3124232292175293, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.8918918967247009 + }, + { + "candidate_nll": 1.512635588645935, + "code_tokens": 28, + "cvar95_kl": 2.7634453773498535, + "delta_nll": 0.7299020886421204, + "max_kl": 3.0156073570251465, + "mean_kl": 0.7275063395500183, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.7345579862594604, + "code_tokens": 143, + "cvar95_kl": 6.432378768920898, + "delta_nll": 1.1682112216949463, + "max_kl": 8.112530708312988, + "mean_kl": 1.150236964225769, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 2.5057480335235596, + "code_tokens": 29, + "cvar95_kl": 5.213614463806152, + "delta_nll": 0.7234417200088501, + "max_kl": 6.646160125732422, + "mean_kl": 1.1417155265808105, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 1.9839590787887573, + "code_tokens": 28, + "cvar95_kl": 3.8970746994018555, + "delta_nll": 0.9114706516265869, + "max_kl": 4.315847873687744, + "mean_kl": 0.8506141901016235, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 2.324286699295044, + "code_tokens": 30, + "cvar95_kl": 6.662139415740967, + "delta_nll": 1.0602489709854126, + "max_kl": 7.575652599334717, + "mean_kl": 1.3029311895370483, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.5333333611488342 + }, + { + "candidate_nll": 1.5511318445205688, + "code_tokens": 85, + "cvar95_kl": 5.213353157043457, + "delta_nll": 1.129321813583374, + "max_kl": 6.161235809326172, + "mean_kl": 0.9853991270065308, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.6705882549285889 + }, + { + "candidate_nll": 1.9771220684051514, + "code_tokens": 44, + "cvar95_kl": 3.613473653793335, + "delta_nll": 0.999878466129303, + "max_kl": 3.7269208431243896, + "mean_kl": 0.9557847380638123, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.7045454382896423 + }, + { + "candidate_nll": 1.700090765953064, + "code_tokens": 19, + "cvar95_kl": 2.5391578674316406, + "delta_nll": 0.8959885239601135, + "max_kl": 2.5391578674316406, + "mean_kl": 0.41961634159088135, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.8421052694320679 + }, + { + "candidate_nll": 1.7778817415237427, + "code_tokens": 113, + "cvar95_kl": 5.010654926300049, + "delta_nll": 1.0404589176177979, + "max_kl": 5.521114826202393, + "mean_kl": 1.0111110210418701, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.6814159154891968 + }, + { + "candidate_nll": 2.1917521953582764, + "code_tokens": 40, + "cvar95_kl": 2.5426506996154785, + "delta_nll": 0.7165577411651611, + "max_kl": 3.0086240768432617, + "mean_kl": 0.7825903296470642, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.6108341217041016, + "code_tokens": 51, + "cvar95_kl": 4.485322952270508, + "delta_nll": 1.0041741132736206, + "max_kl": 5.762330055236816, + "mean_kl": 0.9821942448616028, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.6274510025978088 + }, + { + "candidate_nll": 2.5914008617401123, + "code_tokens": 55, + "cvar95_kl": 3.9831457138061523, + "delta_nll": 1.0103683471679688, + "max_kl": 4.471619606018066, + "mean_kl": 1.1583616733551025, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.5636363625526428 + }, + { + "candidate_nll": 1.587894320487976, + "code_tokens": 27, + "cvar95_kl": 2.653705596923828, + "delta_nll": 0.45782601833343506, + "max_kl": 2.8706603050231934, + "mean_kl": 0.6045097708702087, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 1.3655283451080322, + "code_tokens": 32, + "cvar95_kl": 3.6716599464416504, + "delta_nll": 0.5257221460342407, + "max_kl": 4.817238807678223, + "mean_kl": 0.5330272316932678, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.8125 + }, + { + "candidate_nll": 1.4917309284210205, + "code_tokens": 64, + "cvar95_kl": 3.8507461547851562, + "delta_nll": 0.8391928672790527, + "max_kl": 4.18488883972168, + "mean_kl": 0.9025732278823853, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.640625 + }, + { + "candidate_nll": 1.8944809436798096, + "code_tokens": 35, + "cvar95_kl": 4.026094913482666, + "delta_nll": 0.8448266983032227, + "max_kl": 5.210236549377441, + "mean_kl": 0.6827400326728821, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.7714285850524902 + }, + { + "candidate_nll": 1.8742053508758545, + "code_tokens": 152, + "cvar95_kl": 5.616546630859375, + "delta_nll": 1.4223577976226807, + "max_kl": 7.033771514892578, + "mean_kl": 1.3192098140716553, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.5855262875556946 + }, + { + "candidate_nll": 2.18206787109375, + "code_tokens": 77, + "cvar95_kl": 3.9789440631866455, + "delta_nll": 0.9207335710525513, + "max_kl": 7.354038715362549, + "mean_kl": 0.9330724477767944, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.6233766078948975 + }, + { + "candidate_nll": 1.498720645904541, + "code_tokens": 77, + "cvar95_kl": 4.80938720703125, + "delta_nll": 0.9304395914077759, + "max_kl": 5.985877513885498, + "mean_kl": 0.9007807970046997, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.701298713684082 + }, + { + "candidate_nll": 0.9036263227462769, + "code_tokens": 71, + "cvar95_kl": 2.8585245609283447, + "delta_nll": 0.3415375351905823, + "max_kl": 3.7261784076690674, + "mean_kl": 0.40242427587509155, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.8732394576072693 + }, + { + "candidate_nll": 2.155066728591919, + "code_tokens": 112, + "cvar95_kl": 5.187180995941162, + "delta_nll": 1.2766873836517334, + "max_kl": 7.192046165466309, + "mean_kl": 1.1026033163070679, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.6517857313156128 + }, + { + "candidate_nll": 1.625439167022705, + "code_tokens": 111, + "cvar95_kl": 3.4046356678009033, + "delta_nll": 0.9055354595184326, + "max_kl": 4.674134254455566, + "mean_kl": 0.7453005909919739, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.6936936974525452 + }, + { + "candidate_nll": 1.6928848028182983, + "code_tokens": 62, + "cvar95_kl": 4.2129225730896, + "delta_nll": 0.9049885869026184, + "max_kl": 5.252715587615967, + "mean_kl": 0.8332744836807251, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.725806474685669 + }, + { + "candidate_nll": 1.3746886253356934, + "code_tokens": 52, + "cvar95_kl": 3.6807117462158203, + "delta_nll": 0.7641085982322693, + "max_kl": 4.107074737548828, + "mean_kl": 0.6197985410690308, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 2.527170181274414, + "code_tokens": 24, + "cvar95_kl": 3.3880810737609863, + "delta_nll": 0.6319435834884644, + "max_kl": 3.6072747707366943, + "mean_kl": 0.6365644335746765, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5597962141036987, + "code_tokens": 48, + "cvar95_kl": 3.8562328815460205, + "delta_nll": 0.6680262684822083, + "max_kl": 4.089693546295166, + "mean_kl": 0.7915723919868469, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.239861249923706, + "code_tokens": 98, + "cvar95_kl": 5.3904643058776855, + "delta_nll": 1.189568281173706, + "max_kl": 8.03225326538086, + "mean_kl": 1.131460428237915, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.5918367505073547 + }, + { + "candidate_nll": 2.045419692993164, + "code_tokens": 48, + "cvar95_kl": 4.044170379638672, + "delta_nll": 1.052734136581421, + "max_kl": 4.517620086669922, + "mean_kl": 0.9947978854179382, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.6041666865348816 + }, + { + "candidate_nll": 1.9753788709640503, + "code_tokens": 56, + "cvar95_kl": 4.580817699432373, + "delta_nll": 1.0293834209442139, + "max_kl": 5.7667365074157715, + "mean_kl": 0.9393439292907715, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.5892857313156128 + }, + { + "candidate_nll": 1.6974215507507324, + "code_tokens": 88, + "cvar95_kl": 5.133063316345215, + "delta_nll": 1.096144676208496, + "max_kl": 7.938470840454102, + "mean_kl": 0.9468786120414734, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 2.522062063217163, + "code_tokens": 50, + "cvar95_kl": 4.262121677398682, + "delta_nll": 0.8885318040847778, + "max_kl": 4.9585676193237305, + "mean_kl": 0.7952315807342529, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.0627713203430176, + "code_tokens": 85, + "cvar95_kl": 3.428539991378784, + "delta_nll": 0.625784158706665, + "max_kl": 6.193647861480713, + "mean_kl": 0.623600423336029, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.7176470756530762 + }, + { + "candidate_nll": 2.032162666320801, + "code_tokens": 27, + "cvar95_kl": 4.615974426269531, + "delta_nll": 0.9579495191574097, + "max_kl": 4.630626678466797, + "mean_kl": 0.7586186528205872, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.6635199785232544, + "code_tokens": 48, + "cvar95_kl": 4.345244884490967, + "delta_nll": 0.5459015369415283, + "max_kl": 4.923090934753418, + "mean_kl": 0.6494598388671875, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.7708333134651184 + }, + { + "candidate_nll": 2.4960618019104004, + "code_tokens": 29, + "cvar95_kl": 3.2959439754486084, + "delta_nll": 0.8482145071029663, + "max_kl": 3.5303761959075928, + "mean_kl": 0.8986372947692871, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.7969094514846802, + "code_tokens": 63, + "cvar95_kl": 6.120684623718262, + "delta_nll": 0.8586940765380859, + "max_kl": 8.019082069396973, + "mean_kl": 1.1101628541946411, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.682539701461792 + }, + { + "candidate_nll": 1.994989275932312, + "code_tokens": 89, + "cvar95_kl": 6.566096305847168, + "delta_nll": 1.3199305534362793, + "max_kl": 9.557247161865234, + "mean_kl": 1.2475693225860596, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.6292135119438171 + }, + { + "candidate_nll": 2.7919669151306152, + "code_tokens": 133, + "cvar95_kl": 6.792613506317139, + "delta_nll": 1.4476885795593262, + "max_kl": 9.373529434204102, + "mean_kl": 1.3771607875823975, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.5789473652839661 + }, + { + "candidate_nll": 1.4148375988006592, + "code_tokens": 25, + "cvar95_kl": 2.424076557159424, + "delta_nll": 0.4663008451461792, + "max_kl": 4.026508808135986, + "mean_kl": 0.367410272359848, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.4166722297668457, + "code_tokens": 33, + "cvar95_kl": 1.6157784461975098, + "delta_nll": 0.37937188148498535, + "max_kl": 2.097414970397949, + "mean_kl": 0.4384549856185913, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 1.1931188106536865, + "code_tokens": 93, + "cvar95_kl": 3.671412706375122, + "delta_nll": 0.43634384870529175, + "max_kl": 5.28640604019165, + "mean_kl": 0.575340211391449, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.7419354915618896 + }, + { + "candidate_nll": 2.238922119140625, + "code_tokens": 101, + "cvar95_kl": 4.508175373077393, + "delta_nll": 1.2705225944519043, + "max_kl": 5.371719837188721, + "mean_kl": 1.192131519317627, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.5742574334144592 + }, + { + "candidate_nll": 1.8429956436157227, + "code_tokens": 152, + "cvar95_kl": 5.774280548095703, + "delta_nll": 1.2782323360443115, + "max_kl": 7.950103282928467, + "mean_kl": 1.2130051851272583, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.7994669675827026, + "code_tokens": 33, + "cvar95_kl": 2.515392780303955, + "delta_nll": 0.3985954523086548, + "max_kl": 2.5488643646240234, + "mean_kl": 0.5508022308349609, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8661563396453857, + "code_tokens": 29, + "cvar95_kl": 2.251554012298584, + "delta_nll": 0.5796934366226196, + "max_kl": 2.447996139526367, + "mean_kl": 0.5567067265510559, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.7931034564971924 + }, + { + "candidate_nll": 2.279224157333374, + "code_tokens": 27, + "cvar95_kl": 2.5819664001464844, + "delta_nll": 0.8211543560028076, + "max_kl": 3.250417470932007, + "mean_kl": 0.5394114255905151, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.8518518805503845 + }, + { + "candidate_nll": 0.9362607598304749, + "code_tokens": 33, + "cvar95_kl": 3.2657816410064697, + "delta_nll": 0.6262206435203552, + "max_kl": 3.4824490547180176, + "mean_kl": 0.6449345350265503, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 1.2061724662780762, + "code_tokens": 27, + "cvar95_kl": 1.4551807641983032, + "delta_nll": 0.40977418422698975, + "max_kl": 1.8822896480560303, + "mean_kl": 0.38445231318473816, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.8148148059844971 + }, + { + "candidate_nll": 1.6315175294876099, + "code_tokens": 84, + "cvar95_kl": 3.5841541290283203, + "delta_nll": 0.9276240468025208, + "max_kl": 4.168164253234863, + "mean_kl": 0.8206908702850342, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.7023809552192688 + }, + { + "candidate_nll": 1.9190973043441772, + "code_tokens": 103, + "cvar95_kl": 3.412926435470581, + "delta_nll": 0.8736274242401123, + "max_kl": 4.673380374908447, + "mean_kl": 0.8163552284240723, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.6504854559898376 + }, + { + "candidate_nll": 1.6255496740341187, + "code_tokens": 88, + "cvar95_kl": 3.3105404376983643, + "delta_nll": 0.7092023491859436, + "max_kl": 4.296755790710449, + "mean_kl": 0.7984740734100342, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.6818181872367859 + }, + { + "candidate_nll": 2.5903899669647217, + "code_tokens": 62, + "cvar95_kl": 6.622922420501709, + "delta_nll": 1.2814592123031616, + "max_kl": 7.930595874786377, + "mean_kl": 1.161439299583435, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 2.054344415664673, + "code_tokens": 24, + "cvar95_kl": 1.7230397462844849, + "delta_nll": 0.5176355838775635, + "max_kl": 1.7735570669174194, + "mean_kl": 0.475236177444458, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.329566478729248, + "code_tokens": 34, + "cvar95_kl": 3.791318416595459, + "delta_nll": 0.9911143779754639, + "max_kl": 4.538718223571777, + "mean_kl": 0.7433950901031494, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 1.2177144289016724, + "code_tokens": 43, + "cvar95_kl": 2.812276601791382, + "delta_nll": 0.49796903133392334, + "max_kl": 3.5550522804260254, + "mean_kl": 0.5208929777145386, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.7674418687820435 + }, + { + "candidate_nll": 2.354874849319458, + "code_tokens": 54, + "cvar95_kl": 2.7027556896209717, + "delta_nll": 0.8655695915222168, + "max_kl": 3.183993339538574, + "mean_kl": 0.7881979942321777, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.3539825677871704, + "code_tokens": 50, + "cvar95_kl": 2.2427492141723633, + "delta_nll": 0.6240971088409424, + "max_kl": 2.5996031761169434, + "mean_kl": 0.6260632276535034, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.9906904697418213, + "code_tokens": 235, + "cvar95_kl": 5.352676868438721, + "delta_nll": 1.233180046081543, + "max_kl": 7.757999897003174, + "mean_kl": 1.265448808670044, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.5957446694374084 + }, + { + "candidate_nll": 2.7449393272399902, + "code_tokens": 33, + "cvar95_kl": 3.9554662704467773, + "delta_nll": 0.7162842750549316, + "max_kl": 4.604513168334961, + "mean_kl": 0.8217800855636597, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 1.6091920137405396, + "code_tokens": 45, + "cvar95_kl": 2.164774179458618, + "delta_nll": 0.6226896047592163, + "max_kl": 2.7322998046875, + "mean_kl": 0.40764668583869934, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.8222222328186035 + }, + { + "candidate_nll": 1.4304077625274658, + "code_tokens": 28, + "cvar95_kl": 4.170969486236572, + "delta_nll": 0.8074478507041931, + "max_kl": 4.405911445617676, + "mean_kl": 0.7255359888076782, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.3164142370224, + "code_tokens": 49, + "cvar95_kl": 3.2707908153533936, + "delta_nll": 0.6381498575210571, + "max_kl": 3.873025417327881, + "mean_kl": 0.6754935383796692, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.7755101919174194 + }, + { + "candidate_nll": 1.9847557544708252, + "code_tokens": 107, + "cvar95_kl": 6.388322353363037, + "delta_nll": 1.7353401184082031, + "max_kl": 7.6242170333862305, + "mean_kl": 1.5805730819702148, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.5514018535614014 + }, + { + "candidate_nll": 2.0806782245635986, + "code_tokens": 48, + "cvar95_kl": 5.808570384979248, + "delta_nll": 0.8597681522369385, + "max_kl": 8.68789291381836, + "mean_kl": 0.9656276702880859, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.6458333134651184 + }, + { + "candidate_nll": 2.3674824237823486, + "code_tokens": 40, + "cvar95_kl": 3.4216370582580566, + "delta_nll": 0.5279415845870972, + "max_kl": 3.4371256828308105, + "mean_kl": 0.9865511655807495, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.550000011920929 + }, + { + "candidate_nll": 1.1363399028778076, + "code_tokens": 46, + "cvar95_kl": 4.449655055999756, + "delta_nll": 0.6668562293052673, + "max_kl": 6.841485977172852, + "mean_kl": 0.7823375463485718, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.695652186870575 + }, + { + "candidate_nll": 1.7480570077896118, + "code_tokens": 31, + "cvar95_kl": 4.747533798217773, + "delta_nll": 0.7056072950363159, + "max_kl": 5.806318283081055, + "mean_kl": 0.8250923156738281, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 0.8370722532272339, + "code_tokens": 58, + "cvar95_kl": 2.677840232849121, + "delta_nll": 0.42058777809143066, + "max_kl": 3.656283140182495, + "mean_kl": 0.40103650093078613, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.8620689511299133 + }, + { + "candidate_nll": 1.8198202848434448, + "code_tokens": 49, + "cvar95_kl": 3.2929112911224365, + "delta_nll": 0.7020504474639893, + "max_kl": 3.759334087371826, + "mean_kl": 0.7468516230583191, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.4711377620697021, + "code_tokens": 18, + "cvar95_kl": 1.3338173627853394, + "delta_nll": 0.5245441794395447, + "max_kl": 1.3338173627853394, + "mean_kl": 0.4559480845928192, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.5236079692840576, + "code_tokens": 28, + "cvar95_kl": 1.7586771249771118, + "delta_nll": 0.36419641971588135, + "max_kl": 1.9965360164642334, + "mean_kl": 0.3914887607097626, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.6075725555419922, + "code_tokens": 57, + "cvar95_kl": 3.919938087463379, + "delta_nll": 0.8655058145523071, + "max_kl": 5.161026477813721, + "mean_kl": 0.8400904536247253, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.6491228342056274 + }, + { + "candidate_nll": 1.8485015630722046, + "code_tokens": 88, + "cvar95_kl": 4.441246032714844, + "delta_nll": 1.1483917236328125, + "max_kl": 4.955263137817383, + "mean_kl": 1.162744402885437, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.5795454382896423 + }, + { + "candidate_nll": 1.7367023229599, + "code_tokens": 23, + "cvar95_kl": 2.4674456119537354, + "delta_nll": 0.5379154682159424, + "max_kl": 2.9459657669067383, + "mean_kl": 0.5605763792991638, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.0812032222747803, + "code_tokens": 40, + "cvar95_kl": 5.429068565368652, + "delta_nll": 0.8217458724975586, + "max_kl": 5.740419387817383, + "mean_kl": 1.0132805109024048, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.9877849817276, + "code_tokens": 20, + "cvar95_kl": 2.583487033843994, + "delta_nll": 0.8265699148178101, + "max_kl": 2.583487033843994, + "mean_kl": 0.8181670904159546, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.5018057823181152, + "code_tokens": 63, + "cvar95_kl": 2.653573751449585, + "delta_nll": 0.48602020740509033, + "max_kl": 3.6663224697113037, + "mean_kl": 0.5618097186088562, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 1.8989145755767822, + "code_tokens": 39, + "cvar95_kl": 4.510194301605225, + "delta_nll": 0.9462332725524902, + "max_kl": 5.930915832519531, + "mean_kl": 0.8037245273590088, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 2.5289695262908936, + "code_tokens": 50, + "cvar95_kl": 4.680840969085693, + "delta_nll": 1.1171048879623413, + "max_kl": 5.418771743774414, + "mean_kl": 0.9846261739730835, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 1.694691777229309, + "code_tokens": 63, + "cvar95_kl": 4.905213832855225, + "delta_nll": 0.727214515209198, + "max_kl": 6.441895484924316, + "mean_kl": 0.9299089908599854, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.6984127163887024 + }, + { + "candidate_nll": 1.6146516799926758, + "code_tokens": 46, + "cvar95_kl": 2.89825701713562, + "delta_nll": 0.6420896649360657, + "max_kl": 3.263378858566284, + "mean_kl": 0.6554650068283081, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.6304348111152649 + }, + { + "candidate_nll": 1.2120332717895508, + "code_tokens": 25, + "cvar95_kl": 2.483729839324951, + "delta_nll": 0.5331387519836426, + "max_kl": 3.11545467376709, + "mean_kl": 0.4870784282684326, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.8399999737739563 + }, + { + "candidate_nll": 1.628605604171753, + "code_tokens": 111, + "cvar95_kl": 3.602576971054077, + "delta_nll": 0.8143396973609924, + "max_kl": 5.108577251434326, + "mean_kl": 0.8086228966712952, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.7207207083702087 + }, + { + "candidate_nll": 1.5814217329025269, + "code_tokens": 71, + "cvar95_kl": 4.152688503265381, + "delta_nll": 0.8528639078140259, + "max_kl": 5.25747537612915, + "mean_kl": 0.7992809414863586, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.6056337952613831 + }, + { + "candidate_nll": 1.794948935508728, + "code_tokens": 28, + "cvar95_kl": 2.7291083335876465, + "delta_nll": 0.4242253303527832, + "max_kl": 2.8965158462524414, + "mean_kl": 0.6024642586708069, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.8214285969734192 + }, + { + "candidate_nll": 1.8578033447265625, + "code_tokens": 25, + "cvar95_kl": 3.907350540161133, + "delta_nll": 0.6573474407196045, + "max_kl": 4.988344669342041, + "mean_kl": 0.754241406917572, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 2.242889881134033, + "code_tokens": 40, + "cvar95_kl": 3.62453031539917, + "delta_nll": 0.9401180744171143, + "max_kl": 3.6451587677001953, + "mean_kl": 0.6931511759757996, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.8117915391921997, + "code_tokens": 177, + "cvar95_kl": 4.348735332489014, + "delta_nll": 0.7673473358154297, + "max_kl": 4.992046356201172, + "mean_kl": 1.03815758228302, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.5649717450141907 + }, + { + "candidate_nll": 1.7746394872665405, + "code_tokens": 51, + "cvar95_kl": 2.8033905029296875, + "delta_nll": 0.6834609508514404, + "max_kl": 3.6767616271972656, + "mean_kl": 0.6188623309135437, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 1.3560545444488525, + "code_tokens": 69, + "cvar95_kl": 3.8130247592926025, + "delta_nll": 0.9015941619873047, + "max_kl": 4.985284328460693, + "mean_kl": 0.7402786612510681, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.7246376872062683 + }, + { + "candidate_nll": 1.1247410774230957, + "code_tokens": 26, + "cvar95_kl": 2.673274040222168, + "delta_nll": 0.3764811158180237, + "max_kl": 2.915193557739258, + "mean_kl": 0.4312869906425476, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.8846153616905212 + }, + { + "candidate_nll": 2.18928599357605, + "code_tokens": 24, + "cvar95_kl": 2.4850246906280518, + "delta_nll": 0.6709467172622681, + "max_kl": 2.857862949371338, + "mean_kl": 0.5408361554145813, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5670676231384277, + "code_tokens": 15, + "cvar95_kl": 1.8579357862472534, + "delta_nll": 0.4165917634963989, + "max_kl": 1.8579357862472534, + "mean_kl": 0.46434858441352844, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.8753670454025269, + "code_tokens": 43, + "cvar95_kl": 4.580312252044678, + "delta_nll": 0.7260180711746216, + "max_kl": 6.0887322425842285, + "mean_kl": 0.939144492149353, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.6511628031730652 + }, + { + "candidate_nll": 1.964885950088501, + "code_tokens": 18, + "cvar95_kl": 3.1426448822021484, + "delta_nll": 0.8725602626800537, + "max_kl": 3.1426448822021484, + "mean_kl": 0.5725545883178711, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.6302486658096313, + "code_tokens": 49, + "cvar95_kl": 4.702908515930176, + "delta_nll": 0.902255117893219, + "max_kl": 6.562788963317871, + "mean_kl": 0.8895532488822937, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.4496278762817383, + "code_tokens": 59, + "cvar95_kl": 4.4395012855529785, + "delta_nll": 0.5818718075752258, + "max_kl": 5.216719150543213, + "mean_kl": 0.7620542645454407, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.7118644118309021 + }, + { + "candidate_nll": 1.4793739318847656, + "code_tokens": 221, + "cvar95_kl": 4.145755767822266, + "delta_nll": 0.8941077589988708, + "max_kl": 5.459139823913574, + "mean_kl": 0.949744462966919, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.6380090713500977 + }, + { + "candidate_nll": 1.7008758783340454, + "code_tokens": 25, + "cvar95_kl": 2.5234665870666504, + "delta_nll": 0.4990028142929077, + "max_kl": 2.979872465133667, + "mean_kl": 0.6594104170799255, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 1.4832713603973389, + "code_tokens": 36, + "cvar95_kl": 3.4864540100097656, + "delta_nll": 0.7593507766723633, + "max_kl": 3.8500118255615234, + "mean_kl": 0.9004418849945068, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.6386096477508545, + "code_tokens": 158, + "cvar95_kl": 4.4020795822143555, + "delta_nll": 0.9944648146629333, + "max_kl": 5.722357749938965, + "mean_kl": 1.0565747022628784, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.6392405033111572 + }, + { + "candidate_nll": 2.3318722248077393, + "code_tokens": 92, + "cvar95_kl": 4.1030707359313965, + "delta_nll": 1.0163371562957764, + "max_kl": 4.4947357177734375, + "mean_kl": 0.8861488699913025, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.6739130616188049 + }, + { + "candidate_nll": 2.086777448654175, + "code_tokens": 102, + "cvar95_kl": 3.5176117420196533, + "delta_nll": 1.1727347373962402, + "max_kl": 3.944974660873413, + "mean_kl": 1.0406348705291748, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 2.0272433757781982, + "code_tokens": 171, + "cvar95_kl": 6.224020004272461, + "delta_nll": 1.4172732830047607, + "max_kl": 9.511417388916016, + "mean_kl": 1.298951506614685, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.6023392081260681 + }, + { + "candidate_nll": 2.1939501762390137, + "code_tokens": 40, + "cvar95_kl": 4.299635410308838, + "delta_nll": 0.9890860319137573, + "max_kl": 5.411892890930176, + "mean_kl": 0.8343197703361511, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 2.5446465015411377, + "code_tokens": 48, + "cvar95_kl": 5.993013381958008, + "delta_nll": 1.3086326122283936, + "max_kl": 7.740626811981201, + "mean_kl": 1.3074978590011597, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 2.487170934677124, + "code_tokens": 24, + "cvar95_kl": 4.310807228088379, + "delta_nll": 0.8822505474090576, + "max_kl": 5.371591091156006, + "mean_kl": 1.1495672464370728, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8458298444747925, + "code_tokens": 48, + "cvar95_kl": 3.349505662918091, + "delta_nll": 0.8587256669998169, + "max_kl": 4.369076728820801, + "mean_kl": 0.7298898696899414, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 2.3076891899108887, + "code_tokens": 25, + "cvar95_kl": 3.0880017280578613, + "delta_nll": 0.9208060503005981, + "max_kl": 3.8127524852752686, + "mean_kl": 0.6821739077568054, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 2.7114932537078857, + "code_tokens": 30, + "cvar95_kl": 3.0199058055877686, + "delta_nll": 1.212748408317566, + "max_kl": 3.604248046875, + "mean_kl": 0.7702673673629761, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.8144229650497437, + "code_tokens": 19, + "cvar95_kl": 1.3330941200256348, + "delta_nll": 0.38760507106781006, + "max_kl": 1.3330941200256348, + "mean_kl": 0.31136083602905273, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 1.192251443862915, + "code_tokens": 12, + "cvar95_kl": 0.9215580821037292, + "delta_nll": 0.11317896842956543, + "max_kl": 0.9215580821037292, + "mean_kl": 0.2525990307331085, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.240018844604492, + "code_tokens": 37, + "cvar95_kl": 3.7033417224884033, + "delta_nll": 0.8997091054916382, + "max_kl": 4.424580097198486, + "mean_kl": 0.8891890048980713, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.6216216087341309 + }, + { + "candidate_nll": 1.990130066871643, + "code_tokens": 33, + "cvar95_kl": 3.969839096069336, + "delta_nll": 0.7847002744674683, + "max_kl": 5.106308460235596, + "mean_kl": 0.7631688117980957, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 2.250197649002075, + "code_tokens": 94, + "cvar95_kl": 6.027419090270996, + "delta_nll": 1.3980878591537476, + "max_kl": 7.83931303024292, + "mean_kl": 1.253301978111267, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.5957446694374084 + }, + { + "candidate_nll": 1.4279842376708984, + "code_tokens": 90, + "cvar95_kl": 6.60873556137085, + "delta_nll": 0.8032193183898926, + "max_kl": 12.522397994995117, + "mean_kl": 0.9029083847999573, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.7555555701255798 + }, + { + "candidate_nll": 1.8147941827774048, + "code_tokens": 47, + "cvar95_kl": 2.71942400932312, + "delta_nll": 0.3670302629470825, + "max_kl": 3.8847901821136475, + "mean_kl": 0.4795018434524536, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.8297872543334961 + }, + { + "candidate_nll": 1.5986686944961548, + "code_tokens": 61, + "cvar95_kl": 3.6304187774658203, + "delta_nll": 0.6748345494270325, + "max_kl": 4.022110939025879, + "mean_kl": 0.8659714460372925, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.7049180269241333 + }, + { + "candidate_nll": 1.6789050102233887, + "code_tokens": 39, + "cvar95_kl": 2.813908576965332, + "delta_nll": 0.45539021492004395, + "max_kl": 3.5261008739471436, + "mean_kl": 0.47447508573532104, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.8461538553237915 + }, + { + "candidate_nll": 2.109078884124756, + "code_tokens": 152, + "cvar95_kl": 6.218183517456055, + "delta_nll": 1.1769170761108398, + "max_kl": 8.306821823120117, + "mean_kl": 1.2882473468780518, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.6052631735801697 + }, + { + "candidate_nll": 2.3201797008514404, + "code_tokens": 100, + "cvar95_kl": 6.639056205749512, + "delta_nll": 1.3511972427368164, + "max_kl": 8.059879302978516, + "mean_kl": 1.3853851556777954, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.5400000214576721 + }, + { + "candidate_nll": 2.2011401653289795, + "code_tokens": 89, + "cvar95_kl": 6.332533836364746, + "delta_nll": 1.5802123546600342, + "max_kl": 6.869868278503418, + "mean_kl": 1.3072713613510132, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.6516854166984558 + }, + { + "candidate_nll": 2.657830238342285, + "code_tokens": 40, + "cvar95_kl": 8.30843734741211, + "delta_nll": 1.3468290567398071, + "max_kl": 8.587394714355469, + "mean_kl": 1.1955164670944214, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.7099981307983398, + "code_tokens": 76, + "cvar95_kl": 4.167205810546875, + "delta_nll": 1.0084311962127686, + "max_kl": 6.013448715209961, + "mean_kl": 0.9633511900901794, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.9731255769729614, + "code_tokens": 56, + "cvar95_kl": 4.398210048675537, + "delta_nll": 0.7241382598876953, + "max_kl": 6.134716987609863, + "mean_kl": 0.8193928599357605, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 2.729743480682373, + "code_tokens": 34, + "cvar95_kl": 5.040526866912842, + "delta_nll": 1.240313172340393, + "max_kl": 6.230953693389893, + "mean_kl": 1.0351667404174805, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 2.0100228786468506, + "code_tokens": 96, + "cvar95_kl": 4.323399543762207, + "delta_nll": 1.331911325454712, + "max_kl": 4.757334232330322, + "mean_kl": 1.1791504621505737, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 1.7526545524597168, + "code_tokens": 451, + "cvar95_kl": 4.104779243469238, + "delta_nll": 1.0539593696594238, + "max_kl": 6.036643028259277, + "mean_kl": 0.9818211197853088, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.6430155038833618 + }, + { + "candidate_nll": 2.444943428039551, + "code_tokens": 37, + "cvar95_kl": 3.694509744644165, + "delta_nll": 0.8326337337493896, + "max_kl": 3.999302864074707, + "mean_kl": 0.9852608442306519, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 1.9620448350906372, + "code_tokens": 42, + "cvar95_kl": 2.923443078994751, + "delta_nll": 0.7204469442367554, + "max_kl": 3.1802635192871094, + "mean_kl": 0.82347571849823, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.8664860725402832, + "code_tokens": 33, + "cvar95_kl": 3.0055153369903564, + "delta_nll": 0.8497301340103149, + "max_kl": 3.621992826461792, + "mean_kl": 0.885533332824707, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.5757575631141663 + }, + { + "candidate_nll": 1.7591516971588135, + "code_tokens": 43, + "cvar95_kl": 2.899519681930542, + "delta_nll": 0.8331000208854675, + "max_kl": 4.016707420349121, + "mean_kl": 0.7433263063430786, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.7906976938247681 + }, + { + "candidate_nll": 1.6405245065689087, + "code_tokens": 73, + "cvar95_kl": 5.414511203765869, + "delta_nll": 0.9040880799293518, + "max_kl": 6.315585613250732, + "mean_kl": 0.9300881028175354, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.6438356041908264 + }, + { + "candidate_nll": 1.6506397724151611, + "code_tokens": 15, + "cvar95_kl": 2.4385921955108643, + "delta_nll": 0.7381635904312134, + "max_kl": 2.4385921955108643, + "mean_kl": 0.5809726715087891, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 2.0242769718170166, + "code_tokens": 34, + "cvar95_kl": 5.796079635620117, + "delta_nll": 0.7651970386505127, + "max_kl": 6.729874610900879, + "mean_kl": 0.956612765789032, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 2.102324962615967, + "code_tokens": 136, + "cvar95_kl": 3.926332473754883, + "delta_nll": 0.9938417673110962, + "max_kl": 7.819728374481201, + "mean_kl": 0.9722220301628113, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.6029411554336548 + }, + { + "candidate_nll": 1.7426315546035767, + "code_tokens": 17, + "cvar95_kl": 1.8987520933151245, + "delta_nll": 0.4964754581451416, + "max_kl": 1.8987520933151245, + "mean_kl": 0.6199160218238831, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.6832304000854492, + "code_tokens": 33, + "cvar95_kl": 4.113759517669678, + "delta_nll": 0.8901757001876831, + "max_kl": 5.589293956756592, + "mean_kl": 0.7941402196884155, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.6935676336288452, + "code_tokens": 38, + "cvar95_kl": 3.698211431503296, + "delta_nll": 0.8511208295822144, + "max_kl": 4.401806354522705, + "mean_kl": 0.7705394625663757, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.3498765230178833, + "code_tokens": 61, + "cvar95_kl": 3.5474791526794434, + "delta_nll": 0.9523519277572632, + "max_kl": 5.123800277709961, + "mean_kl": 0.8148788213729858, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.6721311211585999 + }, + { + "candidate_nll": 1.8594954013824463, + "code_tokens": 143, + "cvar95_kl": 5.1855549812316895, + "delta_nll": 1.1237759590148926, + "max_kl": 6.7812347412109375, + "mean_kl": 1.1749000549316406, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.5804196000099182 + }, + { + "candidate_nll": 1.6676586866378784, + "code_tokens": 41, + "cvar95_kl": 3.284538507461548, + "delta_nll": 0.5785927772521973, + "max_kl": 3.730771780014038, + "mean_kl": 0.7019644379615784, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.7560975551605225 + }, + { + "candidate_nll": 0.9157043099403381, + "code_tokens": 67, + "cvar95_kl": 4.738999366760254, + "delta_nll": 0.5510225296020508, + "max_kl": 6.981268405914307, + "mean_kl": 0.7067411541938782, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.7761194109916687 + }, + { + "candidate_nll": 2.416775703430176, + "code_tokens": 76, + "cvar95_kl": 3.672361373901367, + "delta_nll": 1.0265157222747803, + "max_kl": 4.410040378570557, + "mean_kl": 0.9892578125, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.5657894611358643 + }, + { + "candidate_nll": 1.694367527961731, + "code_tokens": 137, + "cvar95_kl": 4.1326823234558105, + "delta_nll": 1.0585148334503174, + "max_kl": 4.647824764251709, + "mean_kl": 0.9976497888565063, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.6715328693389893 + } + ], + "policy": { + "default_bits": 4, + "name": "mse_selected_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 1.9546477794647217, + "median": 0.7915209531784058, + "minimum": -0.09443438053131104, + "q25": 0.5939276367425919, + "q75": 1.0005292445421219 + }, + "task_macro": { + "candidate_nll": 1.8177594603300096, + "delta_nll": 0.8037128749489785, + "mean_kl": 0.8153626573383808, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.6911910179853439 + }, + "token_weighted": { + "candidate_nll": 1.827401041984558, + "cvar95_kl": 4.839138984680176, + "delta_nll": 0.9105634689331055, + "max_kl": 12.522397994995117, + "mean_kl": 0.9145802855491638, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.665189802646637 + } + }, + "random_l13_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.429121050834656, + "macro_mean_kl": 2.5537083740234374, + "macro_top1_agreement": 0.401166699051857, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 2.8439346017837526, + "macro_mean_kl": 2.9936788730621338, + "macro_top1_agreement": 0.3380293816328049, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 2.9676756620407105, + "macro_mean_kl": 3.023242858886719, + "macro_top1_agreement": 0.3332699118256569, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 3.1260268945693968, + "macro_mean_kl": 3.1511305227279665, + "macro_top1_agreement": 0.32465201371908187, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 3.6329097747802734, + "code_tokens": 98, + "cvar95_kl": 8.055329322814941, + "delta_nll": 2.9060540199279785, + "max_kl": 8.748248100280762, + "mean_kl": 3.231685161590576, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.3163265287876129 + }, + { + "candidate_nll": 4.0525336265563965, + "code_tokens": 19, + "cvar95_kl": 9.52585506439209, + "delta_nll": 2.893120527267456, + "max_kl": 9.52585506439209, + "mean_kl": 3.0580568313598633, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 3.856330633163452, + "code_tokens": 39, + "cvar95_kl": 9.715455055236816, + "delta_nll": 2.8409814834594727, + "max_kl": 12.288392066955566, + "mean_kl": 3.2588510513305664, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 4.704852104187012, + "code_tokens": 24, + "cvar95_kl": 7.532789707183838, + "delta_nll": 3.2585625648498535, + "max_kl": 8.205902099609375, + "mean_kl": 3.1335389614105225, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.6999106407165527, + "code_tokens": 25, + "cvar95_kl": 8.477755546569824, + "delta_nll": 2.2921524047851562, + "max_kl": 9.722190856933594, + "mean_kl": 2.3375277519226074, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 4.223059177398682, + "code_tokens": 55, + "cvar95_kl": 9.37523078918457, + "delta_nll": 3.3274476528167725, + "max_kl": 12.28735065460205, + "mean_kl": 3.2740306854248047, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.34545454382896423 + }, + { + "candidate_nll": 3.235170841217041, + "code_tokens": 16, + "cvar95_kl": 6.7525715827941895, + "delta_nll": 2.4057302474975586, + "max_kl": 6.7525715827941895, + "mean_kl": 2.21394681930542, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.4375 + }, + { + "candidate_nll": 4.798648834228516, + "code_tokens": 199, + "cvar95_kl": 9.963300704956055, + "delta_nll": 3.5767555236816406, + "max_kl": 13.566315650939941, + "mean_kl": 3.648836851119995, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.31658291816711426 + }, + { + "candidate_nll": 4.1447014808654785, + "code_tokens": 28, + "cvar95_kl": 5.878242492675781, + "delta_nll": 2.5786232948303223, + "max_kl": 5.971217155456543, + "mean_kl": 2.398080348968506, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.4642857015132904 + }, + { + "candidate_nll": 3.737274408340454, + "code_tokens": 87, + "cvar95_kl": 7.202219486236572, + "delta_nll": 2.8642916679382324, + "max_kl": 8.801186561584473, + "mean_kl": 2.866079807281494, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.29885056614875793 + }, + { + "candidate_nll": 3.7016096115112305, + "code_tokens": 35, + "cvar95_kl": 9.176436424255371, + "delta_nll": 2.4686455726623535, + "max_kl": 9.343332290649414, + "mean_kl": 3.3765621185302734, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.34285715222358704 + }, + { + "candidate_nll": 3.6574254035949707, + "code_tokens": 62, + "cvar95_kl": 8.88882827758789, + "delta_nll": 3.0181338787078857, + "max_kl": 11.19011402130127, + "mean_kl": 2.9212141036987305, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 5.144183158874512, + "code_tokens": 58, + "cvar95_kl": 10.321444511413574, + "delta_nll": 4.039125442504883, + "max_kl": 10.891624450683594, + "mean_kl": 3.908536911010742, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 3.933481454849243, + "code_tokens": 79, + "cvar95_kl": 7.963977336883545, + "delta_nll": 2.774433135986328, + "max_kl": 8.332808494567871, + "mean_kl": 3.0690970420837402, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.27848100662231445 + }, + { + "candidate_nll": 3.811854362487793, + "code_tokens": 58, + "cvar95_kl": 7.584588527679443, + "delta_nll": 2.7187345027923584, + "max_kl": 9.210746765136719, + "mean_kl": 2.885810375213623, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.3448275923728943 + }, + { + "candidate_nll": 4.7106451988220215, + "code_tokens": 48, + "cvar95_kl": 7.531435489654541, + "delta_nll": 3.548447370529175, + "max_kl": 7.878511428833008, + "mean_kl": 3.7450246810913086, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 4.520447731018066, + "code_tokens": 37, + "cvar95_kl": 9.23340129852295, + "delta_nll": 3.395015239715576, + "max_kl": 10.187602996826172, + "mean_kl": 3.20151424407959, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.4303460121154785, + "code_tokens": 61, + "cvar95_kl": 7.320745468139648, + "delta_nll": 2.7229387760162354, + "max_kl": 9.039068222045898, + "mean_kl": 2.7306323051452637, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.3442623019218445 + }, + { + "candidate_nll": 4.004060745239258, + "code_tokens": 77, + "cvar95_kl": 7.712281703948975, + "delta_nll": 3.2343411445617676, + "max_kl": 9.30237865447998, + "mean_kl": 3.1092755794525146, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.25974026322364807 + }, + { + "candidate_nll": 4.447798728942871, + "code_tokens": 98, + "cvar95_kl": 8.30854320526123, + "delta_nll": 3.426095485687256, + "max_kl": 11.127065658569336, + "mean_kl": 3.287552833557129, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.29591837525367737 + }, + { + "candidate_nll": 5.255676746368408, + "code_tokens": 162, + "cvar95_kl": 10.030423164367676, + "delta_nll": 4.670525550842285, + "max_kl": 11.024353981018066, + "mean_kl": 4.728553295135498, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 4.3106818199157715, + "code_tokens": 103, + "cvar95_kl": 9.602532386779785, + "delta_nll": 3.566215753555298, + "max_kl": 10.715997695922852, + "mean_kl": 3.5995678901672363, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.3009708821773529 + }, + { + "candidate_nll": 5.0264081954956055, + "code_tokens": 73, + "cvar95_kl": 7.607296466827393, + "delta_nll": 3.707437753677368, + "max_kl": 7.930570125579834, + "mean_kl": 3.5200839042663574, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.31506848335266113 + }, + { + "candidate_nll": 4.5867600440979, + "code_tokens": 119, + "cvar95_kl": 9.679557800292969, + "delta_nll": 3.6037864685058594, + "max_kl": 13.266568183898926, + "mean_kl": 3.735306739807129, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.21848739683628082 + }, + { + "candidate_nll": 2.306497097015381, + "code_tokens": 17, + "cvar95_kl": 6.726556777954102, + "delta_nll": 1.1169737577438354, + "max_kl": 6.726556777954102, + "mean_kl": 2.0852298736572266, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 4.036786079406738, + "code_tokens": 56, + "cvar95_kl": 8.165696144104004, + "delta_nll": 2.611084461212158, + "max_kl": 8.554071426391602, + "mean_kl": 2.6784827709198, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.23885440826416, + "code_tokens": 57, + "cvar95_kl": 8.072587013244629, + "delta_nll": 2.771493911743164, + "max_kl": 8.77849292755127, + "mean_kl": 2.725067138671875, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.4385964870452881 + }, + { + "candidate_nll": 3.585200548171997, + "code_tokens": 63, + "cvar95_kl": 8.381641387939453, + "delta_nll": 2.9524552822113037, + "max_kl": 9.101977348327637, + "mean_kl": 2.978990316390991, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 5.2038254737854, + "code_tokens": 200, + "cvar95_kl": 11.655916213989258, + "delta_nll": 4.365890026092529, + "max_kl": 13.550088882446289, + "mean_kl": 4.439868927001953, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.2150000035762787 + }, + { + "candidate_nll": 3.7848496437072754, + "code_tokens": 30, + "cvar95_kl": 9.424089431762695, + "delta_nll": 2.7006468772888184, + "max_kl": 10.23027229309082, + "mean_kl": 2.917102575302124, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.7932515144348145, + "code_tokens": 28, + "cvar95_kl": 11.703697204589844, + "delta_nll": 3.1769003868103027, + "max_kl": 13.03071403503418, + "mean_kl": 2.920898199081421, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.5357142686843872 + }, + { + "candidate_nll": 3.581470489501953, + "code_tokens": 26, + "cvar95_kl": 6.491343021392822, + "delta_nll": 2.810513973236084, + "max_kl": 6.5482330322265625, + "mean_kl": 2.6113665103912354, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.1451494693756104, + "code_tokens": 54, + "cvar95_kl": 7.028079986572266, + "delta_nll": 2.3122081756591797, + "max_kl": 10.293176651000977, + "mean_kl": 2.324183225631714, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.334563970565796, + "code_tokens": 47, + "cvar95_kl": 6.204320907592773, + "delta_nll": 2.3778278827667236, + "max_kl": 6.373838901519775, + "mean_kl": 2.454688787460327, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.40425533056259155 + }, + { + "candidate_nll": 4.217667102813721, + "code_tokens": 93, + "cvar95_kl": 8.306978225708008, + "delta_nll": 3.483137607574463, + "max_kl": 9.250154495239258, + "mean_kl": 3.2951619625091553, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.37634408473968506 + }, + { + "candidate_nll": 3.6750497817993164, + "code_tokens": 30, + "cvar95_kl": 8.563119888305664, + "delta_nll": 2.8156752586364746, + "max_kl": 9.715740203857422, + "mean_kl": 2.856915235519409, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.315735816955566, + "code_tokens": 89, + "cvar95_kl": 7.7267866134643555, + "delta_nll": 3.247232437133789, + "max_kl": 9.064905166625977, + "mean_kl": 3.216982364654541, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.2584269642829895 + }, + { + "candidate_nll": 4.4749579429626465, + "code_tokens": 70, + "cvar95_kl": 8.525197982788086, + "delta_nll": 3.5107085704803467, + "max_kl": 9.563936233520508, + "mean_kl": 3.6164729595184326, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.14351749420166, + "code_tokens": 26, + "cvar95_kl": 6.899682521820068, + "delta_nll": 2.530290365219116, + "max_kl": 7.468336582183838, + "mean_kl": 2.9295411109924316, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.606774091720581, + "code_tokens": 48, + "cvar95_kl": 8.648411750793457, + "delta_nll": 2.6490671634674072, + "max_kl": 11.542004585266113, + "mean_kl": 2.7555952072143555, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.345219612121582, + "code_tokens": 29, + "cvar95_kl": 6.314007759094238, + "delta_nll": 2.47607159614563, + "max_kl": 6.442813396453857, + "mean_kl": 2.532736301422119, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 4.100141525268555, + "code_tokens": 17, + "cvar95_kl": 7.508518218994141, + "delta_nll": 2.986978530883789, + "max_kl": 7.508518218994141, + "mean_kl": 2.760864496231079, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 3.1212477684020996, + "code_tokens": 37, + "cvar95_kl": 7.198155403137207, + "delta_nll": 2.2010316848754883, + "max_kl": 7.721660137176514, + "mean_kl": 2.3116538524627686, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 4.5929274559021, + "code_tokens": 121, + "cvar95_kl": 11.944608688354492, + "delta_nll": 4.081525802612305, + "max_kl": 18.288694381713867, + "mean_kl": 4.081521987915039, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.8529868125915527, + "code_tokens": 30, + "cvar95_kl": 7.27288818359375, + "delta_nll": 2.4541430473327637, + "max_kl": 8.80015754699707, + "mean_kl": 2.747617483139038, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.0221357345581055, + "code_tokens": 74, + "cvar95_kl": 8.039169311523438, + "delta_nll": 3.123663902282715, + "max_kl": 11.639261245727539, + "mean_kl": 3.0636556148529053, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.4953086376190186, + "code_tokens": 53, + "cvar95_kl": 7.868740081787109, + "delta_nll": 2.6608848571777344, + "max_kl": 8.419306755065918, + "mean_kl": 2.551097869873047, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.3962264060974121 + }, + { + "candidate_nll": 3.7510972023010254, + "code_tokens": 21, + "cvar95_kl": 5.211759567260742, + "delta_nll": 1.8917014598846436, + "max_kl": 5.216395854949951, + "mean_kl": 2.293757200241089, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 2.7207303047180176, + "code_tokens": 20, + "cvar95_kl": 7.142923355102539, + "delta_nll": 1.3583166599273682, + "max_kl": 7.142923355102539, + "mean_kl": 1.4658582210540771, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 3.020242929458618, + "code_tokens": 122, + "cvar95_kl": 5.629207134246826, + "delta_nll": 2.201244592666626, + "max_kl": 7.803311347961426, + "mean_kl": 2.2799293994903564, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.41803279519081116 + }, + { + "candidate_nll": 4.194502830505371, + "code_tokens": 94, + "cvar95_kl": 8.624273300170898, + "delta_nll": 3.354362964630127, + "max_kl": 8.783571243286133, + "mean_kl": 3.4656074047088623, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.27659574151039124 + }, + { + "candidate_nll": 2.9010698795318604, + "code_tokens": 12, + "cvar95_kl": 5.740634441375732, + "delta_nll": 1.8357683420181274, + "max_kl": 5.740634441375732, + "mean_kl": 1.627508282661438, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.6088385581970215, + "code_tokens": 38, + "cvar95_kl": 7.848127365112305, + "delta_nll": 3.102438449859619, + "max_kl": 8.04623794555664, + "mean_kl": 3.6178483963012695, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 2.067038059234619, + "code_tokens": 58, + "cvar95_kl": 5.395915985107422, + "delta_nll": 1.1622509956359863, + "max_kl": 6.82765531539917, + "mean_kl": 1.1914799213409424, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.6034482717514038 + }, + { + "candidate_nll": 4.598418712615967, + "code_tokens": 54, + "cvar95_kl": 9.453789710998535, + "delta_nll": 4.00215482711792, + "max_kl": 10.124591827392578, + "mean_kl": 3.979879379272461, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.4902024269104004, + "code_tokens": 40, + "cvar95_kl": 6.56636381149292, + "delta_nll": 2.5586624145507812, + "max_kl": 7.1336565017700195, + "mean_kl": 2.771625518798828, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 3.301248788833618, + "code_tokens": 115, + "cvar95_kl": 6.596490859985352, + "delta_nll": 2.788702964782715, + "max_kl": 9.216168403625488, + "mean_kl": 2.7491705417633057, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.313043475151062 + }, + { + "candidate_nll": 3.8309569358825684, + "code_tokens": 60, + "cvar95_kl": 9.20372486114502, + "delta_nll": 3.3063433170318604, + "max_kl": 10.425722122192383, + "mean_kl": 3.3528902530670166, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 4.018204689025879, + "code_tokens": 124, + "cvar95_kl": 7.826095104217529, + "delta_nll": 3.2482175827026367, + "max_kl": 10.56296157836914, + "mean_kl": 3.2017760276794434, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.2822580635547638 + }, + { + "candidate_nll": 5.139254570007324, + "code_tokens": 91, + "cvar95_kl": 10.411510467529297, + "delta_nll": 4.368505954742432, + "max_kl": 11.577062606811523, + "mean_kl": 4.321128845214844, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.17582418024539948 + }, + { + "candidate_nll": 5.656702518463135, + "code_tokens": 125, + "cvar95_kl": 12.191720008850098, + "delta_nll": 4.836785316467285, + "max_kl": 14.277709007263184, + "mean_kl": 4.909350395202637, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.25600001215934753 + }, + { + "candidate_nll": 2.385603666305542, + "code_tokens": 26, + "cvar95_kl": 4.303977966308594, + "delta_nll": 1.2497271299362183, + "max_kl": 4.483548164367676, + "mean_kl": 1.8951040506362915, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.86462140083313, + "code_tokens": 24, + "cvar95_kl": 7.119986534118652, + "delta_nll": 1.6241962909698486, + "max_kl": 7.214775085449219, + "mean_kl": 2.0531961917877197, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.6262729167938232, + "code_tokens": 180, + "cvar95_kl": 8.493305206298828, + "delta_nll": 2.7720911502838135, + "max_kl": 12.880755424499512, + "mean_kl": 2.8170082569122314, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 5.156355857849121, + "code_tokens": 40, + "cvar95_kl": 10.091594696044922, + "delta_nll": 3.7815473079681396, + "max_kl": 10.84343433380127, + "mean_kl": 3.9381966590881348, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.9856724739074707, + "code_tokens": 72, + "cvar95_kl": 7.924861431121826, + "delta_nll": 3.0106756687164307, + "max_kl": 9.3711519241333, + "mean_kl": 2.9844560623168945, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.912897825241089, + "code_tokens": 20, + "cvar95_kl": 3.7060303688049316, + "delta_nll": 1.3353266716003418, + "max_kl": 3.7060303688049316, + "mean_kl": 1.8768047094345093, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 4.065054893493652, + "code_tokens": 76, + "cvar95_kl": 7.158243179321289, + "delta_nll": 3.0744104385375977, + "max_kl": 8.633698463439941, + "mean_kl": 3.1051504611968994, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.30263158679008484 + }, + { + "candidate_nll": 4.293069839477539, + "code_tokens": 42, + "cvar95_kl": 8.778841972351074, + "delta_nll": 2.921328544616699, + "max_kl": 9.242888450622559, + "mean_kl": 2.959480047225952, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 2.43483567237854, + "code_tokens": 28, + "cvar95_kl": 6.015645980834961, + "delta_nll": 1.8516125679016113, + "max_kl": 6.2148332595825195, + "mean_kl": 1.7825689315795898, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.5357142686843872 + }, + { + "candidate_nll": 4.4054179191589355, + "code_tokens": 58, + "cvar95_kl": 9.272234916687012, + "delta_nll": 3.371941089630127, + "max_kl": 10.033976554870605, + "mean_kl": 3.4207990169525146, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.36206895112991333 + }, + { + "candidate_nll": 3.3591840267181396, + "code_tokens": 25, + "cvar95_kl": 6.774226665496826, + "delta_nll": 2.490534543991089, + "max_kl": 7.206204891204834, + "mean_kl": 2.794959783554077, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.596789836883545, + "code_tokens": 88, + "cvar95_kl": 8.229361534118652, + "delta_nll": 3.631782054901123, + "max_kl": 8.97445297241211, + "mean_kl": 3.6154417991638184, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.5073184967041016, + "code_tokens": 42, + "cvar95_kl": 6.333248138427734, + "delta_nll": 2.3798012733459473, + "max_kl": 7.599839210510254, + "mean_kl": 2.2412917613983154, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.615652084350586, + "code_tokens": 24, + "cvar95_kl": 6.822283744812012, + "delta_nll": 2.7961013317108154, + "max_kl": 7.321934700012207, + "mean_kl": 2.752093553543091, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 2.8708908557891846, + "code_tokens": 24, + "cvar95_kl": 5.216815948486328, + "delta_nll": 2.1860194206237793, + "max_kl": 5.318026542663574, + "mean_kl": 2.189774990081787, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.1849944591522217, + "code_tokens": 43, + "cvar95_kl": 7.89160680770874, + "delta_nll": 2.1178154945373535, + "max_kl": 8.703787803649902, + "mean_kl": 2.119652271270752, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.5581395626068115 + }, + { + "candidate_nll": 3.3584134578704834, + "code_tokens": 24, + "cvar95_kl": 8.97757339477539, + "delta_nll": 2.472193717956543, + "max_kl": 9.10887336730957, + "mean_kl": 2.697943925857544, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 3.575653314590454, + "code_tokens": 14, + "cvar95_kl": 4.864448070526123, + "delta_nll": 2.2630276679992676, + "max_kl": 4.864448070526123, + "mean_kl": 2.399156093597412, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.610832452774048, + "code_tokens": 41, + "cvar95_kl": 7.454631328582764, + "delta_nll": 3.1197891235351562, + "max_kl": 8.85711669921875, + "mean_kl": 2.80129075050354, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.3414634168148041 + }, + { + "candidate_nll": 4.055644989013672, + "code_tokens": 32, + "cvar95_kl": 7.741008758544922, + "delta_nll": 3.067305088043213, + "max_kl": 8.03824234008789, + "mean_kl": 2.9247632026672363, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.34375 + }, + { + "candidate_nll": 2.6370434761047363, + "code_tokens": 53, + "cvar95_kl": 5.3728346824646, + "delta_nll": 1.8753485679626465, + "max_kl": 5.665938854217529, + "mean_kl": 1.7014213800430298, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.5283018946647644 + }, + { + "candidate_nll": 3.4696123600006104, + "code_tokens": 51, + "cvar95_kl": 8.676712036132812, + "delta_nll": 3.0913212299346924, + "max_kl": 9.638703346252441, + "mean_kl": 3.090484857559204, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.3921568691730499 + }, + { + "candidate_nll": 4.265464782714844, + "code_tokens": 36, + "cvar95_kl": 6.738497257232666, + "delta_nll": 2.5733094215393066, + "max_kl": 6.7621684074401855, + "mean_kl": 2.8429579734802246, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 3.2938711643218994, + "code_tokens": 24, + "cvar95_kl": 4.951527118682861, + "delta_nll": 1.7331188917160034, + "max_kl": 4.965203762054443, + "mean_kl": 1.7760009765625, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.762222766876221, + "code_tokens": 39, + "cvar95_kl": 9.439563751220703, + "delta_nll": 3.4478883743286133, + "max_kl": 9.966629028320312, + "mean_kl": 3.930797576904297, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 5.773906707763672, + "code_tokens": 84, + "cvar95_kl": 10.95777702331543, + "delta_nll": 5.075494289398193, + "max_kl": 14.16593074798584, + "mean_kl": 5.070546627044678, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 2.941472053527832, + "code_tokens": 32, + "cvar95_kl": 6.102725028991699, + "delta_nll": 2.0187039375305176, + "max_kl": 6.622569561004639, + "mean_kl": 1.9183974266052246, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.4375 + }, + { + "candidate_nll": 4.423862457275391, + "code_tokens": 19, + "cvar95_kl": 8.772175788879395, + "delta_nll": 3.113698959350586, + "max_kl": 8.772175788879395, + "mean_kl": 2.8887062072753906, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.5263158082962036 + }, + { + "candidate_nll": 3.6941633224487305, + "code_tokens": 50, + "cvar95_kl": 8.257587432861328, + "delta_nll": 2.8761866092681885, + "max_kl": 8.958593368530273, + "mean_kl": 2.759572982788086, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.198176383972168, + "code_tokens": 71, + "cvar95_kl": 7.605610370635986, + "delta_nll": 2.6983628273010254, + "max_kl": 12.483941078186035, + "mean_kl": 2.6213016510009766, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.39436620473861694 + }, + { + "candidate_nll": 4.323483467102051, + "code_tokens": 28, + "cvar95_kl": 9.144611358642578, + "delta_nll": 2.9355149269104004, + "max_kl": 9.313859939575195, + "mean_kl": 3.2761404514312744, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 2.673621416091919, + "code_tokens": 76, + "cvar95_kl": 5.355823516845703, + "delta_nll": 1.804360032081604, + "max_kl": 5.731517314910889, + "mean_kl": 1.7973994016647339, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 3.9000189304351807, + "code_tokens": 34, + "cvar95_kl": 6.530857086181641, + "delta_nll": 2.9374337196350098, + "max_kl": 6.536062240600586, + "mean_kl": 3.056819200515747, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 4.025352954864502, + "code_tokens": 11, + "cvar95_kl": 4.060826301574707, + "delta_nll": 1.8779611587524414, + "max_kl": 4.060826301574707, + "mean_kl": 1.7771077156066895, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 3.344594717025757, + "code_tokens": 31, + "cvar95_kl": 6.578172206878662, + "delta_nll": 1.928327202796936, + "max_kl": 7.1022443771362305, + "mean_kl": 2.437537670135498, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 3.0884499549865723, + "code_tokens": 118, + "cvar95_kl": 6.092535495758057, + "delta_nll": 2.241673707962036, + "max_kl": 6.586357116699219, + "mean_kl": 2.269760847091675, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.3813559412956238 + }, + { + "candidate_nll": 2.784593105316162, + "code_tokens": 60, + "cvar95_kl": 10.015922546386719, + "delta_nll": 2.008897304534912, + "max_kl": 11.993035316467285, + "mean_kl": 2.2392003536224365, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.021513938903809, + "code_tokens": 47, + "cvar95_kl": 7.228011608123779, + "delta_nll": 3.1260640621185303, + "max_kl": 7.850344657897949, + "mean_kl": 3.165506601333618, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.27659574151039124 + }, + { + "candidate_nll": 3.7023892402648926, + "code_tokens": 84, + "cvar95_kl": 8.686578750610352, + "delta_nll": 2.550374984741211, + "max_kl": 12.988542556762695, + "mean_kl": 2.6913888454437256, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.4047619104385376 + }, + { + "candidate_nll": 4.629031181335449, + "code_tokens": 27, + "cvar95_kl": 10.33298110961914, + "delta_nll": 3.635004997253418, + "max_kl": 10.367058753967285, + "mean_kl": 3.521436929702759, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.6790878772735596, + "code_tokens": 19, + "cvar95_kl": 5.439894199371338, + "delta_nll": 2.067180633544922, + "max_kl": 5.439894199371338, + "mean_kl": 2.113597869873047, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 4.1463704109191895, + "code_tokens": 114, + "cvar95_kl": 9.066641807556152, + "delta_nll": 3.3733577728271484, + "max_kl": 11.420714378356934, + "mean_kl": 3.5716664791107178, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 4.659701347351074, + "code_tokens": 40, + "cvar95_kl": 9.886030197143555, + "delta_nll": 2.9318079948425293, + "max_kl": 10.078507423400879, + "mean_kl": 3.274716854095459, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.700169324874878, + "code_tokens": 25, + "cvar95_kl": 6.042583465576172, + "delta_nll": 2.214491128921509, + "max_kl": 7.156935214996338, + "mean_kl": 2.2528998851776123, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 3.74839448928833, + "code_tokens": 23, + "cvar95_kl": 9.777359008789062, + "delta_nll": 2.977705478668213, + "max_kl": 11.06005573272705, + "mean_kl": 2.7939860820770264, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.47826087474823 + }, + { + "candidate_nll": 4.821878433227539, + "code_tokens": 77, + "cvar95_kl": 9.397276878356934, + "delta_nll": 3.8055288791656494, + "max_kl": 11.298822402954102, + "mean_kl": 3.7361068725585938, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.29870128631591797 + }, + { + "candidate_nll": 4.026378631591797, + "code_tokens": 19, + "cvar95_kl": 7.942623615264893, + "delta_nll": 2.918713092803955, + "max_kl": 7.942623615264893, + "mean_kl": 2.6485815048217773, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.997559070587158, + "code_tokens": 37, + "cvar95_kl": 6.9716362953186035, + "delta_nll": 3.014315366744995, + "max_kl": 7.359827518463135, + "mean_kl": 3.1137869358062744, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 3.83298921585083, + "code_tokens": 33, + "cvar95_kl": 6.3604865074157715, + "delta_nll": 2.449824333190918, + "max_kl": 7.376956939697266, + "mean_kl": 2.700761079788208, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 3.9677703380584717, + "code_tokens": 95, + "cvar95_kl": 8.145652770996094, + "delta_nll": 3.307929039001465, + "max_kl": 10.57262134552002, + "mean_kl": 3.2266299724578857, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.3263157904148102 + }, + { + "candidate_nll": 3.618833303451538, + "code_tokens": 125, + "cvar95_kl": 6.9218668937683105, + "delta_nll": 2.605672836303711, + "max_kl": 7.870822906494141, + "mean_kl": 2.6767094135284424, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 5.244914531707764, + "code_tokens": 168, + "cvar95_kl": 10.64597225189209, + "delta_nll": 4.531866073608398, + "max_kl": 11.928614616394043, + "mean_kl": 4.474259853363037, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 3.345757484436035, + "code_tokens": 31, + "cvar95_kl": 11.152687072753906, + "delta_nll": 2.030355453491211, + "max_kl": 13.132161140441895, + "mean_kl": 2.731574773788452, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.4193548262119293 + }, + { + "candidate_nll": 4.053913116455078, + "code_tokens": 83, + "cvar95_kl": 8.822319030761719, + "delta_nll": 3.292206287384033, + "max_kl": 9.76469898223877, + "mean_kl": 3.2597439289093018, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.3734939694404602 + }, + { + "candidate_nll": 4.7042341232299805, + "code_tokens": 50, + "cvar95_kl": 8.487092018127441, + "delta_nll": 4.004291534423828, + "max_kl": 10.377372741699219, + "mean_kl": 3.898825168609619, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 2.982070207595825, + "code_tokens": 69, + "cvar95_kl": 7.078595161437988, + "delta_nll": 2.6148881912231445, + "max_kl": 7.534358501434326, + "mean_kl": 2.6491971015930176, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.43478259444236755 + }, + { + "candidate_nll": 4.0026140213012695, + "code_tokens": 50, + "cvar95_kl": 7.962657928466797, + "delta_nll": 2.917717933654785, + "max_kl": 8.53264045715332, + "mean_kl": 3.1669702529907227, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.551426887512207, + "code_tokens": 161, + "cvar95_kl": 7.941140174865723, + "delta_nll": 3.547645092010498, + "max_kl": 9.720252990722656, + "mean_kl": 3.5097925662994385, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 5.19462776184082, + "code_tokens": 53, + "cvar95_kl": 11.295387268066406, + "delta_nll": 4.538851737976074, + "max_kl": 13.760066032409668, + "mean_kl": 4.406388759613037, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 4.153054237365723, + "code_tokens": 97, + "cvar95_kl": 8.83063793182373, + "delta_nll": 3.707526206970215, + "max_kl": 9.661468505859375, + "mean_kl": 3.490177631378174, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.39175257086753845 + }, + { + "candidate_nll": 3.2872931957244873, + "code_tokens": 22, + "cvar95_kl": 7.65336799621582, + "delta_nll": 1.973480224609375, + "max_kl": 9.718481063842773, + "mean_kl": 2.1864054203033447, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 2.9211440086364746, + "code_tokens": 34, + "cvar95_kl": 11.849899291992188, + "delta_nll": 2.0522398948669434, + "max_kl": 11.934725761413574, + "mean_kl": 2.5961644649505615, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 4.217153072357178, + "code_tokens": 81, + "cvar95_kl": 8.469791412353516, + "delta_nll": 3.3585097789764404, + "max_kl": 9.037009239196777, + "mean_kl": 3.547222375869751, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.2469135820865631 + }, + { + "candidate_nll": 2.2118759155273438, + "code_tokens": 17, + "cvar95_kl": 4.3246941566467285, + "delta_nll": 0.9497725963592529, + "max_kl": 4.3246941566467285, + "mean_kl": 1.0584627389907837, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 4.379289150238037, + "code_tokens": 181, + "cvar95_kl": 7.7154221534729, + "delta_nll": 3.5540318489074707, + "max_kl": 8.841346740722656, + "mean_kl": 3.5654566287994385, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.28729280829429626 + }, + { + "candidate_nll": 4.032923221588135, + "code_tokens": 63, + "cvar95_kl": 7.518209934234619, + "delta_nll": 3.0525147914886475, + "max_kl": 8.059112548828125, + "mean_kl": 3.1469972133636475, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.2698412835597992 + }, + { + "candidate_nll": 1.867303490638733, + "code_tokens": 35, + "cvar95_kl": 4.24521541595459, + "delta_nll": 1.1737804412841797, + "max_kl": 4.809912204742432, + "mean_kl": 1.3842389583587646, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 3.7477457523345947, + "code_tokens": 25, + "cvar95_kl": 5.670744895935059, + "delta_nll": 2.942958354949951, + "max_kl": 6.017920970916748, + "mean_kl": 2.6128053665161133, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.925334453582764, + "code_tokens": 60, + "cvar95_kl": 9.437179565429688, + "delta_nll": 3.8369688987731934, + "max_kl": 10.04562759399414, + "mean_kl": 4.074987411499023, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 4.455182075500488, + "code_tokens": 86, + "cvar95_kl": 9.2921724319458, + "delta_nll": 3.5435352325439453, + "max_kl": 11.45085334777832, + "mean_kl": 3.5924274921417236, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 4.116549015045166, + "code_tokens": 40, + "cvar95_kl": 10.161130905151367, + "delta_nll": 3.190383195877075, + "max_kl": 10.777144432067871, + "mean_kl": 3.3825390338897705, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 3.4999449253082275, + "code_tokens": 27, + "cvar95_kl": 6.04339599609375, + "delta_nll": 1.936384677886963, + "max_kl": 6.376384735107422, + "mean_kl": 1.9603098630905151, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.0503644943237305, + "code_tokens": 52, + "cvar95_kl": 6.841410160064697, + "delta_nll": 2.9607815742492676, + "max_kl": 7.694720268249512, + "mean_kl": 3.1457455158233643, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 3.1272923946380615, + "code_tokens": 71, + "cvar95_kl": 6.935268402099609, + "delta_nll": 2.38541841506958, + "max_kl": 7.791135311126709, + "mean_kl": 2.4393527507781982, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.3802816867828369 + }, + { + "candidate_nll": 3.695378303527832, + "code_tokens": 25, + "cvar95_kl": 7.637895107269287, + "delta_nll": 2.593526601791382, + "max_kl": 7.696938514709473, + "mean_kl": 2.6167855262756348, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 2.285973072052002, + "code_tokens": 103, + "cvar95_kl": 5.759994983673096, + "delta_nll": 1.6914774179458618, + "max_kl": 7.878668308258057, + "mean_kl": 1.805801272392273, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.5242718458175659 + }, + { + "candidate_nll": 3.7893221378326416, + "code_tokens": 116, + "cvar95_kl": 7.903413772583008, + "delta_nll": 2.7317841053009033, + "max_kl": 9.19098949432373, + "mean_kl": 2.793156147003174, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.3534482717514038 + }, + { + "candidate_nll": 3.815580368041992, + "code_tokens": 107, + "cvar95_kl": 7.082277774810791, + "delta_nll": 3.167234182357788, + "max_kl": 8.54782772064209, + "mean_kl": 3.1968603134155273, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.3177570104598999 + }, + { + "candidate_nll": 3.6977083683013916, + "code_tokens": 54, + "cvar95_kl": 6.410790920257568, + "delta_nll": 2.7546706199645996, + "max_kl": 7.302250385284424, + "mean_kl": 2.7273097038269043, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.24074074625968933 + }, + { + "candidate_nll": 4.312080383300781, + "code_tokens": 46, + "cvar95_kl": 9.64364242553711, + "delta_nll": 3.6786880493164062, + "max_kl": 11.447850227355957, + "mean_kl": 3.5165374279022217, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.163379669189453, + "code_tokens": 170, + "cvar95_kl": 9.395096778869629, + "delta_nll": 3.834232807159424, + "max_kl": 14.358067512512207, + "mean_kl": 3.7710328102111816, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.2705882489681244 + }, + { + "candidate_nll": 3.693528175354004, + "code_tokens": 45, + "cvar95_kl": 6.260714054107666, + "delta_nll": 2.266835927963257, + "max_kl": 7.382214546203613, + "mean_kl": 2.4521541595458984, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.320131301879883, + "code_tokens": 24, + "cvar95_kl": 7.214895725250244, + "delta_nll": 2.91182804107666, + "max_kl": 8.220372200012207, + "mean_kl": 3.0849807262420654, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9400627613067627, + "code_tokens": 78, + "cvar95_kl": 7.752481460571289, + "delta_nll": 2.7781028747558594, + "max_kl": 8.434541702270508, + "mean_kl": 2.828617572784424, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.39743590354919434 + }, + { + "candidate_nll": 3.939624309539795, + "code_tokens": 33, + "cvar95_kl": 7.490231990814209, + "delta_nll": 3.3090672492980957, + "max_kl": 8.383538246154785, + "mean_kl": 3.121694326400757, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.728476524353027, + "code_tokens": 32, + "cvar95_kl": 9.435108184814453, + "delta_nll": 3.612806797027588, + "max_kl": 10.356942176818848, + "mean_kl": 3.424196720123291, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.34375 + }, + { + "candidate_nll": 5.064863204956055, + "code_tokens": 77, + "cvar95_kl": 8.474512100219727, + "delta_nll": 4.076113224029541, + "max_kl": 9.04771900177002, + "mean_kl": 3.907357692718506, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.16883116960525513 + }, + { + "candidate_nll": 2.2948801517486572, + "code_tokens": 222, + "cvar95_kl": 7.114038467407227, + "delta_nll": 1.8726295232772827, + "max_kl": 10.239022254943848, + "mean_kl": 1.9096680879592896, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.5855855941772461 + }, + { + "candidate_nll": 3.752305030822754, + "code_tokens": 86, + "cvar95_kl": 7.7425994873046875, + "delta_nll": 2.7037100791931152, + "max_kl": 9.360763549804688, + "mean_kl": 2.850666046142578, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 2.747559070587158, + "code_tokens": 31, + "cvar95_kl": 7.171835899353027, + "delta_nll": 2.4135096073150635, + "max_kl": 7.814181804656982, + "mean_kl": 2.371086835861206, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.4838709533214569 + }, + { + "candidate_nll": 3.3065571784973145, + "code_tokens": 34, + "cvar95_kl": 6.639692306518555, + "delta_nll": 2.794957160949707, + "max_kl": 7.208677768707275, + "mean_kl": 2.800832748413086, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 4.454214572906494, + "code_tokens": 38, + "cvar95_kl": 7.788163661956787, + "delta_nll": 3.0667543411254883, + "max_kl": 7.891308307647705, + "mean_kl": 3.1442136764526367, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 3.834498643875122, + "code_tokens": 93, + "cvar95_kl": 7.7158026695251465, + "delta_nll": 2.8714540004730225, + "max_kl": 8.839159965515137, + "mean_kl": 2.9961981773376465, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.34408602118492126 + }, + { + "candidate_nll": 4.056421756744385, + "code_tokens": 70, + "cvar95_kl": 8.267263412475586, + "delta_nll": 3.2778160572052, + "max_kl": 8.97049617767334, + "mean_kl": 3.267692804336548, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.27142858505249023 + }, + { + "candidate_nll": 5.376054763793945, + "code_tokens": 67, + "cvar95_kl": 10.186043739318848, + "delta_nll": 4.565562725067139, + "max_kl": 11.304210662841797, + "mean_kl": 4.516234397888184, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.19402985274791718 + }, + { + "candidate_nll": 4.201249599456787, + "code_tokens": 66, + "cvar95_kl": 7.334239959716797, + "delta_nll": 2.9151222705841064, + "max_kl": 9.714086532592773, + "mean_kl": 2.984682321548462, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 3.7623229026794434, + "code_tokens": 34, + "cvar95_kl": 5.178117752075195, + "delta_nll": 2.750028610229492, + "max_kl": 5.179970741271973, + "mean_kl": 2.871076822280884, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 3.3042025566101074, + "code_tokens": 64, + "cvar95_kl": 6.756838798522949, + "delta_nll": 2.394177198410034, + "max_kl": 7.024138450622559, + "mean_kl": 2.3956966400146484, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.421875 + }, + { + "candidate_nll": 3.6628329753875732, + "code_tokens": 56, + "cvar95_kl": 6.932291030883789, + "delta_nll": 2.352477550506592, + "max_kl": 7.1943511962890625, + "mean_kl": 2.4939169883728027, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 2.9553189277648926, + "code_tokens": 21, + "cvar95_kl": 5.339939117431641, + "delta_nll": 1.830236554145813, + "max_kl": 6.705240249633789, + "mean_kl": 2.004164457321167, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.599843740463257, + "code_tokens": 60, + "cvar95_kl": 6.7117085456848145, + "delta_nll": 2.9345033168792725, + "max_kl": 7.4439616203308105, + "mean_kl": 2.995230197906494, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.3166666626930237 + }, + { + "candidate_nll": 4.187854290008545, + "code_tokens": 29, + "cvar95_kl": 7.286620616912842, + "delta_nll": 2.834078311920166, + "max_kl": 7.5157976150512695, + "mean_kl": 2.744678258895874, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 5.48168420791626, + "code_tokens": 34, + "cvar95_kl": 10.84099292755127, + "delta_nll": 4.759639263153076, + "max_kl": 11.483819007873535, + "mean_kl": 4.749030113220215, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 5.352231979370117, + "code_tokens": 88, + "cvar95_kl": 10.177335739135742, + "delta_nll": 4.433124542236328, + "max_kl": 12.692008018493652, + "mean_kl": 4.534321308135986, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.23863635957241058 + }, + { + "candidate_nll": 2.6127536296844482, + "code_tokens": 17, + "cvar95_kl": 5.640226364135742, + "delta_nll": 2.0695300102233887, + "max_kl": 5.640226364135742, + "mean_kl": 2.051298141479492, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 4.3548407554626465, + "code_tokens": 35, + "cvar95_kl": 6.365209579467773, + "delta_nll": 2.4642934799194336, + "max_kl": 7.100642204284668, + "mean_kl": 2.7589378356933594, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.2657110691070557, + "code_tokens": 48, + "cvar95_kl": 7.795518398284912, + "delta_nll": 2.375934600830078, + "max_kl": 9.03173828125, + "mean_kl": 2.5199029445648193, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.7031025886535645, + "code_tokens": 130, + "cvar95_kl": 7.302969455718994, + "delta_nll": 2.712830066680908, + "max_kl": 9.932597160339355, + "mean_kl": 2.586195468902588, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.1067681312561035, + "code_tokens": 65, + "cvar95_kl": 8.227455139160156, + "delta_nll": 4.011828422546387, + "max_kl": 8.563462257385254, + "mean_kl": 4.0326313972473145, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 4.414394855499268, + "code_tokens": 145, + "cvar95_kl": 9.142355918884277, + "delta_nll": 3.539379835128784, + "max_kl": 11.42798900604248, + "mean_kl": 3.583495616912842, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.2827586233615875 + }, + { + "candidate_nll": 2.640784502029419, + "code_tokens": 80, + "cvar95_kl": 7.048245429992676, + "delta_nll": 1.925085186958313, + "max_kl": 9.993934631347656, + "mean_kl": 1.9369277954101562, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.48750001192092896 + }, + { + "candidate_nll": 3.688459634780884, + "code_tokens": 70, + "cvar95_kl": 7.190515995025635, + "delta_nll": 3.106322765350342, + "max_kl": 8.009532928466797, + "mean_kl": 3.253612995147705, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.014726161956787, + "code_tokens": 25, + "cvar95_kl": 6.593520164489746, + "delta_nll": 2.526437282562256, + "max_kl": 7.4868950843811035, + "mean_kl": 2.556086778640747, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.543538808822632, + "code_tokens": 50, + "cvar95_kl": 5.39513635635376, + "delta_nll": 1.889155387878418, + "max_kl": 5.6276092529296875, + "mean_kl": 2.367694139480591, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 3.598980665206909, + "code_tokens": 48, + "cvar95_kl": 6.516425609588623, + "delta_nll": 2.8009376525878906, + "max_kl": 6.6560258865356445, + "mean_kl": 3.042141914367676, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.3541666567325592 + }, + { + "candidate_nll": 3.3091769218444824, + "code_tokens": 104, + "cvar95_kl": 9.636539459228516, + "delta_nll": 2.965104579925537, + "max_kl": 12.225075721740723, + "mean_kl": 2.8114264011383057, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.4423076808452606 + }, + { + "candidate_nll": 3.7910237312316895, + "code_tokens": 68, + "cvar95_kl": 6.568106651306152, + "delta_nll": 2.8010294437408447, + "max_kl": 7.2607245445251465, + "mean_kl": 2.842883825302124, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 3.64033579826355, + "code_tokens": 145, + "cvar95_kl": 7.090956211090088, + "delta_nll": 2.8854236602783203, + "max_kl": 8.037004470825195, + "mean_kl": 2.853065252304077, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 3.114711046218872, + "code_tokens": 41, + "cvar95_kl": 5.98574686050415, + "delta_nll": 2.2698233127593994, + "max_kl": 7.0195465087890625, + "mean_kl": 2.4386610984802246, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.31707316637039185 + }, + { + "candidate_nll": 2.606175184249878, + "code_tokens": 53, + "cvar95_kl": 5.939585208892822, + "delta_nll": 1.84505295753479, + "max_kl": 7.064458847045898, + "mean_kl": 1.7569563388824463, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.5283018946647644 + }, + { + "candidate_nll": 3.919569253921509, + "code_tokens": 58, + "cvar95_kl": 9.394296646118164, + "delta_nll": 3.1273393630981445, + "max_kl": 12.278731346130371, + "mean_kl": 3.0778191089630127, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.3448275923728943 + }, + { + "candidate_nll": 4.5189948081970215, + "code_tokens": 24, + "cvar95_kl": 9.747037887573242, + "delta_nll": 2.7869348526000977, + "max_kl": 9.92180347442627, + "mean_kl": 3.0536365509033203, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.181246757507324, + "code_tokens": 89, + "cvar95_kl": 7.859694480895996, + "delta_nll": 2.1893928050994873, + "max_kl": 9.826988220214844, + "mean_kl": 2.407557725906372, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.43820226192474365 + }, + { + "candidate_nll": 4.142261505126953, + "code_tokens": 101, + "cvar95_kl": 9.331418991088867, + "delta_nll": 3.6571853160858154, + "max_kl": 10.103479385375977, + "mean_kl": 3.606489419937134, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.23762376606464386 + }, + { + "candidate_nll": 4.9852728843688965, + "code_tokens": 33, + "cvar95_kl": 10.137592315673828, + "delta_nll": 3.5277152061462402, + "max_kl": 11.89284610748291, + "mean_kl": 3.5599281787872314, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.97309947013855, + "code_tokens": 50, + "cvar95_kl": 9.831671714782715, + "delta_nll": 2.92643404006958, + "max_kl": 10.906719207763672, + "mean_kl": 3.1264946460723877, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.41999998688697815 + }, + { + "candidate_nll": 4.440979957580566, + "code_tokens": 58, + "cvar95_kl": 8.141020774841309, + "delta_nll": 3.032227039337158, + "max_kl": 10.594991683959961, + "mean_kl": 3.27176570892334, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 3.972139835357666, + "code_tokens": 62, + "cvar95_kl": 7.805920124053955, + "delta_nll": 2.99298095703125, + "max_kl": 9.254980087280273, + "mean_kl": 3.1404168605804443, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 3.708310842514038, + "code_tokens": 42, + "cvar95_kl": 7.742997646331787, + "delta_nll": 3.0166208744049072, + "max_kl": 9.24355697631836, + "mean_kl": 3.0331356525421143, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.62322735786438, + "code_tokens": 15, + "cvar95_kl": 7.332801818847656, + "delta_nll": 2.698486328125, + "max_kl": 7.332801818847656, + "mean_kl": 2.48818302154541, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.14994478225708, + "code_tokens": 57, + "cvar95_kl": 7.190323352813721, + "delta_nll": 3.4111626148223877, + "max_kl": 8.160894393920898, + "mean_kl": 3.4660799503326416, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.22807016968727112 + }, + { + "candidate_nll": 3.910867691040039, + "code_tokens": 58, + "cvar95_kl": 6.608730316162109, + "delta_nll": 2.8673317432403564, + "max_kl": 7.007795333862305, + "mean_kl": 2.721160650253296, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.32758620381355286 + }, + { + "candidate_nll": 3.9994757175445557, + "code_tokens": 45, + "cvar95_kl": 7.419292449951172, + "delta_nll": 3.127504587173462, + "max_kl": 7.994058609008789, + "mean_kl": 3.1405975818634033, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.2888889014720917 + }, + { + "candidate_nll": 4.635395050048828, + "code_tokens": 32, + "cvar95_kl": 7.965895652770996, + "delta_nll": 2.6061646938323975, + "max_kl": 7.996843338012695, + "mean_kl": 3.179081439971924, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 4.548201084136963, + "code_tokens": 39, + "cvar95_kl": 10.266155242919922, + "delta_nll": 3.227534770965576, + "max_kl": 11.396401405334473, + "mean_kl": 3.5912959575653076, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 2.7699105739593506, + "code_tokens": 136, + "cvar95_kl": 6.622248649597168, + "delta_nll": 2.359149217605591, + "max_kl": 8.25406551361084, + "mean_kl": 2.2942588329315186, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.4485294222831726 + }, + { + "candidate_nll": 4.375323295593262, + "code_tokens": 50, + "cvar95_kl": 9.711027145385742, + "delta_nll": 3.154566764831543, + "max_kl": 10.29765796661377, + "mean_kl": 3.267285108566284, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.3799999952316284 + }, + { + "candidate_nll": 3.316917657852173, + "code_tokens": 34, + "cvar95_kl": 7.632244110107422, + "delta_nll": 2.4510018825531006, + "max_kl": 8.010072708129883, + "mean_kl": 2.5914294719696045, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 4.2431559562683105, + "code_tokens": 44, + "cvar95_kl": 10.22789192199707, + "delta_nll": 3.064365863800049, + "max_kl": 12.566516876220703, + "mean_kl": 3.348889112472534, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 2.956841468811035, + "code_tokens": 36, + "cvar95_kl": 5.487085342407227, + "delta_nll": 1.5234196186065674, + "max_kl": 5.685575008392334, + "mean_kl": 1.8344230651855469, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.41741943359375, + "code_tokens": 46, + "cvar95_kl": 9.046309471130371, + "delta_nll": 3.614262104034424, + "max_kl": 9.463956832885742, + "mean_kl": 3.691896438598633, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 4.202293395996094, + "code_tokens": 48, + "cvar95_kl": 9.085095405578613, + "delta_nll": 3.021953582763672, + "max_kl": 9.226244926452637, + "mean_kl": 3.32615065574646, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.816542387008667, + "code_tokens": 25, + "cvar95_kl": 8.236883163452148, + "delta_nll": 2.9211313724517822, + "max_kl": 9.435705184936523, + "mean_kl": 2.7478716373443604, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.332804203033447, + "code_tokens": 69, + "cvar95_kl": 8.582473754882812, + "delta_nll": 2.8285250663757324, + "max_kl": 10.159003257751465, + "mean_kl": 3.236504316329956, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.23188406229019165 + }, + { + "candidate_nll": 3.8917250633239746, + "code_tokens": 60, + "cvar95_kl": 7.907249450683594, + "delta_nll": 2.926267385482788, + "max_kl": 8.362892150878906, + "mean_kl": 3.221402406692505, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.3389058113098145, + "code_tokens": 49, + "cvar95_kl": 7.706254959106445, + "delta_nll": 2.4244284629821777, + "max_kl": 11.067802429199219, + "mean_kl": 2.4807000160217285, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.189732551574707, + "code_tokens": 44, + "cvar95_kl": 8.83084774017334, + "delta_nll": 2.4892044067382812, + "max_kl": 10.441742897033691, + "mean_kl": 2.45157790184021, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 3.9113008975982666, + "code_tokens": 81, + "cvar95_kl": 7.9122443199157715, + "delta_nll": 2.7658867835998535, + "max_kl": 8.913541793823242, + "mean_kl": 2.7655012607574463, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.3580246865749359 + }, + { + "candidate_nll": 4.209658622741699, + "code_tokens": 26, + "cvar95_kl": 8.609386444091797, + "delta_nll": 2.772923469543457, + "max_kl": 9.666048049926758, + "mean_kl": 2.7651002407073975, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.1326398849487305, + "code_tokens": 32, + "cvar95_kl": 10.1619291305542, + "delta_nll": 3.252718925476074, + "max_kl": 10.61397647857666, + "mean_kl": 3.7281510829925537, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 4.76840353012085, + "code_tokens": 48, + "cvar95_kl": 8.94096851348877, + "delta_nll": 3.8206615447998047, + "max_kl": 9.463146209716797, + "mean_kl": 3.741548776626587, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.6271796226501465, + "code_tokens": 180, + "cvar95_kl": 7.829273700714111, + "delta_nll": 2.927858352661133, + "max_kl": 8.833888053894043, + "mean_kl": 2.9326276779174805, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.3166666626930237 + }, + { + "candidate_nll": 3.8695366382598877, + "code_tokens": 38, + "cvar95_kl": 8.871532440185547, + "delta_nll": 3.147775173187256, + "max_kl": 8.938105583190918, + "mean_kl": 3.260155439376831, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 3.373886823654175, + "code_tokens": 90, + "cvar95_kl": 8.717618942260742, + "delta_nll": 2.6788251399993896, + "max_kl": 9.962666511535645, + "mean_kl": 2.7313077449798584, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 4.457173824310303, + "code_tokens": 44, + "cvar95_kl": 8.492900848388672, + "delta_nll": 3.8825438022613525, + "max_kl": 8.827361106872559, + "mean_kl": 3.771888017654419, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.841280937194824, + "code_tokens": 60, + "cvar95_kl": 6.412673473358154, + "delta_nll": 2.0973806381225586, + "max_kl": 6.670175075531006, + "mean_kl": 2.1979100704193115, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 3.2972195148468018, + "code_tokens": 69, + "cvar95_kl": 5.815661430358887, + "delta_nll": 2.294421434402466, + "max_kl": 6.418903827667236, + "mean_kl": 2.1783831119537354, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.4492753744125366 + }, + { + "candidate_nll": 3.632310390472412, + "code_tokens": 71, + "cvar95_kl": 7.295122146606445, + "delta_nll": 3.114748477935791, + "max_kl": 8.57671070098877, + "mean_kl": 3.2370643615722656, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.28169015049934387 + }, + { + "candidate_nll": 3.0373690128326416, + "code_tokens": 22, + "cvar95_kl": 6.194770812988281, + "delta_nll": 1.7946958541870117, + "max_kl": 6.207554340362549, + "mean_kl": 1.6718500852584839, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 2.2671751976013184, + "code_tokens": 215, + "cvar95_kl": 5.604542255401611, + "delta_nll": 1.7133957147598267, + "max_kl": 6.662557125091553, + "mean_kl": 1.774112582206726, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.48372092843055725 + }, + { + "candidate_nll": 4.682403564453125, + "code_tokens": 26, + "cvar95_kl": 8.514935493469238, + "delta_nll": 3.5796139240264893, + "max_kl": 8.813502311706543, + "mean_kl": 3.385509729385376, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.6265993118286133, + "code_tokens": 33, + "cvar95_kl": 7.894242763519287, + "delta_nll": 2.758352756500244, + "max_kl": 9.370359420776367, + "mean_kl": 2.7704319953918457, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 2.462599515914917, + "code_tokens": 18, + "cvar95_kl": 5.961932182312012, + "delta_nll": 1.5264513492584229, + "max_kl": 5.961932182312012, + "mean_kl": 1.7348437309265137, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.808675765991211, + "code_tokens": 71, + "cvar95_kl": 10.772238731384277, + "delta_nll": 3.528614044189453, + "max_kl": 14.342732429504395, + "mean_kl": 3.8379976749420166, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.2957746386528015 + }, + { + "candidate_nll": 3.5109174251556396, + "code_tokens": 107, + "cvar95_kl": 6.5713019371032715, + "delta_nll": 2.72951602935791, + "max_kl": 7.388566017150879, + "mean_kl": 2.6346518993377686, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.38317757844924927 + }, + { + "candidate_nll": 4.766513347625732, + "code_tokens": 40, + "cvar95_kl": 8.72776985168457, + "delta_nll": 2.711488723754883, + "max_kl": 9.611017227172852, + "mean_kl": 3.3543028831481934, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 3.8751494884490967, + "code_tokens": 32, + "cvar95_kl": 6.486760139465332, + "delta_nll": 2.510852575302124, + "max_kl": 7.400227069854736, + "mean_kl": 2.616386651992798, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 3.5780320167541504, + "code_tokens": 130, + "cvar95_kl": 7.141390323638916, + "delta_nll": 2.8787782192230225, + "max_kl": 9.371297836303711, + "mean_kl": 3.0206139087677, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 2.9373891353607178, + "code_tokens": 27, + "cvar95_kl": 4.312370300292969, + "delta_nll": 1.4659674167633057, + "max_kl": 4.482844829559326, + "mean_kl": 1.6788195371627808, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 3.8376972675323486, + "code_tokens": 42, + "cvar95_kl": 8.184231758117676, + "delta_nll": 2.8647570610046387, + "max_kl": 10.664491653442383, + "mean_kl": 2.788747787475586, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.4523809552192688 + }, + { + "candidate_nll": 4.661892890930176, + "code_tokens": 30, + "cvar95_kl": 10.135139465332031, + "delta_nll": 3.916379690170288, + "max_kl": 12.430122375488281, + "mean_kl": 4.004593849182129, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.143995761871338, + "code_tokens": 58, + "cvar95_kl": 10.30187702178955, + "delta_nll": 3.7247629165649414, + "max_kl": 11.019184112548828, + "mean_kl": 3.745898485183716, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 4.151830673217773, + "code_tokens": 34, + "cvar95_kl": 7.752490997314453, + "delta_nll": 2.9216034412384033, + "max_kl": 8.667580604553223, + "mean_kl": 2.965981960296631, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 2.4728236198425293, + "code_tokens": 208, + "cvar95_kl": 5.5990891456604, + "delta_nll": 1.7713611125946045, + "max_kl": 8.400440216064453, + "mean_kl": 1.7803667783737183, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.5192307829856873 + }, + { + "candidate_nll": 3.3931076526641846, + "code_tokens": 83, + "cvar95_kl": 6.0772786140441895, + "delta_nll": 2.494201898574829, + "max_kl": 7.1208295822143555, + "mean_kl": 2.5920424461364746, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.33734938502311707 + }, + { + "candidate_nll": 3.5319695472717285, + "code_tokens": 170, + "cvar95_kl": 8.783759117126465, + "delta_nll": 3.009413957595825, + "max_kl": 12.19965934753418, + "mean_kl": 3.035884380340576, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 3.3408305644989014, + "code_tokens": 40, + "cvar95_kl": 7.6684980392456055, + "delta_nll": 2.5594100952148438, + "max_kl": 7.97002649307251, + "mean_kl": 2.7354331016540527, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.42500001192092896 + }, + { + "candidate_nll": 3.5180559158325195, + "code_tokens": 31, + "cvar95_kl": 7.576714515686035, + "delta_nll": 2.2713463306427, + "max_kl": 7.827843189239502, + "mean_kl": 2.3391244411468506, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.4838709533214569 + }, + { + "candidate_nll": 4.756565093994141, + "code_tokens": 42, + "cvar95_kl": 8.565040588378906, + "delta_nll": 3.9617950916290283, + "max_kl": 11.191054344177246, + "mean_kl": 4.207080841064453, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.022602081298828, + "code_tokens": 27, + "cvar95_kl": 7.470250129699707, + "delta_nll": 2.2535362243652344, + "max_kl": 8.626555442810059, + "mean_kl": 2.878448963165283, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.810364007949829, + "code_tokens": 25, + "cvar95_kl": 6.608105659484863, + "delta_nll": 1.610553503036499, + "max_kl": 7.258140563964844, + "mean_kl": 2.041593551635742, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.037397861480713, + "code_tokens": 40, + "cvar95_kl": 8.771672248840332, + "delta_nll": 3.4690113067626953, + "max_kl": 10.289599418640137, + "mean_kl": 3.5339267253875732, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.7521677017211914, + "code_tokens": 24, + "cvar95_kl": 6.56171989440918, + "delta_nll": 2.3678131103515625, + "max_kl": 7.380253791809082, + "mean_kl": 2.2557201385498047, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 3.2834489345550537, + "code_tokens": 28, + "cvar95_kl": 7.120732307434082, + "delta_nll": 2.4007039070129395, + "max_kl": 8.040350914001465, + "mean_kl": 2.320481777191162, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.4642857015132904 + }, + { + "candidate_nll": 3.9068264961242676, + "code_tokens": 71, + "cvar95_kl": 8.070730209350586, + "delta_nll": 3.427962064743042, + "max_kl": 8.947785377502441, + "mean_kl": 3.2801151275634766, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.3239436745643616 + }, + { + "candidate_nll": 3.383046865463257, + "code_tokens": 24, + "cvar95_kl": 7.494290351867676, + "delta_nll": 2.8536860942840576, + "max_kl": 7.9328436851501465, + "mean_kl": 2.7135980129241943, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9250168800354004, + "code_tokens": 34, + "cvar95_kl": 8.702484130859375, + "delta_nll": 2.653435707092285, + "max_kl": 9.318812370300293, + "mean_kl": 3.055145740509033, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 4.3675031661987305, + "code_tokens": 64, + "cvar95_kl": 8.79946517944336, + "delta_nll": 3.248209238052368, + "max_kl": 9.747532844543457, + "mean_kl": 3.699157953262329, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.296875 + }, + { + "candidate_nll": 2.800114631652832, + "code_tokens": 45, + "cvar95_kl": 6.9912638664245605, + "delta_nll": 2.0935232639312744, + "max_kl": 7.966099739074707, + "mean_kl": 2.005074977874756, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.4888888895511627 + }, + { + "candidate_nll": 4.254907131195068, + "code_tokens": 48, + "cvar95_kl": 8.72220230102539, + "delta_nll": 3.0562028884887695, + "max_kl": 8.90407657623291, + "mean_kl": 3.366995096206665, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.1655795574188232, + "code_tokens": 21, + "cvar95_kl": 7.43908166885376, + "delta_nll": 2.4653782844543457, + "max_kl": 8.043671607971191, + "mean_kl": 2.5154266357421875, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.190927982330322, + "code_tokens": 24, + "cvar95_kl": 11.14508056640625, + "delta_nll": 3.436513900756836, + "max_kl": 11.819474220275879, + "mean_kl": 3.7745368480682373, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.237282752990723, + "code_tokens": 63, + "cvar95_kl": 6.467024803161621, + "delta_nll": 2.6653554439544678, + "max_kl": 6.866143703460693, + "mean_kl": 2.7106144428253174, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.242742538452148, + "code_tokens": 22, + "cvar95_kl": 8.112385749816895, + "delta_nll": 3.9639835357666016, + "max_kl": 8.478845596313477, + "mean_kl": 3.740203857421875, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 3.1451048851013184, + "code_tokens": 25, + "cvar95_kl": 4.989568710327148, + "delta_nll": 2.0164425373077393, + "max_kl": 5.208180904388428, + "mean_kl": 2.193951368331909, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 2.9103026390075684, + "code_tokens": 21, + "cvar95_kl": 4.651495933532715, + "delta_nll": 0.949976921081543, + "max_kl": 4.910966873168945, + "mean_kl": 1.4371241331100464, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 2.554738759994507, + "code_tokens": 24, + "cvar95_kl": 4.246367454528809, + "delta_nll": 1.0884066820144653, + "max_kl": 4.805075645446777, + "mean_kl": 1.3898639678955078, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 5.456536293029785, + "code_tokens": 15, + "cvar95_kl": 9.862174034118652, + "delta_nll": 3.663048505783081, + "max_kl": 9.862174034118652, + "mean_kl": 3.8466482162475586, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.150855541229248, + "code_tokens": 59, + "cvar95_kl": 8.055451393127441, + "delta_nll": 3.3699991703033447, + "max_kl": 8.367718696594238, + "mean_kl": 3.2239990234375, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.2711864411830902 + }, + { + "candidate_nll": 4.139138698577881, + "code_tokens": 54, + "cvar95_kl": 8.769786834716797, + "delta_nll": 3.2050204277038574, + "max_kl": 9.820764541625977, + "mean_kl": 3.2012171745300293, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 4.602283954620361, + "code_tokens": 28, + "cvar95_kl": 7.540069580078125, + "delta_nll": 2.6504249572753906, + "max_kl": 7.924525260925293, + "mean_kl": 3.1084134578704834, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.45664381980896, + "code_tokens": 46, + "cvar95_kl": 6.209259510040283, + "delta_nll": 2.371917724609375, + "max_kl": 6.442793846130371, + "mean_kl": 2.5766823291778564, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.43478259444236755 + }, + { + "candidate_nll": 2.4994258880615234, + "code_tokens": 27, + "cvar95_kl": 4.902386665344238, + "delta_nll": 0.9846035242080688, + "max_kl": 5.063511848449707, + "mean_kl": 1.5273300409317017, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 4.5232415199279785, + "code_tokens": 97, + "cvar95_kl": 7.23861837387085, + "delta_nll": 3.265810966491699, + "max_kl": 8.56358814239502, + "mean_kl": 3.2848825454711914, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.2680412232875824 + }, + { + "candidate_nll": 4.095901012420654, + "code_tokens": 24, + "cvar95_kl": 6.8827595710754395, + "delta_nll": 3.293668508529663, + "max_kl": 6.883689880371094, + "mean_kl": 3.0348997116088867, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.07563591003418, + "code_tokens": 30, + "cvar95_kl": 9.708115577697754, + "delta_nll": 3.8104872703552246, + "max_kl": 10.755523681640625, + "mean_kl": 4.231987953186035, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.23333333432674408 + }, + { + "candidate_nll": 4.383833408355713, + "code_tokens": 38, + "cvar95_kl": 9.807019233703613, + "delta_nll": 2.950763702392578, + "max_kl": 9.99212646484375, + "mean_kl": 3.1732704639434814, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 2.4335453510284424, + "code_tokens": 21, + "cvar95_kl": 4.023517608642578, + "delta_nll": 0.9457663297653198, + "max_kl": 4.259716987609863, + "mean_kl": 1.241549015045166, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 4.626064777374268, + "code_tokens": 65, + "cvar95_kl": 8.042531967163086, + "delta_nll": 4.089102745056152, + "max_kl": 9.598551750183105, + "mean_kl": 4.0598907470703125, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 2.9310901165008545, + "code_tokens": 27, + "cvar95_kl": 5.8500823974609375, + "delta_nll": 2.146563768386841, + "max_kl": 5.887962341308594, + "mean_kl": 2.1587066650390625, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.5606274604797363, + "code_tokens": 35, + "cvar95_kl": 7.781365394592285, + "delta_nll": 2.8368113040924072, + "max_kl": 9.697003364562988, + "mean_kl": 2.763796806335449, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 3.455869436264038, + "code_tokens": 75, + "cvar95_kl": 7.53785514831543, + "delta_nll": 2.5313453674316406, + "max_kl": 9.423772811889648, + "mean_kl": 2.715806484222412, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 3.970632553100586, + "code_tokens": 25, + "cvar95_kl": 6.878783702850342, + "delta_nll": 2.328896999359131, + "max_kl": 6.966729164123535, + "mean_kl": 2.862185478210449, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.6869218349456787, + "code_tokens": 52, + "cvar95_kl": 7.71198844909668, + "delta_nll": 2.762932777404785, + "max_kl": 8.607898712158203, + "mean_kl": 2.7758915424346924, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.2693283557891846, + "code_tokens": 110, + "cvar95_kl": 9.203076362609863, + "delta_nll": 2.658668279647827, + "max_kl": 11.6675443649292, + "mean_kl": 2.766258716583252, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.41818180680274963 + }, + { + "candidate_nll": 2.7769691944122314, + "code_tokens": 25, + "cvar95_kl": 5.009729385375977, + "delta_nll": 1.544147253036499, + "max_kl": 5.176729202270508, + "mean_kl": 1.7179888486862183, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 4.054093360900879, + "code_tokens": 56, + "cvar95_kl": 5.802335262298584, + "delta_nll": 3.037107467651367, + "max_kl": 6.808192253112793, + "mean_kl": 2.812063455581665, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.3035714328289032 + }, + { + "candidate_nll": 4.643557071685791, + "code_tokens": 108, + "cvar95_kl": 6.504149913787842, + "delta_nll": 3.0897088050842285, + "max_kl": 7.225639820098877, + "mean_kl": 3.175481081008911, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.21296297013759613 + }, + { + "candidate_nll": 4.0251030921936035, + "code_tokens": 47, + "cvar95_kl": 8.078551292419434, + "delta_nll": 2.6533632278442383, + "max_kl": 9.295454025268555, + "mean_kl": 3.1609435081481934, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 3.901331901550293, + "code_tokens": 102, + "cvar95_kl": 8.22189998626709, + "delta_nll": 2.902923345565796, + "max_kl": 8.916255950927734, + "mean_kl": 3.065086841583252, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.3921568691730499 + }, + { + "candidate_nll": 2.9446518421173096, + "code_tokens": 18, + "cvar95_kl": 4.869255542755127, + "delta_nll": 1.319026231765747, + "max_kl": 4.869255542755127, + "mean_kl": 1.5880376100540161, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 4.825125217437744, + "code_tokens": 28, + "cvar95_kl": 8.909452438354492, + "delta_nll": 3.725461006164551, + "max_kl": 10.626250267028809, + "mean_kl": 3.647861957550049, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.132324695587158, + "code_tokens": 30, + "cvar95_kl": 6.776724815368652, + "delta_nll": 2.588923931121826, + "max_kl": 7.010861396789551, + "mean_kl": 2.777707099914551, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 3.4814815521240234, + "code_tokens": 48, + "cvar95_kl": 8.397522926330566, + "delta_nll": 3.006364583969116, + "max_kl": 11.231527328491211, + "mean_kl": 2.976672887802124, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.645915985107422, + "code_tokens": 59, + "cvar95_kl": 7.303903102874756, + "delta_nll": 3.108901023864746, + "max_kl": 7.802358627319336, + "mean_kl": 3.1606688499450684, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.33898305892944336 + }, + { + "candidate_nll": 3.9778692722320557, + "code_tokens": 102, + "cvar95_kl": 9.425411224365234, + "delta_nll": 2.9356753826141357, + "max_kl": 10.46629524230957, + "mean_kl": 2.979914903640747, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.4215686321258545 + }, + { + "candidate_nll": 3.4555554389953613, + "code_tokens": 36, + "cvar95_kl": 6.478847980499268, + "delta_nll": 2.7151899337768555, + "max_kl": 6.753492832183838, + "mean_kl": 2.7378602027893066, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 5.304402828216553, + "code_tokens": 51, + "cvar95_kl": 11.0346097946167, + "delta_nll": 4.415471076965332, + "max_kl": 12.614202499389648, + "mean_kl": 4.212658882141113, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 3.5688741207122803, + "code_tokens": 62, + "cvar95_kl": 6.547532558441162, + "delta_nll": 2.323195457458496, + "max_kl": 6.854584217071533, + "mean_kl": 2.6500604152679443, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.33870968222618103 + }, + { + "candidate_nll": 4.108728885650635, + "code_tokens": 39, + "cvar95_kl": 10.406615257263184, + "delta_nll": 3.53244686126709, + "max_kl": 11.800945281982422, + "mean_kl": 3.436708450317383, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 3.30857253074646, + "code_tokens": 76, + "cvar95_kl": 6.876107215881348, + "delta_nll": 2.5688881874084473, + "max_kl": 8.190765380859375, + "mean_kl": 2.640353202819824, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.3815789520740509 + }, + { + "candidate_nll": 5.16420316696167, + "code_tokens": 58, + "cvar95_kl": 9.455992698669434, + "delta_nll": 3.6595520973205566, + "max_kl": 9.681370735168457, + "mean_kl": 3.64143967628479, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 3.7188267707824707, + "code_tokens": 93, + "cvar95_kl": 8.481634140014648, + "delta_nll": 2.5914316177368164, + "max_kl": 13.48200511932373, + "mean_kl": 2.552600383758545, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.144510746002197, + "code_tokens": 45, + "cvar95_kl": 9.443694114685059, + "delta_nll": 3.7892651557922363, + "max_kl": 10.554518699645996, + "mean_kl": 4.035417556762695, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 2.007570505142212, + "code_tokens": 209, + "cvar95_kl": 5.453721523284912, + "delta_nll": 1.4505951404571533, + "max_kl": 7.977738857269287, + "mean_kl": 1.5337926149368286, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.5263158082962036 + }, + { + "candidate_nll": 3.8727078437805176, + "code_tokens": 46, + "cvar95_kl": 7.678358554840088, + "delta_nll": 2.904240131378174, + "max_kl": 8.399930953979492, + "mean_kl": 2.5515148639678955, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.52173912525177 + }, + { + "candidate_nll": 3.749453067779541, + "code_tokens": 39, + "cvar95_kl": 8.479910850524902, + "delta_nll": 3.0286660194396973, + "max_kl": 9.103399276733398, + "mean_kl": 2.99470591545105, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 2.5368998050689697, + "code_tokens": 25, + "cvar95_kl": 4.869482517242432, + "delta_nll": 1.5952043533325195, + "max_kl": 5.62544584274292, + "mean_kl": 1.6861915588378906, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.149829387664795, + "code_tokens": 28, + "cvar95_kl": 12.333528518676758, + "delta_nll": 2.97463321685791, + "max_kl": 15.299991607666016, + "mean_kl": 3.1447582244873047, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.305094242095947, + "code_tokens": 84, + "cvar95_kl": 7.555650234222412, + "delta_nll": 3.0511717796325684, + "max_kl": 8.350482940673828, + "mean_kl": 3.209994316101074, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 3.439094066619873, + "code_tokens": 34, + "cvar95_kl": 6.1960530281066895, + "delta_nll": 2.6268138885498047, + "max_kl": 6.508756637573242, + "mean_kl": 2.7703492641448975, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 3.1890618801116943, + "code_tokens": 28, + "cvar95_kl": 5.111540794372559, + "delta_nll": 1.9340147972106934, + "max_kl": 5.589962005615234, + "mean_kl": 2.197493553161621, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.0500104427337646, + "code_tokens": 92, + "cvar95_kl": 6.8184943199157715, + "delta_nll": 1.951164960861206, + "max_kl": 7.741484642028809, + "mean_kl": 2.249603748321533, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.44565218687057495 + }, + { + "candidate_nll": 4.707659721374512, + "code_tokens": 185, + "cvar95_kl": 9.612910270690918, + "delta_nll": 3.9775450229644775, + "max_kl": 14.344467163085938, + "mean_kl": 4.114881992340088, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.17297297716140747 + }, + { + "candidate_nll": 3.564903736114502, + "code_tokens": 106, + "cvar95_kl": 8.57319164276123, + "delta_nll": 3.211909294128418, + "max_kl": 10.566874504089355, + "mean_kl": 3.264091730117798, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.3207547068595886 + }, + { + "candidate_nll": 5.171430587768555, + "code_tokens": 64, + "cvar95_kl": 8.781606674194336, + "delta_nll": 3.628876209259033, + "max_kl": 9.959992408752441, + "mean_kl": 3.5885167121887207, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 3.8783211708068848, + "code_tokens": 74, + "cvar95_kl": 7.6545915603637695, + "delta_nll": 2.7675371170043945, + "max_kl": 8.055583953857422, + "mean_kl": 2.865170478820801, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 2.9471418857574463, + "code_tokens": 27, + "cvar95_kl": 7.347048759460449, + "delta_nll": 1.95973801612854, + "max_kl": 9.117673873901367, + "mean_kl": 2.2195510864257812, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.86620569229126, + "code_tokens": 72, + "cvar95_kl": 9.9547119140625, + "delta_nll": 4.242367744445801, + "max_kl": 10.693598747253418, + "mean_kl": 4.307538986206055, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.1805555522441864 + }, + { + "candidate_nll": 3.5721511840820312, + "code_tokens": 62, + "cvar95_kl": 5.880098342895508, + "delta_nll": 2.6388487815856934, + "max_kl": 6.860450744628906, + "mean_kl": 2.473172903060913, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.4677419364452362 + }, + { + "candidate_nll": 3.500393867492676, + "code_tokens": 42, + "cvar95_kl": 7.974227428436279, + "delta_nll": 2.80222749710083, + "max_kl": 9.96810245513916, + "mean_kl": 2.884450912475586, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.807082653045654, + "code_tokens": 238, + "cvar95_kl": 9.433436393737793, + "delta_nll": 3.9197497367858887, + "max_kl": 12.410538673400879, + "mean_kl": 3.9798707962036133, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.19327731430530548 + }, + { + "candidate_nll": 4.106525421142578, + "code_tokens": 68, + "cvar95_kl": 8.533653259277344, + "delta_nll": 3.192013740539551, + "max_kl": 9.710580825805664, + "mean_kl": 3.176730155944824, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 4.065533638000488, + "code_tokens": 83, + "cvar95_kl": 10.1563720703125, + "delta_nll": 3.08138370513916, + "max_kl": 11.707256317138672, + "mean_kl": 3.2381129264831543, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.3132530152797699 + }, + { + "candidate_nll": 6.124907970428467, + "code_tokens": 64, + "cvar95_kl": 10.510738372802734, + "delta_nll": 4.861140251159668, + "max_kl": 12.383749961853027, + "mean_kl": 4.856654644012451, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 2.4202804565429688, + "code_tokens": 37, + "cvar95_kl": 4.040976524353027, + "delta_nll": 1.763480544090271, + "max_kl": 4.246326923370361, + "mean_kl": 1.7754299640655518, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.45945945382118225 + }, + { + "candidate_nll": 4.146208763122559, + "code_tokens": 27, + "cvar95_kl": 7.17087459564209, + "delta_nll": 2.6362814903259277, + "max_kl": 7.469880104064941, + "mean_kl": 2.89099383354187, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.827180862426758, + "code_tokens": 40, + "cvar95_kl": 7.703758239746094, + "delta_nll": 2.935464382171631, + "max_kl": 7.916969299316406, + "mean_kl": 3.1729555130004883, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 3.9782791137695312, + "code_tokens": 28, + "cvar95_kl": 8.202245712280273, + "delta_nll": 2.980506420135498, + "max_kl": 9.551106452941895, + "mean_kl": 3.1490886211395264, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.033463716506958, + "code_tokens": 59, + "cvar95_kl": 6.911319732666016, + "delta_nll": 2.1788339614868164, + "max_kl": 6.963494300842285, + "mean_kl": 2.4190824031829834, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.38983049988746643 + }, + { + "candidate_nll": 3.9910590648651123, + "code_tokens": 57, + "cvar95_kl": 8.455559730529785, + "delta_nll": 3.4210588932037354, + "max_kl": 9.954326629638672, + "mean_kl": 3.2249948978424072, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.2982456088066101 + }, + { + "candidate_nll": 4.3675312995910645, + "code_tokens": 25, + "cvar95_kl": 6.839855670928955, + "delta_nll": 2.983591079711914, + "max_kl": 7.120569705963135, + "mean_kl": 3.2084801197052, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 2.651963233947754, + "code_tokens": 48, + "cvar95_kl": 5.584285259246826, + "delta_nll": 1.904810905456543, + "max_kl": 7.020132064819336, + "mean_kl": 1.857293725013733, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.3958333432674408 + }, + { + "candidate_nll": 3.8271374702453613, + "code_tokens": 35, + "cvar95_kl": 6.807836532592773, + "delta_nll": 2.9653615951538086, + "max_kl": 6.847902774810791, + "mean_kl": 2.8659393787384033, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 2.894305944442749, + "code_tokens": 30, + "cvar95_kl": 5.76302433013916, + "delta_nll": 2.249598741531372, + "max_kl": 5.791143417358398, + "mean_kl": 2.1910688877105713, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.3486573696136475, + "code_tokens": 49, + "cvar95_kl": 6.353256702423096, + "delta_nll": 2.2255797386169434, + "max_kl": 8.121053695678711, + "mean_kl": 2.4354312419891357, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 4.773329734802246, + "code_tokens": 98, + "cvar95_kl": 8.46200942993164, + "delta_nll": 3.757112979888916, + "max_kl": 9.13479995727539, + "mean_kl": 3.689932107925415, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.23469388484954834 + }, + { + "candidate_nll": 2.9591522216796875, + "code_tokens": 25, + "cvar95_kl": 4.939708232879639, + "delta_nll": 1.5783458948135376, + "max_kl": 5.1248931884765625, + "mean_kl": 1.8849174976348877, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.27467679977417, + "code_tokens": 27, + "cvar95_kl": 7.8313751220703125, + "delta_nll": 2.4969534873962402, + "max_kl": 8.559452056884766, + "mean_kl": 2.499979257583618, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 2.6596598625183105, + "code_tokens": 20, + "cvar95_kl": 6.9293694496154785, + "delta_nll": 1.6145751476287842, + "max_kl": 6.9293694496154785, + "mean_kl": 1.9398990869522095, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.550000011920929 + }, + { + "candidate_nll": 4.672250747680664, + "code_tokens": 253, + "cvar95_kl": 9.62491226196289, + "delta_nll": 3.979580879211426, + "max_kl": 11.580108642578125, + "mean_kl": 4.047435760498047, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.2134387344121933 + }, + { + "candidate_nll": 4.208224773406982, + "code_tokens": 55, + "cvar95_kl": 8.029579162597656, + "delta_nll": 3.3791046142578125, + "max_kl": 8.461231231689453, + "mean_kl": 3.4881997108459473, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.290909081697464 + }, + { + "candidate_nll": 3.6641347408294678, + "code_tokens": 31, + "cvar95_kl": 5.602606773376465, + "delta_nll": 2.4576363563537598, + "max_kl": 5.879108905792236, + "mean_kl": 2.5611467361450195, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 4.33246374130249, + "code_tokens": 35, + "cvar95_kl": 11.755317687988281, + "delta_nll": 3.6282176971435547, + "max_kl": 12.228717803955078, + "mean_kl": 3.576936721801758, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.2571428716182709 + }, + { + "candidate_nll": 2.9155972003936768, + "code_tokens": 61, + "cvar95_kl": 6.397940158843994, + "delta_nll": 2.1127285957336426, + "max_kl": 6.810038089752197, + "mean_kl": 2.2192816734313965, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.4590163826942444 + }, + { + "candidate_nll": 3.758049964904785, + "code_tokens": 55, + "cvar95_kl": 7.602392673492432, + "delta_nll": 2.8033323287963867, + "max_kl": 8.22330379486084, + "mean_kl": 2.8313770294189453, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.2545454502105713 + }, + { + "candidate_nll": 4.820023536682129, + "code_tokens": 108, + "cvar95_kl": 9.345358848571777, + "delta_nll": 3.313056468963623, + "max_kl": 15.277609825134277, + "mean_kl": 3.485971689224243, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.572472095489502, + "code_tokens": 58, + "cvar95_kl": 6.561483383178711, + "delta_nll": 2.285166025161743, + "max_kl": 8.601494789123535, + "mean_kl": 2.387906312942505, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.3965517282485962 + }, + { + "candidate_nll": 4.0948333740234375, + "code_tokens": 91, + "cvar95_kl": 8.088418006896973, + "delta_nll": 3.267378091812134, + "max_kl": 10.034159660339355, + "mean_kl": 3.2571194171905518, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.4224889278411865, + "code_tokens": 105, + "cvar95_kl": 6.8147711753845215, + "delta_nll": 2.658749580383301, + "max_kl": 8.036609649658203, + "mean_kl": 2.7551093101501465, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.3523809611797333 + }, + { + "candidate_nll": 3.6988604068756104, + "code_tokens": 49, + "cvar95_kl": 7.455820083618164, + "delta_nll": 3.051607847213745, + "max_kl": 8.161615371704102, + "mean_kl": 2.8254473209381104, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.8134753704071045, + "code_tokens": 27, + "cvar95_kl": 8.758859634399414, + "delta_nll": 2.3513123989105225, + "max_kl": 9.035856246948242, + "mean_kl": 2.6258838176727295, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 3.0558338165283203, + "code_tokens": 25, + "cvar95_kl": 5.384978294372559, + "delta_nll": 2.0154571533203125, + "max_kl": 5.422891616821289, + "mean_kl": 2.2576260566711426, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.5003771781921387, + "code_tokens": 98, + "cvar95_kl": 7.389438629150391, + "delta_nll": 2.205322742462158, + "max_kl": 9.64173698425293, + "mean_kl": 2.490189790725708, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 3.386939287185669, + "code_tokens": 23, + "cvar95_kl": 5.752326965332031, + "delta_nll": 2.7371575832366943, + "max_kl": 5.848945140838623, + "mean_kl": 2.6174678802490234, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 4.825920581817627, + "code_tokens": 31, + "cvar95_kl": 7.3768768310546875, + "delta_nll": 2.9605469703674316, + "max_kl": 7.385696887969971, + "mean_kl": 3.481050729751587, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 3.754147529602051, + "code_tokens": 36, + "cvar95_kl": 11.369157791137695, + "delta_nll": 3.006054401397705, + "max_kl": 12.36591911315918, + "mean_kl": 3.0113601684570312, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 3.877789258956909, + "code_tokens": 38, + "cvar95_kl": 6.173545837402344, + "delta_nll": 2.4441118240356445, + "max_kl": 6.246753215789795, + "mean_kl": 2.8746161460876465, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 3.0115456581115723, + "code_tokens": 25, + "cvar95_kl": 5.265403747558594, + "delta_nll": 1.4054558277130127, + "max_kl": 6.783333778381348, + "mean_kl": 1.5365898609161377, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.962289333343506, + "code_tokens": 26, + "cvar95_kl": 6.245510578155518, + "delta_nll": 2.0828402042388916, + "max_kl": 6.600547790527344, + "mean_kl": 2.5411605834960938, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.20059871673584, + "code_tokens": 62, + "cvar95_kl": 7.840406894683838, + "delta_nll": 3.4581565856933594, + "max_kl": 10.068835258483887, + "mean_kl": 3.294588565826416, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.27419355511665344 + }, + { + "candidate_nll": 4.414811611175537, + "code_tokens": 36, + "cvar95_kl": 6.425894260406494, + "delta_nll": 2.6762924194335938, + "max_kl": 7.0284953117370605, + "mean_kl": 3.0471174716949463, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.107329368591309, + "code_tokens": 119, + "cvar95_kl": 7.669893741607666, + "delta_nll": 2.776740550994873, + "max_kl": 8.898923873901367, + "mean_kl": 2.962515115737915, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.3193277418613434 + }, + { + "candidate_nll": 3.5862653255462646, + "code_tokens": 37, + "cvar95_kl": 7.556941986083984, + "delta_nll": 3.273557662963867, + "max_kl": 7.8524274826049805, + "mean_kl": 3.0931084156036377, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.45945945382118225 + }, + { + "candidate_nll": 4.085914134979248, + "code_tokens": 28, + "cvar95_kl": 9.058191299438477, + "delta_nll": 3.303180694580078, + "max_kl": 9.830732345581055, + "mean_kl": 3.484743118286133, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.9174036979675293, + "code_tokens": 143, + "cvar95_kl": 8.858000755310059, + "delta_nll": 3.3510568141937256, + "max_kl": 10.938206672668457, + "mean_kl": 3.414752244949341, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.014261722564697, + "code_tokens": 29, + "cvar95_kl": 7.769869804382324, + "delta_nll": 2.2319555282592773, + "max_kl": 8.287446022033691, + "mean_kl": 2.8031668663024902, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.4482758641242981 + }, + { + "candidate_nll": 4.592342376708984, + "code_tokens": 28, + "cvar95_kl": 8.512909889221191, + "delta_nll": 3.5198540687561035, + "max_kl": 9.516456604003906, + "mean_kl": 3.3904378414154053, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 4.694944381713867, + "code_tokens": 30, + "cvar95_kl": 10.423651695251465, + "delta_nll": 3.4309067726135254, + "max_kl": 11.073334693908691, + "mean_kl": 3.9495832920074463, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.8359169960021973, + "code_tokens": 85, + "cvar95_kl": 10.131686210632324, + "delta_nll": 3.414106845855713, + "max_kl": 12.991327285766602, + "mean_kl": 3.3166282176971436, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.34117648005485535 + }, + { + "candidate_nll": 4.489693641662598, + "code_tokens": 44, + "cvar95_kl": 8.569987297058105, + "delta_nll": 3.5124499797821045, + "max_kl": 9.385522842407227, + "mean_kl": 3.367690086364746, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.069340229034424, + "code_tokens": 19, + "cvar95_kl": 10.12033748626709, + "delta_nll": 3.265238046646118, + "max_kl": 10.12033748626709, + "mean_kl": 3.0394961833953857, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.4736842215061188 + }, + { + "candidate_nll": 3.895134687423706, + "code_tokens": 113, + "cvar95_kl": 7.254978656768799, + "delta_nll": 3.1577117443084717, + "max_kl": 8.242959022521973, + "mean_kl": 3.0578746795654297, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.3008849620819092 + }, + { + "candidate_nll": 3.7592358589172363, + "code_tokens": 40, + "cvar95_kl": 8.11822509765625, + "delta_nll": 2.284041404724121, + "max_kl": 9.708270072937012, + "mean_kl": 2.9643733501434326, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 5.307436466217041, + "code_tokens": 51, + "cvar95_kl": 9.920289039611816, + "delta_nll": 3.7007765769958496, + "max_kl": 10.277143478393555, + "mean_kl": 4.019830703735352, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.2549019753932953 + }, + { + "candidate_nll": 4.264968395233154, + "code_tokens": 55, + "cvar95_kl": 8.916967391967773, + "delta_nll": 2.6839358806610107, + "max_kl": 10.939213752746582, + "mean_kl": 3.0088396072387695, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.23636363446712494 + }, + { + "candidate_nll": 3.1362812519073486, + "code_tokens": 27, + "cvar95_kl": 5.873288154602051, + "delta_nll": 2.0062129497528076, + "max_kl": 6.284228324890137, + "mean_kl": 1.875086784362793, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 4.693787097930908, + "code_tokens": 32, + "cvar95_kl": 8.361629486083984, + "delta_nll": 3.8539810180664062, + "max_kl": 8.87597370147705, + "mean_kl": 3.7587015628814697, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 3.374856472015381, + "code_tokens": 64, + "cvar95_kl": 7.4467973709106445, + "delta_nll": 2.722318410873413, + "max_kl": 7.795247554779053, + "mean_kl": 2.772235870361328, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.328125 + }, + { + "candidate_nll": 3.9136440753936768, + "code_tokens": 35, + "cvar95_kl": 7.774649620056152, + "delta_nll": 2.86398983001709, + "max_kl": 7.91188383102417, + "mean_kl": 2.8635737895965576, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.465147018432617, + "code_tokens": 152, + "cvar95_kl": 8.927391052246094, + "delta_nll": 4.013299465179443, + "max_kl": 10.450813293457031, + "mean_kl": 3.8925909996032715, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 4.325758934020996, + "code_tokens": 77, + "cvar95_kl": 8.138954162597656, + "delta_nll": 3.064424514770508, + "max_kl": 9.421307563781738, + "mean_kl": 2.9300687313079834, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.3246753215789795 + }, + { + "candidate_nll": 3.3493998050689697, + "code_tokens": 77, + "cvar95_kl": 8.498827934265137, + "delta_nll": 2.781118869781494, + "max_kl": 11.321470260620117, + "mean_kl": 2.7660858631134033, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.37662336230278015 + }, + { + "candidate_nll": 2.5451161861419678, + "code_tokens": 71, + "cvar95_kl": 6.265505790710449, + "delta_nll": 1.983027458190918, + "max_kl": 7.699175834655762, + "mean_kl": 2.1099510192871094, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.5633803009986877 + }, + { + "candidate_nll": 5.582945346832275, + "code_tokens": 112, + "cvar95_kl": 10.734698295593262, + "delta_nll": 4.70456600189209, + "max_kl": 13.233384132385254, + "mean_kl": 4.522200584411621, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.2321428507566452 + }, + { + "candidate_nll": 2.7839443683624268, + "code_tokens": 111, + "cvar95_kl": 6.547414779663086, + "delta_nll": 2.0640406608581543, + "max_kl": 7.277243614196777, + "mean_kl": 2.0953867435455322, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.522522509098053 + }, + { + "candidate_nll": 4.220188140869141, + "code_tokens": 62, + "cvar95_kl": 9.684996604919434, + "delta_nll": 3.4322919845581055, + "max_kl": 10.70062255859375, + "mean_kl": 3.590590476989746, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 2.259019374847412, + "code_tokens": 52, + "cvar95_kl": 4.965773105621338, + "delta_nll": 1.6484394073486328, + "max_kl": 5.425001621246338, + "mean_kl": 1.591780185699463, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.557692289352417 + }, + { + "candidate_nll": 3.905245542526245, + "code_tokens": 24, + "cvar95_kl": 6.464766502380371, + "delta_nll": 2.010018825531006, + "max_kl": 7.55283784866333, + "mean_kl": 2.4094078540802, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.4288318157196045, + "code_tokens": 48, + "cvar95_kl": 8.001630783081055, + "delta_nll": 2.537061929702759, + "max_kl": 8.021754264831543, + "mean_kl": 2.910797357559204, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.8517892360687256, + "code_tokens": 98, + "cvar95_kl": 8.150991439819336, + "delta_nll": 2.8014962673187256, + "max_kl": 9.797379493713379, + "mean_kl": 2.9577341079711914, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.7335994243621826, + "code_tokens": 48, + "cvar95_kl": 7.808475971221924, + "delta_nll": 2.7409138679504395, + "max_kl": 9.680529594421387, + "mean_kl": 2.6925525665283203, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.770689964294434, + "code_tokens": 56, + "cvar95_kl": 11.006145477294922, + "delta_nll": 3.8246946334838867, + "max_kl": 11.913005828857422, + "mean_kl": 4.00930643081665, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.2678571343421936 + }, + { + "candidate_nll": 4.590874195098877, + "code_tokens": 88, + "cvar95_kl": 9.858302116394043, + "delta_nll": 3.9895973205566406, + "max_kl": 12.387337684631348, + "mean_kl": 3.897498846054077, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.685904502868652, + "code_tokens": 50, + "cvar95_kl": 8.163041114807129, + "delta_nll": 3.0523743629455566, + "max_kl": 9.498499870300293, + "mean_kl": 3.1223554611206055, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 2.494797706604004, + "code_tokens": 85, + "cvar95_kl": 5.9764933586120605, + "delta_nll": 2.0578105449676514, + "max_kl": 8.082311630249023, + "mean_kl": 1.9048150777816772, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.5411764979362488 + }, + { + "candidate_nll": 4.267807483673096, + "code_tokens": 27, + "cvar95_kl": 7.7368268966674805, + "delta_nll": 3.193594455718994, + "max_kl": 7.841042995452881, + "mean_kl": 3.4907476902008057, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.4250264167785645, + "code_tokens": 48, + "cvar95_kl": 9.952433586120605, + "delta_nll": 3.307407855987549, + "max_kl": 11.058871269226074, + "mean_kl": 3.6593542098999023, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.051474094390869, + "code_tokens": 29, + "cvar95_kl": 9.320785522460938, + "delta_nll": 3.4036269187927246, + "max_kl": 9.56149959564209, + "mean_kl": 3.669032335281372, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 3.9112253189086914, + "code_tokens": 63, + "cvar95_kl": 8.00925064086914, + "delta_nll": 2.9730100631713867, + "max_kl": 9.012495040893555, + "mean_kl": 3.214749813079834, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 4.377763748168945, + "code_tokens": 89, + "cvar95_kl": 7.7223920822143555, + "delta_nll": 3.702705144882202, + "max_kl": 9.390485763549805, + "mean_kl": 3.5976290702819824, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.24719101190567017 + }, + { + "candidate_nll": 4.6571879386901855, + "code_tokens": 133, + "cvar95_kl": 8.234757423400879, + "delta_nll": 3.3129096031188965, + "max_kl": 10.127464294433594, + "mean_kl": 3.4524085521698, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.23308271169662476 + }, + { + "candidate_nll": 2.5314416885375977, + "code_tokens": 25, + "cvar95_kl": 4.741483688354492, + "delta_nll": 1.5829049348831177, + "max_kl": 5.045314311981201, + "mean_kl": 1.6569095849990845, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.091783046722412, + "code_tokens": 33, + "cvar95_kl": 5.862120628356934, + "delta_nll": 2.0544826984405518, + "max_kl": 6.229869842529297, + "mean_kl": 2.4593029022216797, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.6300361156463623, + "code_tokens": 93, + "cvar95_kl": 7.606135368347168, + "delta_nll": 1.8732612133026123, + "max_kl": 12.01096248626709, + "mean_kl": 2.1089816093444824, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.4193548262119293 + }, + { + "candidate_nll": 4.1940155029296875, + "code_tokens": 101, + "cvar95_kl": 7.741258144378662, + "delta_nll": 3.225615978240967, + "max_kl": 9.461267471313477, + "mean_kl": 3.128856658935547, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.3366336524486542 + }, + { + "candidate_nll": 4.460697174072266, + "code_tokens": 152, + "cvar95_kl": 10.054137229919434, + "delta_nll": 3.8959338665008545, + "max_kl": 12.1721773147583, + "mean_kl": 3.8152084350585938, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.28289473056793213 + }, + { + "candidate_nll": 3.503760576248169, + "code_tokens": 33, + "cvar95_kl": 6.152459621429443, + "delta_nll": 2.102889060974121, + "max_kl": 6.671088695526123, + "mean_kl": 2.4422006607055664, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.728804588317871, + "code_tokens": 29, + "cvar95_kl": 7.1723833084106445, + "delta_nll": 2.4423418045043945, + "max_kl": 7.848186016082764, + "mean_kl": 2.5440690517425537, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.4137931168079376 + }, + { + "candidate_nll": 4.7537617683410645, + "code_tokens": 27, + "cvar95_kl": 10.228979110717773, + "delta_nll": 3.295691967010498, + "max_kl": 12.019438743591309, + "mean_kl": 3.6241393089294434, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 3.8176932334899902, + "code_tokens": 33, + "cvar95_kl": 7.306041717529297, + "delta_nll": 3.50765323638916, + "max_kl": 7.487814426422119, + "mean_kl": 3.424762487411499, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 3.6836636066436768, + "code_tokens": 27, + "cvar95_kl": 7.673203468322754, + "delta_nll": 2.887265205383301, + "max_kl": 7.915756702423096, + "mean_kl": 3.0857746601104736, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.758047580718994, + "code_tokens": 84, + "cvar95_kl": 9.207071304321289, + "delta_nll": 3.05415415763855, + "max_kl": 10.906866073608398, + "mean_kl": 3.1580874919891357, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.3452380895614624 + }, + { + "candidate_nll": 3.3685781955718994, + "code_tokens": 103, + "cvar95_kl": 5.816004276275635, + "delta_nll": 2.323108196258545, + "max_kl": 6.899046421051025, + "mean_kl": 2.3857786655426025, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.3980582654476166 + }, + { + "candidate_nll": 4.1532416343688965, + "code_tokens": 88, + "cvar95_kl": 7.082659721374512, + "delta_nll": 3.236894369125366, + "max_kl": 7.526912212371826, + "mean_kl": 3.17851185798645, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 5.425111770629883, + "code_tokens": 62, + "cvar95_kl": 8.464122772216797, + "delta_nll": 4.116180896759033, + "max_kl": 8.953084945678711, + "mean_kl": 3.8392953872680664, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 3.9952192306518555, + "code_tokens": 24, + "cvar95_kl": 5.104869842529297, + "delta_nll": 2.458510398864746, + "max_kl": 5.5429463386535645, + "mean_kl": 2.7017548084259033, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.82045841217041, + "code_tokens": 34, + "cvar95_kl": 7.882086277008057, + "delta_nll": 2.482006311416626, + "max_kl": 7.995326042175293, + "mean_kl": 2.3790535926818848, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.198818683624268, + "code_tokens": 43, + "cvar95_kl": 7.744855880737305, + "delta_nll": 3.4790732860565186, + "max_kl": 8.590738296508789, + "mean_kl": 3.4905800819396973, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 4.685117244720459, + "code_tokens": 54, + "cvar95_kl": 9.194653511047363, + "delta_nll": 3.1958119869232178, + "max_kl": 10.360200881958008, + "mean_kl": 3.089264392852783, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.31481480598449707 + }, + { + "candidate_nll": 3.5379161834716797, + "code_tokens": 50, + "cvar95_kl": 6.581552028656006, + "delta_nll": 2.808030605316162, + "max_kl": 6.979666709899902, + "mean_kl": 2.796138286590576, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.4282402992248535, + "code_tokens": 235, + "cvar95_kl": 8.7581787109375, + "delta_nll": 3.670729875564575, + "max_kl": 10.759210586547852, + "mean_kl": 3.6733176708221436, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.31489360332489014 + }, + { + "candidate_nll": 4.677292823791504, + "code_tokens": 33, + "cvar95_kl": 6.695250511169434, + "delta_nll": 2.6486377716064453, + "max_kl": 6.945952892303467, + "mean_kl": 3.2335243225097656, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 4.025272369384766, + "code_tokens": 45, + "cvar95_kl": 6.684231281280518, + "delta_nll": 3.0387699604034424, + "max_kl": 6.916632175445557, + "mean_kl": 2.9058074951171875, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.1484575271606445, + "code_tokens": 28, + "cvar95_kl": 10.040220260620117, + "delta_nll": 3.5254976749420166, + "max_kl": 11.457362174987793, + "mean_kl": 3.3200771808624268, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.4077694416046143, + "code_tokens": 49, + "cvar95_kl": 6.69069242477417, + "delta_nll": 2.7295050621032715, + "max_kl": 7.346279144287109, + "mean_kl": 2.766716480255127, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 4.5633015632629395, + "code_tokens": 107, + "cvar95_kl": 9.888256072998047, + "delta_nll": 4.3138861656188965, + "max_kl": 12.219732284545898, + "mean_kl": 4.112032413482666, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.25233644247055054 + }, + { + "candidate_nll": 4.3835930824279785, + "code_tokens": 48, + "cvar95_kl": 9.24953556060791, + "delta_nll": 3.1626830101013184, + "max_kl": 10.163021087646484, + "mean_kl": 3.5334904193878174, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.331068754196167, + "code_tokens": 40, + "cvar95_kl": 5.269242286682129, + "delta_nll": 1.4915279150009155, + "max_kl": 5.655367374420166, + "mean_kl": 1.8464882373809814, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 2.755636215209961, + "code_tokens": 46, + "cvar95_kl": 7.524255275726318, + "delta_nll": 2.2861526012420654, + "max_kl": 9.518563270568848, + "mean_kl": 2.4480888843536377, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.45652174949645996 + }, + { + "candidate_nll": 3.732455253601074, + "code_tokens": 31, + "cvar95_kl": 6.893960952758789, + "delta_nll": 2.6900055408477783, + "max_kl": 7.356243133544922, + "mean_kl": 2.716346263885498, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.4516128897666931 + }, + { + "candidate_nll": 1.783747911453247, + "code_tokens": 58, + "cvar95_kl": 5.5459113121032715, + "delta_nll": 1.3672634363174438, + "max_kl": 7.2857842445373535, + "mean_kl": 1.3696975708007812, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.6034482717514038 + }, + { + "candidate_nll": 3.344712018966675, + "code_tokens": 49, + "cvar95_kl": 8.152676582336426, + "delta_nll": 2.2269420623779297, + "max_kl": 11.271051406860352, + "mean_kl": 2.3511946201324463, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.4693877696990967 + }, + { + "candidate_nll": 2.985682487487793, + "code_tokens": 18, + "cvar95_kl": 7.476484775543213, + "delta_nll": 2.0390889644622803, + "max_kl": 7.476484775543213, + "mean_kl": 1.9789305925369263, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.4404542446136475, + "code_tokens": 28, + "cvar95_kl": 6.035353183746338, + "delta_nll": 2.2810425758361816, + "max_kl": 6.412529468536377, + "mean_kl": 2.531658887863159, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.8996119499206543, + "code_tokens": 57, + "cvar95_kl": 6.892649173736572, + "delta_nll": 3.1575450897216797, + "max_kl": 7.079639434814453, + "mean_kl": 3.192652702331543, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 3.4731011390686035, + "code_tokens": 88, + "cvar95_kl": 7.765056610107422, + "delta_nll": 2.772991180419922, + "max_kl": 9.232954025268555, + "mean_kl": 2.9300954341888428, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.2954545319080353 + }, + { + "candidate_nll": 4.506433010101318, + "code_tokens": 23, + "cvar95_kl": 9.43829345703125, + "delta_nll": 3.3076462745666504, + "max_kl": 9.453438758850098, + "mean_kl": 3.402967691421509, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 5.06937313079834, + "code_tokens": 40, + "cvar95_kl": 10.496138572692871, + "delta_nll": 3.809915781021118, + "max_kl": 11.81523609161377, + "mean_kl": 4.0546488761901855, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.8780486583709717, + "code_tokens": 20, + "cvar95_kl": 8.393904685974121, + "delta_nll": 2.7168335914611816, + "max_kl": 8.393904685974121, + "mean_kl": 3.0694377422332764, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.942516565322876, + "code_tokens": 63, + "cvar95_kl": 7.644972801208496, + "delta_nll": 2.9267311096191406, + "max_kl": 9.349157333374023, + "mean_kl": 2.952731132507324, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 3.4855377674102783, + "code_tokens": 39, + "cvar95_kl": 8.040003776550293, + "delta_nll": 2.5328564643859863, + "max_kl": 8.232375144958496, + "mean_kl": 2.638566017150879, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 4.585461616516113, + "code_tokens": 50, + "cvar95_kl": 8.457428932189941, + "delta_nll": 3.1735968589782715, + "max_kl": 9.027512550354004, + "mean_kl": 3.3378899097442627, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 4.150986671447754, + "code_tokens": 63, + "cvar95_kl": 8.499689102172852, + "delta_nll": 3.183509349822998, + "max_kl": 9.301249504089355, + "mean_kl": 3.474865198135376, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.676019668579102, + "code_tokens": 46, + "cvar95_kl": 7.979352951049805, + "delta_nll": 3.7034575939178467, + "max_kl": 8.527246475219727, + "mean_kl": 3.7098803520202637, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 3.4330849647521973, + "code_tokens": 25, + "cvar95_kl": 7.898374557495117, + "delta_nll": 2.754190444946289, + "max_kl": 8.286746978759766, + "mean_kl": 2.852245807647705, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 3.463066339492798, + "code_tokens": 111, + "cvar95_kl": 5.917394638061523, + "delta_nll": 2.6488003730773926, + "max_kl": 7.566391944885254, + "mean_kl": 2.701338052749634, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.3963963985443115 + }, + { + "candidate_nll": 3.8230178356170654, + "code_tokens": 71, + "cvar95_kl": 7.359728813171387, + "delta_nll": 3.0944600105285645, + "max_kl": 8.351617813110352, + "mean_kl": 3.031318187713623, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.2535211145877838 + }, + { + "candidate_nll": 4.9148149490356445, + "code_tokens": 28, + "cvar95_kl": 7.5524492263793945, + "delta_nll": 3.54409122467041, + "max_kl": 7.57109260559082, + "mean_kl": 3.8348097801208496, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 4.538502216339111, + "code_tokens": 25, + "cvar95_kl": 9.72201156616211, + "delta_nll": 3.3380463123321533, + "max_kl": 10.810713768005371, + "mean_kl": 3.520031213760376, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.6810882091522217, + "code_tokens": 40, + "cvar95_kl": 6.906167984008789, + "delta_nll": 2.3783164024353027, + "max_kl": 7.0977373123168945, + "mean_kl": 2.358424425125122, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.9495325088500977, + "code_tokens": 177, + "cvar95_kl": 7.892244815826416, + "delta_nll": 2.905088424682617, + "max_kl": 9.75378131866455, + "mean_kl": 3.1939170360565186, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.33898305892944336 + }, + { + "candidate_nll": 3.743462562561035, + "code_tokens": 51, + "cvar95_kl": 6.077003002166748, + "delta_nll": 2.6522841453552246, + "max_kl": 6.333522319793701, + "mean_kl": 2.68051815032959, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.37254902720451355 + }, + { + "candidate_nll": 2.721198320388794, + "code_tokens": 69, + "cvar95_kl": 7.031912803649902, + "delta_nll": 2.266737937927246, + "max_kl": 7.617204666137695, + "mean_kl": 2.1993253231048584, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.4637681245803833 + }, + { + "candidate_nll": 3.1732544898986816, + "code_tokens": 26, + "cvar95_kl": 7.666181564331055, + "delta_nll": 2.424994468688965, + "max_kl": 8.146122932434082, + "mean_kl": 2.475752115249634, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.2528746128082275, + "code_tokens": 24, + "cvar95_kl": 4.807531833648682, + "delta_nll": 1.7345353364944458, + "max_kl": 5.123647689819336, + "mean_kl": 1.9276429414749146, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.180885314941406, + "code_tokens": 15, + "cvar95_kl": 8.940790176391602, + "delta_nll": 3.030409336090088, + "max_kl": 8.940790176391602, + "mean_kl": 2.7750461101531982, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 3.782055139541626, + "code_tokens": 43, + "cvar95_kl": 8.57280158996582, + "delta_nll": 2.6327061653137207, + "max_kl": 9.91303539276123, + "mean_kl": 2.9563136100769043, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.3720930218696594 + }, + { + "candidate_nll": 3.582864999771118, + "code_tokens": 18, + "cvar95_kl": 5.745783805847168, + "delta_nll": 2.490539312362671, + "max_kl": 5.745783805847168, + "mean_kl": 2.309659004211426, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.2162883281707764, + "code_tokens": 49, + "cvar95_kl": 6.414535999298096, + "delta_nll": 2.488294839859009, + "max_kl": 6.9036335945129395, + "mean_kl": 2.511274814605713, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.40816327929496765 + }, + { + "candidate_nll": 3.435961961746216, + "code_tokens": 59, + "cvar95_kl": 6.998498916625977, + "delta_nll": 2.5682058334350586, + "max_kl": 8.30756950378418, + "mean_kl": 2.666322708129883, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.32203391194343567 + }, + { + "candidate_nll": 3.0072669982910156, + "code_tokens": 221, + "cvar95_kl": 6.877547740936279, + "delta_nll": 2.4220008850097656, + "max_kl": 8.4724760055542, + "mean_kl": 2.4710402488708496, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.44796380400657654 + }, + { + "candidate_nll": 4.619349956512451, + "code_tokens": 25, + "cvar95_kl": 10.076062202453613, + "delta_nll": 3.4174768924713135, + "max_kl": 10.560651779174805, + "mean_kl": 3.5805225372314453, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.4511497020721436, + "code_tokens": 36, + "cvar95_kl": 7.799508094787598, + "delta_nll": 2.727229118347168, + "max_kl": 8.604866027832031, + "mean_kl": 2.894563674926758, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 4.32676887512207, + "code_tokens": 158, + "cvar95_kl": 9.29021167755127, + "delta_nll": 3.682624101638794, + "max_kl": 11.559188842773438, + "mean_kl": 3.645185947418213, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.1582278460264206 + }, + { + "candidate_nll": 4.43522834777832, + "code_tokens": 92, + "cvar95_kl": 8.499582290649414, + "delta_nll": 3.1196932792663574, + "max_kl": 10.275510787963867, + "mean_kl": 3.056124448776245, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.4021739065647125 + }, + { + "candidate_nll": 3.1385955810546875, + "code_tokens": 102, + "cvar95_kl": 6.309454441070557, + "delta_nll": 2.224552869796753, + "max_kl": 7.372613906860352, + "mean_kl": 2.1844985485076904, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.3921568691730499 + }, + { + "candidate_nll": 3.989751100540161, + "code_tokens": 171, + "cvar95_kl": 9.141079902648926, + "delta_nll": 3.3797810077667236, + "max_kl": 11.442209243774414, + "mean_kl": 3.377480983734131, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 4.002072334289551, + "code_tokens": 40, + "cvar95_kl": 9.908188819885254, + "delta_nll": 2.797208309173584, + "max_kl": 11.190491676330566, + "mean_kl": 2.9982070922851562, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.371395587921143, + "code_tokens": 48, + "cvar95_kl": 7.075033187866211, + "delta_nll": 3.1353816986083984, + "max_kl": 7.802196502685547, + "mean_kl": 3.114060163497925, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.522826194763184, + "code_tokens": 24, + "cvar95_kl": 8.17225456237793, + "delta_nll": 2.917905807495117, + "max_kl": 8.669073104858398, + "mean_kl": 3.420656204223633, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.530897855758667, + "code_tokens": 48, + "cvar95_kl": 6.451480865478516, + "delta_nll": 2.5437936782836914, + "max_kl": 6.95653772354126, + "mean_kl": 2.4565722942352295, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 2.894230365753174, + "code_tokens": 25, + "cvar95_kl": 5.728975296020508, + "delta_nll": 1.5073472261428833, + "max_kl": 5.870097637176514, + "mean_kl": 1.7816917896270752, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 4.764283657073975, + "code_tokens": 30, + "cvar95_kl": 8.461737632751465, + "delta_nll": 3.2655386924743652, + "max_kl": 9.409608840942383, + "mean_kl": 3.132901430130005, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.4922797679901123, + "code_tokens": 19, + "cvar95_kl": 7.186466693878174, + "delta_nll": 2.0654618740081787, + "max_kl": 7.186466693878174, + "mean_kl": 2.2820777893066406, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 2.6483023166656494, + "code_tokens": 12, + "cvar95_kl": 4.717710494995117, + "delta_nll": 1.5692298412322998, + "max_kl": 4.717710494995117, + "mean_kl": 1.9909361600875854, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.076097011566162, + "code_tokens": 37, + "cvar95_kl": 9.942049026489258, + "delta_nll": 2.7357873916625977, + "max_kl": 10.447565078735352, + "mean_kl": 2.921478748321533, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.118455648422241, + "code_tokens": 33, + "cvar95_kl": 5.794014930725098, + "delta_nll": 1.9130258560180664, + "max_kl": 6.58373498916626, + "mean_kl": 2.1336894035339355, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.88808012008667, + "code_tokens": 94, + "cvar95_kl": 9.328218460083008, + "delta_nll": 4.035970211029053, + "max_kl": 10.49376392364502, + "mean_kl": 4.234086990356445, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.24468085169792175 + }, + { + "candidate_nll": 4.277074337005615, + "code_tokens": 90, + "cvar95_kl": 10.20773696899414, + "delta_nll": 3.6523094177246094, + "max_kl": 10.987202644348145, + "mean_kl": 3.6561801433563232, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.442533493041992, + "code_tokens": 47, + "cvar95_kl": 5.635097503662109, + "delta_nll": 1.99476957321167, + "max_kl": 6.575863361358643, + "mean_kl": 2.423017978668213, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 3.690051555633545, + "code_tokens": 61, + "cvar95_kl": 7.739269256591797, + "delta_nll": 2.7662174701690674, + "max_kl": 9.299558639526367, + "mean_kl": 3.076956272125244, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.2950819730758667 + }, + { + "candidate_nll": 3.719672679901123, + "code_tokens": 39, + "cvar95_kl": 6.217904567718506, + "delta_nll": 2.4961578845977783, + "max_kl": 6.301850318908691, + "mean_kl": 2.6244797706604004, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.43589743971824646 + }, + { + "candidate_nll": 3.878516435623169, + "code_tokens": 152, + "cvar95_kl": 8.22654914855957, + "delta_nll": 2.946354627609253, + "max_kl": 9.766874313354492, + "mean_kl": 3.015110969543457, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.33552631735801697 + }, + { + "candidate_nll": 4.288018703460693, + "code_tokens": 100, + "cvar95_kl": 8.547540664672852, + "delta_nll": 3.3190362453460693, + "max_kl": 10.822314262390137, + "mean_kl": 3.235185146331787, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.485544204711914, + "code_tokens": 89, + "cvar95_kl": 7.765100002288818, + "delta_nll": 2.8646163940429688, + "max_kl": 8.716588020324707, + "mean_kl": 2.655360460281372, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.43820226192474365 + }, + { + "candidate_nll": 4.391819953918457, + "code_tokens": 40, + "cvar95_kl": 8.727289199829102, + "delta_nll": 3.0808186531066895, + "max_kl": 9.150653839111328, + "mean_kl": 3.3332486152648926, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 3.465055465698242, + "code_tokens": 76, + "cvar95_kl": 7.023812294006348, + "delta_nll": 2.763488531112671, + "max_kl": 9.376991271972656, + "mean_kl": 2.7561423778533936, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.30263158679008484 + }, + { + "candidate_nll": 3.4662795066833496, + "code_tokens": 56, + "cvar95_kl": 7.886758804321289, + "delta_nll": 2.217292308807373, + "max_kl": 8.89538860321045, + "mean_kl": 2.308408737182617, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.4464285671710968 + }, + { + "candidate_nll": 5.036661624908447, + "code_tokens": 34, + "cvar95_kl": 8.516210556030273, + "delta_nll": 3.5472311973571777, + "max_kl": 9.386819839477539, + "mean_kl": 3.5216524600982666, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 5.066275119781494, + "code_tokens": 96, + "cvar95_kl": 11.320711135864258, + "delta_nll": 4.3881635665893555, + "max_kl": 12.15442180633545, + "mean_kl": 4.2397308349609375, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 3.3865418434143066, + "code_tokens": 451, + "cvar95_kl": 9.03865909576416, + "delta_nll": 2.6878466606140137, + "max_kl": 14.081175804138184, + "mean_kl": 2.655384063720703, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.38359200954437256 + }, + { + "candidate_nll": 4.797451496124268, + "code_tokens": 37, + "cvar95_kl": 10.116357803344727, + "delta_nll": 3.1851418018341064, + "max_kl": 11.963661193847656, + "mean_kl": 3.453181505203247, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.9189794063568115, + "code_tokens": 42, + "cvar95_kl": 8.34898853302002, + "delta_nll": 2.6773815155029297, + "max_kl": 9.447853088378906, + "mean_kl": 2.936580181121826, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.261904776096344 + }, + { + "candidate_nll": 3.642789363861084, + "code_tokens": 33, + "cvar95_kl": 6.845289707183838, + "delta_nll": 2.626033306121826, + "max_kl": 7.374529838562012, + "mean_kl": 2.5786850452423096, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.801136016845703, + "code_tokens": 43, + "cvar95_kl": 7.630995273590088, + "delta_nll": 2.875084400177002, + "max_kl": 8.190771102905273, + "mean_kl": 2.91093111038208, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 4.056360721588135, + "code_tokens": 73, + "cvar95_kl": 7.943275451660156, + "delta_nll": 3.3199243545532227, + "max_kl": 8.715255737304688, + "mean_kl": 3.3655354976654053, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.24657534062862396 + }, + { + "candidate_nll": 3.214185953140259, + "code_tokens": 15, + "cvar95_kl": 6.469430923461914, + "delta_nll": 2.3017096519470215, + "max_kl": 6.469430923461914, + "mean_kl": 2.073618173599243, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 4.367806434631348, + "code_tokens": 34, + "cvar95_kl": 8.243708610534668, + "delta_nll": 3.1087265014648438, + "max_kl": 8.320597648620605, + "mean_kl": 3.793396472930908, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 4.1568827629089355, + "code_tokens": 136, + "cvar95_kl": 7.6391496658325195, + "delta_nll": 3.0483994483947754, + "max_kl": 9.954266548156738, + "mean_kl": 2.928671360015869, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.27941176295280457 + }, + { + "candidate_nll": 2.9610512256622314, + "code_tokens": 17, + "cvar95_kl": 3.398441791534424, + "delta_nll": 1.7148951292037964, + "max_kl": 3.398441791534424, + "mean_kl": 1.9286119937896729, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.259590148925781, + "code_tokens": 33, + "cvar95_kl": 10.549267768859863, + "delta_nll": 3.4665355682373047, + "max_kl": 11.57019329071045, + "mean_kl": 3.4944722652435303, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.1958842277526855, + "code_tokens": 38, + "cvar95_kl": 6.355371952056885, + "delta_nll": 3.3534374237060547, + "max_kl": 6.389277935028076, + "mean_kl": 3.2669835090637207, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 3.7892112731933594, + "code_tokens": 61, + "cvar95_kl": 9.153039932250977, + "delta_nll": 3.3916866779327393, + "max_kl": 10.813023567199707, + "mean_kl": 3.4532582759857178, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.26229506731033325 + }, + { + "candidate_nll": 3.2532944679260254, + "code_tokens": 143, + "cvar95_kl": 7.662820816040039, + "delta_nll": 2.5175750255584717, + "max_kl": 9.185209274291992, + "mean_kl": 2.632648229598999, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.41258740425109863 + }, + { + "candidate_nll": 3.108098030090332, + "code_tokens": 41, + "cvar95_kl": 7.590242385864258, + "delta_nll": 2.0190320014953613, + "max_kl": 8.510108947753906, + "mean_kl": 2.43302845954895, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.4146341383457184 + }, + { + "candidate_nll": 2.6033706665039062, + "code_tokens": 67, + "cvar95_kl": 8.234369277954102, + "delta_nll": 2.2386889457702637, + "max_kl": 10.513299942016602, + "mean_kl": 2.502105712890625, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.447761207818985 + }, + { + "candidate_nll": 4.576602458953857, + "code_tokens": 76, + "cvar95_kl": 9.0535306930542, + "delta_nll": 3.186342477798462, + "max_kl": 10.016695022583008, + "mean_kl": 3.209146499633789, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.22368420660495758 + }, + { + "candidate_nll": 3.5362446308135986, + "code_tokens": 137, + "cvar95_kl": 7.398899555206299, + "delta_nll": 2.9003920555114746, + "max_kl": 8.809808731079102, + "mean_kl": 2.9159433841705322, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.37226277589797974 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l13_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 13 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 5.075494289398193, + "median": 2.8644540309906006, + "minimum": 0.9457663297653198, + "q25": 2.399072229862213, + "q75": 3.294174373149872 + }, + "task_macro": { + "candidate_nll": 3.8557361352443693, + "delta_nll": 2.841689552307129, + "mean_kl": 2.930440157175064, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.34927950155735016 + }, + "token_weighted": { + "candidate_nll": 3.8986752033233643, + "cvar95_kl": 8.757649421691895, + "delta_nll": 2.981837749481201, + "max_kl": 18.288694381713867, + "mean_kl": 3.04441237449646, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.3365626335144043 + } + }, + "random_l18_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.5038116931915284, + "macro_mean_kl": 2.631141842842102, + "macro_top1_agreement": 0.3867150011062622, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 2.9509490556716917, + "macro_mean_kl": 3.0957938261032103, + "macro_top1_agreement": 0.31811008107662203, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 3.0493591680526735, + "macro_mean_kl": 3.1024735078811645, + "macro_top1_agreement": 0.3214999887943268, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 3.2107197017669677, + "macro_mean_kl": 3.2248175373077395, + "macro_top1_agreement": 0.3134842063784599, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 3.70989990234375, + "code_tokens": 98, + "cvar95_kl": 8.322578430175781, + "delta_nll": 2.983044147491455, + "max_kl": 10.579853057861328, + "mean_kl": 3.2960755825042725, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.33673468232154846 + }, + { + "candidate_nll": 4.052554607391357, + "code_tokens": 19, + "cvar95_kl": 8.701200485229492, + "delta_nll": 2.893141508102417, + "max_kl": 8.701200485229492, + "mean_kl": 3.1374905109405518, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.9430079460144043, + "code_tokens": 39, + "cvar95_kl": 10.169504165649414, + "delta_nll": 2.9276585578918457, + "max_kl": 13.024130821228027, + "mean_kl": 3.2627968788146973, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 4.81631326675415, + "code_tokens": 24, + "cvar95_kl": 7.0300798416137695, + "delta_nll": 3.370023727416992, + "max_kl": 7.536533355712891, + "mean_kl": 3.219552755355835, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.875211715698242, + "code_tokens": 25, + "cvar95_kl": 8.483504295349121, + "delta_nll": 2.4674534797668457, + "max_kl": 10.129018783569336, + "mean_kl": 2.4476559162139893, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.491006851196289, + "code_tokens": 55, + "cvar95_kl": 9.70478343963623, + "delta_nll": 3.59539532661438, + "max_kl": 12.548402786254883, + "mean_kl": 3.457770347595215, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.30909091234207153 + }, + { + "candidate_nll": 3.134915828704834, + "code_tokens": 16, + "cvar95_kl": 6.0306291580200195, + "delta_nll": 2.3054752349853516, + "max_kl": 6.0306291580200195, + "mean_kl": 2.205747365951538, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 5.066781997680664, + "code_tokens": 199, + "cvar95_kl": 10.142195701599121, + "delta_nll": 3.844888687133789, + "max_kl": 13.23518180847168, + "mean_kl": 3.8979878425598145, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.23618090152740479 + }, + { + "candidate_nll": 4.3746232986450195, + "code_tokens": 28, + "cvar95_kl": 5.716506004333496, + "delta_nll": 2.8085451126098633, + "max_kl": 5.7608160972595215, + "mean_kl": 2.6228179931640625, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.852185010910034, + "code_tokens": 87, + "cvar95_kl": 7.920856475830078, + "delta_nll": 2.9792022705078125, + "max_kl": 9.57504940032959, + "mean_kl": 2.9622802734375, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.28735631704330444 + }, + { + "candidate_nll": 3.8655810356140137, + "code_tokens": 35, + "cvar95_kl": 9.241825103759766, + "delta_nll": 2.6326169967651367, + "max_kl": 9.605985641479492, + "mean_kl": 3.517037868499756, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 3.8251001834869385, + "code_tokens": 62, + "cvar95_kl": 8.884933471679688, + "delta_nll": 3.1858086585998535, + "max_kl": 10.679398536682129, + "mean_kl": 3.057346820831299, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.4032258093357086 + }, + { + "candidate_nll": 5.257329940795898, + "code_tokens": 58, + "cvar95_kl": 10.436347961425781, + "delta_nll": 4.1522722244262695, + "max_kl": 10.594806671142578, + "mean_kl": 4.017022609710693, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 4.2677226066589355, + "code_tokens": 79, + "cvar95_kl": 7.491622447967529, + "delta_nll": 3.1086745262145996, + "max_kl": 7.639619827270508, + "mean_kl": 3.2985105514526367, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.27848100662231445 + }, + { + "candidate_nll": 3.8281922340393066, + "code_tokens": 58, + "cvar95_kl": 6.940710067749023, + "delta_nll": 2.735072374343872, + "max_kl": 7.722843170166016, + "mean_kl": 2.862011671066284, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.789583683013916, + "code_tokens": 48, + "cvar95_kl": 8.3093900680542, + "delta_nll": 3.6273858547210693, + "max_kl": 9.166685104370117, + "mean_kl": 3.897073984146118, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 4.507297515869141, + "code_tokens": 37, + "cvar95_kl": 9.446073532104492, + "delta_nll": 3.3818650245666504, + "max_kl": 10.598751068115234, + "mean_kl": 3.186691999435425, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.5482819080352783, + "code_tokens": 61, + "cvar95_kl": 7.366473197937012, + "delta_nll": 2.840874671936035, + "max_kl": 9.846263885498047, + "mean_kl": 2.8312134742736816, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.32786884903907776 + }, + { + "candidate_nll": 3.823202133178711, + "code_tokens": 77, + "cvar95_kl": 7.580494403839111, + "delta_nll": 3.0534825325012207, + "max_kl": 8.916223526000977, + "mean_kl": 2.9255731105804443, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.380863666534424, + "code_tokens": 98, + "cvar95_kl": 8.057682991027832, + "delta_nll": 3.3591604232788086, + "max_kl": 10.225848197937012, + "mean_kl": 3.234907388687134, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 5.262636184692383, + "code_tokens": 162, + "cvar95_kl": 10.518219947814941, + "delta_nll": 4.67748498916626, + "max_kl": 12.24717903137207, + "mean_kl": 4.748258113861084, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.12962962687015533 + }, + { + "candidate_nll": 4.585906505584717, + "code_tokens": 103, + "cvar95_kl": 9.3546724319458, + "delta_nll": 3.841440439224243, + "max_kl": 10.115129470825195, + "mean_kl": 3.8192055225372314, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.2330097109079361 + }, + { + "candidate_nll": 5.0287909507751465, + "code_tokens": 73, + "cvar95_kl": 7.521305084228516, + "delta_nll": 3.709820508956909, + "max_kl": 8.156673431396484, + "mean_kl": 3.470097064971924, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.3287671208381653 + }, + { + "candidate_nll": 4.5585808753967285, + "code_tokens": 119, + "cvar95_kl": 9.634954452514648, + "delta_nll": 3.5756072998046875, + "max_kl": 13.468085289001465, + "mean_kl": 3.715654134750366, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.21848739683628082 + }, + { + "candidate_nll": 2.2491064071655273, + "code_tokens": 17, + "cvar95_kl": 6.851127624511719, + "delta_nll": 1.059583067893982, + "max_kl": 6.851127624511719, + "mean_kl": 2.0745599269866943, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.101869106292725, + "code_tokens": 56, + "cvar95_kl": 7.889273166656494, + "delta_nll": 2.6761674880981445, + "max_kl": 8.552401542663574, + "mean_kl": 2.7394659519195557, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.6104087829589844, + "code_tokens": 57, + "cvar95_kl": 8.116435050964355, + "delta_nll": 3.1430482864379883, + "max_kl": 8.497602462768555, + "mean_kl": 3.052755355834961, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.5446231365203857, + "code_tokens": 63, + "cvar95_kl": 7.893267631530762, + "delta_nll": 2.9118778705596924, + "max_kl": 8.247265815734863, + "mean_kl": 2.940920829772949, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 5.220050811767578, + "code_tokens": 200, + "cvar95_kl": 11.761218070983887, + "delta_nll": 4.382115364074707, + "max_kl": 14.375763893127441, + "mean_kl": 4.431615829467773, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.23499999940395355 + }, + { + "candidate_nll": 4.101775169372559, + "code_tokens": 30, + "cvar95_kl": 7.570754528045654, + "delta_nll": 3.0175724029541016, + "max_kl": 8.101629257202148, + "mean_kl": 3.1060867309570312, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.687589645385742, + "code_tokens": 28, + "cvar95_kl": 11.402727127075195, + "delta_nll": 3.0712385177612305, + "max_kl": 12.232163429260254, + "mean_kl": 2.898650884628296, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.612377643585205, + "code_tokens": 26, + "cvar95_kl": 6.769450664520264, + "delta_nll": 2.841421127319336, + "max_kl": 6.899691104888916, + "mean_kl": 2.7188470363616943, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.3833019733428955, + "code_tokens": 54, + "cvar95_kl": 7.677497863769531, + "delta_nll": 2.550360679626465, + "max_kl": 10.03904914855957, + "mean_kl": 2.537705898284912, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 3.2630345821380615, + "code_tokens": 47, + "cvar95_kl": 5.764184474945068, + "delta_nll": 2.3062984943389893, + "max_kl": 6.3349151611328125, + "mean_kl": 2.377530574798584, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.3404255211353302 + }, + { + "candidate_nll": 4.2377753257751465, + "code_tokens": 93, + "cvar95_kl": 8.492947578430176, + "delta_nll": 3.5032458305358887, + "max_kl": 9.239307403564453, + "mean_kl": 3.297086000442505, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.34408602118492126 + }, + { + "candidate_nll": 3.8704161643981934, + "code_tokens": 30, + "cvar95_kl": 8.869156837463379, + "delta_nll": 3.0110416412353516, + "max_kl": 10.35401439666748, + "mean_kl": 3.0475847721099854, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.328987121582031, + "code_tokens": 89, + "cvar95_kl": 7.563601493835449, + "delta_nll": 3.260483741760254, + "max_kl": 9.109837532043457, + "mean_kl": 3.242568016052246, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.24719101190567017 + }, + { + "candidate_nll": 4.53508996963501, + "code_tokens": 70, + "cvar95_kl": 9.571547508239746, + "delta_nll": 3.57084059715271, + "max_kl": 11.689229011535645, + "mean_kl": 3.7467803955078125, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.346528053283691, + "code_tokens": 26, + "cvar95_kl": 6.5939812660217285, + "delta_nll": 2.7333009243011475, + "max_kl": 6.947154998779297, + "mean_kl": 3.2277402877807617, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.5965538024902344, + "code_tokens": 48, + "cvar95_kl": 8.815651893615723, + "delta_nll": 2.6388468742370605, + "max_kl": 12.169116020202637, + "mean_kl": 2.7839479446411133, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.47057843208313, + "code_tokens": 29, + "cvar95_kl": 7.776031017303467, + "delta_nll": 2.6014304161071777, + "max_kl": 7.973322868347168, + "mean_kl": 2.698563575744629, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 4.032767295837402, + "code_tokens": 17, + "cvar95_kl": 7.805535793304443, + "delta_nll": 2.9196043014526367, + "max_kl": 7.805535793304443, + "mean_kl": 2.7943248748779297, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 3.0566964149475098, + "code_tokens": 37, + "cvar95_kl": 6.143359184265137, + "delta_nll": 2.1364803314208984, + "max_kl": 6.689383506774902, + "mean_kl": 2.2119133472442627, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.3513513505458832 + }, + { + "candidate_nll": 5.052004337310791, + "code_tokens": 121, + "cvar95_kl": 11.689730644226074, + "delta_nll": 4.540602684020996, + "max_kl": 16.496646881103516, + "mean_kl": 4.481089115142822, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.21487602591514587 + }, + { + "candidate_nll": 3.920259714126587, + "code_tokens": 30, + "cvar95_kl": 8.35438060760498, + "delta_nll": 2.521415948867798, + "max_kl": 10.074835777282715, + "mean_kl": 2.8444714546203613, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.086486339569092, + "code_tokens": 74, + "cvar95_kl": 8.355668067932129, + "delta_nll": 3.188014507293701, + "max_kl": 9.338788986206055, + "mean_kl": 3.2250301837921143, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.583967685699463, + "code_tokens": 53, + "cvar95_kl": 7.55525541305542, + "delta_nll": 2.7495439052581787, + "max_kl": 8.019559860229492, + "mean_kl": 2.5951011180877686, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.3962264060974121 + }, + { + "candidate_nll": 4.000837802886963, + "code_tokens": 21, + "cvar95_kl": 5.246932029724121, + "delta_nll": 2.141442060470581, + "max_kl": 5.271576881408691, + "mean_kl": 2.498265504837036, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.788787603378296, + "code_tokens": 20, + "cvar95_kl": 7.322728157043457, + "delta_nll": 1.4263739585876465, + "max_kl": 7.322728157043457, + "mean_kl": 1.5214214324951172, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 3.2066195011138916, + "code_tokens": 122, + "cvar95_kl": 6.127039432525635, + "delta_nll": 2.3876211643218994, + "max_kl": 7.635927200317383, + "mean_kl": 2.426684856414795, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.3852458894252777 + }, + { + "candidate_nll": 4.10105562210083, + "code_tokens": 94, + "cvar95_kl": 8.398103713989258, + "delta_nll": 3.260915756225586, + "max_kl": 8.727595329284668, + "mean_kl": 3.310546875, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 3.0529391765594482, + "code_tokens": 12, + "cvar95_kl": 5.2806572914123535, + "delta_nll": 1.9876376390457153, + "max_kl": 5.2806572914123535, + "mean_kl": 1.8605700731277466, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.82847261428833, + "code_tokens": 38, + "cvar95_kl": 8.72420883178711, + "delta_nll": 3.3220725059509277, + "max_kl": 9.366352081298828, + "mean_kl": 3.8080170154571533, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 2.113170862197876, + "code_tokens": 58, + "cvar95_kl": 5.900074005126953, + "delta_nll": 1.2083837985992432, + "max_kl": 6.3458685874938965, + "mean_kl": 1.375970721244812, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.6034482717514038 + }, + { + "candidate_nll": 4.801281452178955, + "code_tokens": 54, + "cvar95_kl": 9.860047340393066, + "delta_nll": 4.205017566680908, + "max_kl": 11.275351524353027, + "mean_kl": 4.138827323913574, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.6371371746063232, + "code_tokens": 40, + "cvar95_kl": 6.860194206237793, + "delta_nll": 2.705597400665283, + "max_kl": 7.068366527557373, + "mean_kl": 2.968705892562866, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.428689479827881, + "code_tokens": 115, + "cvar95_kl": 7.346038818359375, + "delta_nll": 2.9161434173583984, + "max_kl": 10.217085838317871, + "mean_kl": 2.881976842880249, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.313043475151062 + }, + { + "candidate_nll": 4.015039443969727, + "code_tokens": 60, + "cvar95_kl": 9.582165718078613, + "delta_nll": 3.4904258251190186, + "max_kl": 10.603326797485352, + "mean_kl": 3.553147554397583, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 3.9490606784820557, + "code_tokens": 124, + "cvar95_kl": 7.828566074371338, + "delta_nll": 3.1790735721588135, + "max_kl": 10.029915809631348, + "mean_kl": 3.145932197570801, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.221773147583008, + "code_tokens": 91, + "cvar95_kl": 11.613157272338867, + "delta_nll": 4.451024532318115, + "max_kl": 12.530877113342285, + "mean_kl": 4.4276604652404785, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.19780220091342926 + }, + { + "candidate_nll": 5.792234897613525, + "code_tokens": 125, + "cvar95_kl": 12.777719497680664, + "delta_nll": 4.972317695617676, + "max_kl": 14.445263862609863, + "mean_kl": 5.038930892944336, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.20800000429153442 + }, + { + "candidate_nll": 2.5686514377593994, + "code_tokens": 26, + "cvar95_kl": 4.185728549957275, + "delta_nll": 1.4327749013900757, + "max_kl": 4.237484931945801, + "mean_kl": 1.9134196043014526, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 3.9652926921844482, + "code_tokens": 24, + "cvar95_kl": 6.701950550079346, + "delta_nll": 1.724867582321167, + "max_kl": 7.697032928466797, + "mean_kl": 2.1792380809783936, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.6277577877044678, + "code_tokens": 180, + "cvar95_kl": 8.580789566040039, + "delta_nll": 2.773576021194458, + "max_kl": 11.667401313781738, + "mean_kl": 2.8364017009735107, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.35555556416511536 + }, + { + "candidate_nll": 5.210252285003662, + "code_tokens": 40, + "cvar95_kl": 9.964037895202637, + "delta_nll": 3.8354437351226807, + "max_kl": 11.59042739868164, + "mean_kl": 3.9403598308563232, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 4.049832344055176, + "code_tokens": 72, + "cvar95_kl": 8.238871574401855, + "delta_nll": 3.0748355388641357, + "max_kl": 9.231865882873535, + "mean_kl": 3.0305237770080566, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.9848952293395996, + "code_tokens": 20, + "cvar95_kl": 4.908429145812988, + "delta_nll": 1.4073240756988525, + "max_kl": 4.908429145812988, + "mean_kl": 1.9506791830062866, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.2868499755859375, + "code_tokens": 76, + "cvar95_kl": 7.822705268859863, + "delta_nll": 3.296205520629883, + "max_kl": 8.252739906311035, + "mean_kl": 3.341217517852783, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 4.424980640411377, + "code_tokens": 42, + "cvar95_kl": 9.121451377868652, + "delta_nll": 3.053239345550537, + "max_kl": 9.91227912902832, + "mean_kl": 3.015810489654541, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 2.373901605606079, + "code_tokens": 28, + "cvar95_kl": 5.941412925720215, + "delta_nll": 1.7906785011291504, + "max_kl": 6.358156681060791, + "mean_kl": 1.6681058406829834, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.6071428656578064 + }, + { + "candidate_nll": 4.746124744415283, + "code_tokens": 58, + "cvar95_kl": 10.256787300109863, + "delta_nll": 3.7126479148864746, + "max_kl": 10.828311920166016, + "mean_kl": 3.8107781410217285, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 3.767188787460327, + "code_tokens": 25, + "cvar95_kl": 7.02468204498291, + "delta_nll": 2.8985393047332764, + "max_kl": 7.238971710205078, + "mean_kl": 3.1131887435913086, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.630029678344727, + "code_tokens": 88, + "cvar95_kl": 8.829310417175293, + "delta_nll": 3.6650218963623047, + "max_kl": 9.823257446289062, + "mean_kl": 3.6767585277557373, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.759829521179199, + "code_tokens": 42, + "cvar95_kl": 7.010704040527344, + "delta_nll": 2.632312297821045, + "max_kl": 7.502548694610596, + "mean_kl": 2.4695568084716797, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.80303955078125, + "code_tokens": 24, + "cvar95_kl": 6.626101493835449, + "delta_nll": 2.9834887981414795, + "max_kl": 6.931756019592285, + "mean_kl": 2.9351909160614014, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.799591064453125, + "code_tokens": 24, + "cvar95_kl": 5.194944381713867, + "delta_nll": 2.1147196292877197, + "max_kl": 5.303036689758301, + "mean_kl": 2.1523051261901855, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.3132638931274414, + "code_tokens": 43, + "cvar95_kl": 7.477359771728516, + "delta_nll": 2.246084690093994, + "max_kl": 9.159460067749023, + "mean_kl": 2.204225778579712, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.5348837375640869 + }, + { + "candidate_nll": 3.4412002563476562, + "code_tokens": 24, + "cvar95_kl": 9.270530700683594, + "delta_nll": 2.554980516433716, + "max_kl": 9.739564895629883, + "mean_kl": 2.799034357070923, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.811619520187378, + "code_tokens": 14, + "cvar95_kl": 5.408828258514404, + "delta_nll": 2.4989938735961914, + "max_kl": 5.408828258514404, + "mean_kl": 2.481755495071411, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.762702703475952, + "code_tokens": 41, + "cvar95_kl": 7.150827884674072, + "delta_nll": 3.2716593742370605, + "max_kl": 8.095575332641602, + "mean_kl": 2.9349899291992188, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.31707316637039185 + }, + { + "candidate_nll": 4.453085899353027, + "code_tokens": 32, + "cvar95_kl": 8.99603271484375, + "delta_nll": 3.4647459983825684, + "max_kl": 9.570990562438965, + "mean_kl": 3.3901143074035645, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 2.7736856937408447, + "code_tokens": 53, + "cvar95_kl": 5.929008483886719, + "delta_nll": 2.011990785598755, + "max_kl": 6.563986778259277, + "mean_kl": 1.8861089944839478, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.4716981053352356 + }, + { + "candidate_nll": 3.309993028640747, + "code_tokens": 51, + "cvar95_kl": 8.353545188903809, + "delta_nll": 2.931701898574829, + "max_kl": 9.081379890441895, + "mean_kl": 2.940201759338379, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.3137255012989044 + }, + { + "candidate_nll": 4.406529426574707, + "code_tokens": 36, + "cvar95_kl": 7.2890753746032715, + "delta_nll": 2.71437406539917, + "max_kl": 7.872932434082031, + "mean_kl": 2.8801956176757812, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.448045492172241, + "code_tokens": 24, + "cvar95_kl": 5.125918388366699, + "delta_nll": 1.8872932195663452, + "max_kl": 5.505677223205566, + "mean_kl": 2.007335901260376, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 4.993149757385254, + "code_tokens": 39, + "cvar95_kl": 9.907585144042969, + "delta_nll": 3.6788153648376465, + "max_kl": 10.377482414245605, + "mean_kl": 4.111523628234863, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 5.6880598068237305, + "code_tokens": 84, + "cvar95_kl": 10.362805366516113, + "delta_nll": 4.989647388458252, + "max_kl": 14.64201831817627, + "mean_kl": 5.0307722091674805, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 3.2650647163391113, + "code_tokens": 32, + "cvar95_kl": 6.397024154663086, + "delta_nll": 2.342296600341797, + "max_kl": 6.896480560302734, + "mean_kl": 2.2103309631347656, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.515883922576904, + "code_tokens": 19, + "cvar95_kl": 10.063623428344727, + "delta_nll": 3.2057204246520996, + "max_kl": 10.063623428344727, + "mean_kl": 3.1376700401306152, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.4736842215061188 + }, + { + "candidate_nll": 3.912717342376709, + "code_tokens": 50, + "cvar95_kl": 8.565524101257324, + "delta_nll": 3.094740629196167, + "max_kl": 10.00715160369873, + "mean_kl": 2.979215145111084, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 3.2739083766937256, + "code_tokens": 71, + "cvar95_kl": 7.642982006072998, + "delta_nll": 2.774094820022583, + "max_kl": 11.284255027770996, + "mean_kl": 2.6930432319641113, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.39436620473861694 + }, + { + "candidate_nll": 4.332018852233887, + "code_tokens": 28, + "cvar95_kl": 9.661160469055176, + "delta_nll": 2.9440503120422363, + "max_kl": 9.869839668273926, + "mean_kl": 3.2172696590423584, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 2.9612748622894287, + "code_tokens": 76, + "cvar95_kl": 6.0572919845581055, + "delta_nll": 2.092013359069824, + "max_kl": 7.203418254852295, + "mean_kl": 2.0875046253204346, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.40789473056793213 + }, + { + "candidate_nll": 3.8230748176574707, + "code_tokens": 34, + "cvar95_kl": 6.006806373596191, + "delta_nll": 2.8604896068573, + "max_kl": 6.146864414215088, + "mean_kl": 2.9594719409942627, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 4.175703048706055, + "code_tokens": 11, + "cvar95_kl": 4.162962436676025, + "delta_nll": 2.028311252593994, + "max_kl": 4.162962436676025, + "mean_kl": 1.9882010221481323, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 3.5956599712371826, + "code_tokens": 31, + "cvar95_kl": 8.680103302001953, + "delta_nll": 2.1793923377990723, + "max_kl": 8.853286743164062, + "mean_kl": 2.759459972381592, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 3.275393009185791, + "code_tokens": 118, + "cvar95_kl": 6.395573139190674, + "delta_nll": 2.428616762161255, + "max_kl": 6.892744064331055, + "mean_kl": 2.375847578048706, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.3813559412956238 + }, + { + "candidate_nll": 2.7663912773132324, + "code_tokens": 60, + "cvar95_kl": 9.743992805480957, + "delta_nll": 1.990695595741272, + "max_kl": 11.47535514831543, + "mean_kl": 2.2654404640197754, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.122115612030029, + "code_tokens": 47, + "cvar95_kl": 7.202381134033203, + "delta_nll": 3.226665735244751, + "max_kl": 7.726921081542969, + "mean_kl": 3.3011391162872314, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.23404255509376526 + }, + { + "candidate_nll": 3.8447582721710205, + "code_tokens": 84, + "cvar95_kl": 8.431634902954102, + "delta_nll": 2.6927437782287598, + "max_kl": 11.816529273986816, + "mean_kl": 2.8174915313720703, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.722680568695068, + "code_tokens": 27, + "cvar95_kl": 10.705411911010742, + "delta_nll": 3.728654384613037, + "max_kl": 10.74129581451416, + "mean_kl": 3.59365177154541, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.657522201538086, + "code_tokens": 19, + "cvar95_kl": 5.585982799530029, + "delta_nll": 2.0456151962280273, + "max_kl": 5.585982799530029, + "mean_kl": 1.9888830184936523, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 3.9815566539764404, + "code_tokens": 114, + "cvar95_kl": 8.866265296936035, + "delta_nll": 3.2085442543029785, + "max_kl": 10.039682388305664, + "mean_kl": 3.41296648979187, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 4.770220756530762, + "code_tokens": 40, + "cvar95_kl": 9.118658065795898, + "delta_nll": 3.042327404022217, + "max_kl": 9.288849830627441, + "mean_kl": 3.455263614654541, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.8383781909942627, + "code_tokens": 25, + "cvar95_kl": 6.410867691040039, + "delta_nll": 2.3526999950408936, + "max_kl": 7.212282657623291, + "mean_kl": 2.335973024368286, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 3.850551128387451, + "code_tokens": 23, + "cvar95_kl": 9.068923950195312, + "delta_nll": 3.079862117767334, + "max_kl": 9.670859336853027, + "mean_kl": 2.810330867767334, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.43478259444236755 + }, + { + "candidate_nll": 4.698312759399414, + "code_tokens": 77, + "cvar95_kl": 9.48544692993164, + "delta_nll": 3.6819632053375244, + "max_kl": 11.72766399383545, + "mean_kl": 3.6798770427703857, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.7118117809295654, + "code_tokens": 19, + "cvar95_kl": 7.942556858062744, + "delta_nll": 2.6041464805603027, + "max_kl": 7.942556858062744, + "mean_kl": 2.337299346923828, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 4.1765031814575195, + "code_tokens": 37, + "cvar95_kl": 7.124434471130371, + "delta_nll": 3.1932594776153564, + "max_kl": 8.155378341674805, + "mean_kl": 3.172898530960083, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.8633322715759277, + "code_tokens": 33, + "cvar95_kl": 6.269662380218506, + "delta_nll": 2.4801673889160156, + "max_kl": 7.348388671875, + "mean_kl": 2.7476084232330322, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.071136951446533, + "code_tokens": 95, + "cvar95_kl": 7.918298244476318, + "delta_nll": 3.4112954139709473, + "max_kl": 9.758261680603027, + "mean_kl": 3.3211984634399414, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.3052631616592407 + }, + { + "candidate_nll": 3.8318092823028564, + "code_tokens": 125, + "cvar95_kl": 7.6744585037231445, + "delta_nll": 2.8186488151550293, + "max_kl": 8.240564346313477, + "mean_kl": 2.8985400199890137, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.31200000643730164 + }, + { + "candidate_nll": 5.373783588409424, + "code_tokens": 168, + "cvar95_kl": 11.301513671875, + "delta_nll": 4.660735130310059, + "max_kl": 12.320066452026367, + "mean_kl": 4.599817752838135, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.2202380895614624 + }, + { + "candidate_nll": 3.1719417572021484, + "code_tokens": 31, + "cvar95_kl": 11.850833892822266, + "delta_nll": 1.8565396070480347, + "max_kl": 12.646994590759277, + "mean_kl": 2.6006686687469482, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.4516128897666931 + }, + { + "candidate_nll": 3.949054718017578, + "code_tokens": 83, + "cvar95_kl": 8.851978302001953, + "delta_nll": 3.187347888946533, + "max_kl": 10.39550495147705, + "mean_kl": 3.1328835487365723, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.34939759969711304 + }, + { + "candidate_nll": 4.631404876708984, + "code_tokens": 50, + "cvar95_kl": 8.479079246520996, + "delta_nll": 3.931462287902832, + "max_kl": 10.325839042663574, + "mean_kl": 3.789097309112549, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 3.156048059463501, + "code_tokens": 69, + "cvar95_kl": 6.602736473083496, + "delta_nll": 2.7888660430908203, + "max_kl": 6.901549339294434, + "mean_kl": 2.7925734519958496, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.4057970941066742 + }, + { + "candidate_nll": 4.041256904602051, + "code_tokens": 50, + "cvar95_kl": 8.3121976852417, + "delta_nll": 2.9563608169555664, + "max_kl": 9.676983833312988, + "mean_kl": 3.2230618000030518, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.5647501945495605, + "code_tokens": 161, + "cvar95_kl": 7.981903076171875, + "delta_nll": 3.5609683990478516, + "max_kl": 9.838943481445312, + "mean_kl": 3.493988275527954, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.2732919156551361 + }, + { + "candidate_nll": 5.268013000488281, + "code_tokens": 53, + "cvar95_kl": 11.320048332214355, + "delta_nll": 4.612236976623535, + "max_kl": 13.906181335449219, + "mean_kl": 4.499020099639893, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 4.217673301696777, + "code_tokens": 97, + "cvar95_kl": 9.410329818725586, + "delta_nll": 3.7721452713012695, + "max_kl": 10.272208213806152, + "mean_kl": 3.5837254524230957, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.3711340129375458 + }, + { + "candidate_nll": 3.1126441955566406, + "code_tokens": 22, + "cvar95_kl": 7.075899124145508, + "delta_nll": 1.7988312244415283, + "max_kl": 8.99866771697998, + "mean_kl": 2.0802085399627686, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.5909090638160706 + }, + { + "candidate_nll": 3.1062135696411133, + "code_tokens": 34, + "cvar95_kl": 11.94859790802002, + "delta_nll": 2.237309455871582, + "max_kl": 12.357726097106934, + "mean_kl": 2.822747230529785, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.303642749786377, + "code_tokens": 81, + "cvar95_kl": 8.598745346069336, + "delta_nll": 3.4449994564056396, + "max_kl": 9.381546974182129, + "mean_kl": 3.6449363231658936, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.2469135820865631 + }, + { + "candidate_nll": 2.306974172592163, + "code_tokens": 17, + "cvar95_kl": 4.304419040679932, + "delta_nll": 1.0448708534240723, + "max_kl": 4.304419040679932, + "mean_kl": 1.1512964963912964, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 4.536556720733643, + "code_tokens": 181, + "cvar95_kl": 8.214374542236328, + "delta_nll": 3.711299419403076, + "max_kl": 9.78341293334961, + "mean_kl": 3.667130708694458, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.28176796436309814 + }, + { + "candidate_nll": 3.995229959487915, + "code_tokens": 63, + "cvar95_kl": 6.949040412902832, + "delta_nll": 3.0148215293884277, + "max_kl": 7.349326133728027, + "mean_kl": 3.0820205211639404, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 1.8432854413986206, + "code_tokens": 35, + "cvar95_kl": 4.628061294555664, + "delta_nll": 1.1497623920440674, + "max_kl": 4.722884654998779, + "mean_kl": 1.3565446138381958, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 3.7826874256134033, + "code_tokens": 25, + "cvar95_kl": 5.511308670043945, + "delta_nll": 2.9779000282287598, + "max_kl": 5.702487468719482, + "mean_kl": 2.7341644763946533, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.848271369934082, + "code_tokens": 60, + "cvar95_kl": 8.962112426757812, + "delta_nll": 3.7599058151245117, + "max_kl": 10.186333656311035, + "mean_kl": 3.989363193511963, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.5959062576293945, + "code_tokens": 86, + "cvar95_kl": 9.642512321472168, + "delta_nll": 3.6842594146728516, + "max_kl": 11.814579963684082, + "mean_kl": 3.6118571758270264, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 4.448822975158691, + "code_tokens": 40, + "cvar95_kl": 9.397321701049805, + "delta_nll": 3.5226571559906006, + "max_kl": 10.038260459899902, + "mean_kl": 3.62650990486145, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 3.7109553813934326, + "code_tokens": 27, + "cvar95_kl": 5.365860939025879, + "delta_nll": 2.147395133972168, + "max_kl": 6.08551549911499, + "mean_kl": 2.2140259742736816, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 4.187138557434082, + "code_tokens": 52, + "cvar95_kl": 7.06926155090332, + "delta_nll": 3.097555637359619, + "max_kl": 7.450497627258301, + "mean_kl": 3.4621615409851074, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.2803380489349365, + "code_tokens": 71, + "cvar95_kl": 8.046963691711426, + "delta_nll": 2.538464069366455, + "max_kl": 9.482998847961426, + "mean_kl": 2.580320119857788, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.3802816867828369 + }, + { + "candidate_nll": 3.618210792541504, + "code_tokens": 25, + "cvar95_kl": 7.396241188049316, + "delta_nll": 2.5163590908050537, + "max_kl": 7.518944263458252, + "mean_kl": 2.6330771446228027, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 2.3039917945861816, + "code_tokens": 103, + "cvar95_kl": 5.105419635772705, + "delta_nll": 1.7094961404800415, + "max_kl": 6.198106288909912, + "mean_kl": 1.751166820526123, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.5436893105506897 + }, + { + "candidate_nll": 3.6751487255096436, + "code_tokens": 116, + "cvar95_kl": 8.113242149353027, + "delta_nll": 2.6176106929779053, + "max_kl": 8.929288864135742, + "mean_kl": 2.7149250507354736, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.38793104887008667 + }, + { + "candidate_nll": 4.115263938903809, + "code_tokens": 107, + "cvar95_kl": 8.378853797912598, + "delta_nll": 3.4669177532196045, + "max_kl": 10.373708724975586, + "mean_kl": 3.543414831161499, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.2710280418395996 + }, + { + "candidate_nll": 3.6444685459136963, + "code_tokens": 54, + "cvar95_kl": 6.395841121673584, + "delta_nll": 2.7014307975769043, + "max_kl": 7.978433132171631, + "mean_kl": 2.5741055011749268, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 4.39202356338501, + "code_tokens": 46, + "cvar95_kl": 9.12759017944336, + "delta_nll": 3.7586312294006348, + "max_kl": 10.554317474365234, + "mean_kl": 3.5851941108703613, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.180532932281494, + "code_tokens": 170, + "cvar95_kl": 9.45885944366455, + "delta_nll": 3.851386070251465, + "max_kl": 13.227709770202637, + "mean_kl": 3.7710578441619873, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 3.823624610900879, + "code_tokens": 45, + "cvar95_kl": 6.735342025756836, + "delta_nll": 2.396932363510132, + "max_kl": 7.699026107788086, + "mean_kl": 2.639596939086914, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.2888889014720917 + }, + { + "candidate_nll": 4.25349235534668, + "code_tokens": 24, + "cvar95_kl": 7.1982245445251465, + "delta_nll": 2.845189094543457, + "max_kl": 7.665313243865967, + "mean_kl": 3.1293985843658447, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 4.041115760803223, + "code_tokens": 78, + "cvar95_kl": 8.512816429138184, + "delta_nll": 2.8791558742523193, + "max_kl": 9.92905044555664, + "mean_kl": 2.9568560123443604, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 4.044880390167236, + "code_tokens": 33, + "cvar95_kl": 7.083530902862549, + "delta_nll": 3.414323329925537, + "max_kl": 7.230129241943359, + "mean_kl": 3.2114973068237305, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.799749374389648, + "code_tokens": 32, + "cvar95_kl": 9.231512069702148, + "delta_nll": 3.684079647064209, + "max_kl": 9.42420482635498, + "mean_kl": 3.516507863998413, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.173980236053467, + "code_tokens": 77, + "cvar95_kl": 9.058392524719238, + "delta_nll": 4.185230255126953, + "max_kl": 9.404396057128906, + "mean_kl": 4.0290303230285645, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.16883116960525513 + }, + { + "candidate_nll": 2.4303529262542725, + "code_tokens": 222, + "cvar95_kl": 7.019921779632568, + "delta_nll": 2.0081024169921875, + "max_kl": 10.148489952087402, + "mean_kl": 2.032017230987549, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.5495495200157166 + }, + { + "candidate_nll": 3.9115054607391357, + "code_tokens": 86, + "cvar95_kl": 7.893591403961182, + "delta_nll": 2.862910747528076, + "max_kl": 9.398555755615234, + "mean_kl": 2.9537277221679688, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.24418604373931885 + }, + { + "candidate_nll": 2.6377193927764893, + "code_tokens": 31, + "cvar95_kl": 6.260285377502441, + "delta_nll": 2.3036699295043945, + "max_kl": 6.8739728927612305, + "mean_kl": 2.2893166542053223, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.4193548262119293 + }, + { + "candidate_nll": 3.5197978019714355, + "code_tokens": 34, + "cvar95_kl": 7.306445121765137, + "delta_nll": 3.008197784423828, + "max_kl": 7.3273396492004395, + "mean_kl": 3.031116247177124, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.357820510864258, + "code_tokens": 38, + "cvar95_kl": 7.824626922607422, + "delta_nll": 2.970360279083252, + "max_kl": 7.899691581726074, + "mean_kl": 3.0458431243896484, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 4.015258312225342, + "code_tokens": 93, + "cvar95_kl": 7.816094398498535, + "delta_nll": 3.052213668823242, + "max_kl": 9.309334754943848, + "mean_kl": 3.122697353363037, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 4.192813396453857, + "code_tokens": 70, + "cvar95_kl": 8.411836624145508, + "delta_nll": 3.414207696914673, + "max_kl": 9.265669822692871, + "mean_kl": 3.392256736755371, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 5.501105785369873, + "code_tokens": 67, + "cvar95_kl": 10.183948516845703, + "delta_nll": 4.690613746643066, + "max_kl": 10.946342468261719, + "mean_kl": 4.644940376281738, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.1492537260055542 + }, + { + "candidate_nll": 4.294070243835449, + "code_tokens": 66, + "cvar95_kl": 7.619813442230225, + "delta_nll": 3.0079429149627686, + "max_kl": 9.342126846313477, + "mean_kl": 3.0630438327789307, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 3.692593574523926, + "code_tokens": 34, + "cvar95_kl": 6.3879714012146, + "delta_nll": 2.6802992820739746, + "max_kl": 6.683073043823242, + "mean_kl": 2.7668020725250244, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 3.30316162109375, + "code_tokens": 64, + "cvar95_kl": 6.8046488761901855, + "delta_nll": 2.3931362628936768, + "max_kl": 7.245183944702148, + "mean_kl": 2.456265687942505, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.421875 + }, + { + "candidate_nll": 3.668781280517578, + "code_tokens": 56, + "cvar95_kl": 6.81134557723999, + "delta_nll": 2.3584256172180176, + "max_kl": 7.761061668395996, + "mean_kl": 2.4911937713623047, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.2320334911346436, + "code_tokens": 21, + "cvar95_kl": 6.000507831573486, + "delta_nll": 2.1069512367248535, + "max_kl": 6.424929618835449, + "mean_kl": 2.2791354656219482, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.7665750980377197, + "code_tokens": 60, + "cvar95_kl": 6.937555313110352, + "delta_nll": 3.1012346744537354, + "max_kl": 7.931604385375977, + "mean_kl": 3.164490222930908, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.215765953063965, + "code_tokens": 29, + "cvar95_kl": 7.314833164215088, + "delta_nll": 2.861989974975586, + "max_kl": 8.027786254882812, + "mean_kl": 2.8257863521575928, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 5.487743377685547, + "code_tokens": 34, + "cvar95_kl": 10.39635944366455, + "delta_nll": 4.765698432922363, + "max_kl": 10.53091812133789, + "mean_kl": 4.831206321716309, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.14705882966518402 + }, + { + "candidate_nll": 5.353956699371338, + "code_tokens": 88, + "cvar95_kl": 9.7469482421875, + "delta_nll": 4.434849262237549, + "max_kl": 12.063387870788574, + "mean_kl": 4.504237651824951, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 2.8467071056365967, + "code_tokens": 17, + "cvar95_kl": 5.615901470184326, + "delta_nll": 2.303483486175537, + "max_kl": 5.615901470184326, + "mean_kl": 2.199235677719116, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.269216537475586, + "code_tokens": 35, + "cvar95_kl": 6.36236572265625, + "delta_nll": 2.378669261932373, + "max_kl": 6.422388076782227, + "mean_kl": 2.6165976524353027, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 3.171173334121704, + "code_tokens": 48, + "cvar95_kl": 6.855409622192383, + "delta_nll": 2.2813968658447266, + "max_kl": 8.592443466186523, + "mean_kl": 2.3911726474761963, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.892594814300537, + "code_tokens": 130, + "cvar95_kl": 7.85114049911499, + "delta_nll": 2.902322292327881, + "max_kl": 10.100557327270508, + "mean_kl": 2.838160753250122, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.35384616255760193 + }, + { + "candidate_nll": 5.2674407958984375, + "code_tokens": 65, + "cvar95_kl": 8.829581260681152, + "delta_nll": 4.172501564025879, + "max_kl": 9.86618423461914, + "mean_kl": 4.216352462768555, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 4.649488925933838, + "code_tokens": 145, + "cvar95_kl": 10.019237518310547, + "delta_nll": 3.7744739055633545, + "max_kl": 12.40185832977295, + "mean_kl": 3.8058786392211914, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.2344827651977539 + }, + { + "candidate_nll": 2.843113422393799, + "code_tokens": 80, + "cvar95_kl": 8.03795337677002, + "delta_nll": 2.1274142265319824, + "max_kl": 9.929292678833008, + "mean_kl": 2.1213603019714355, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.48750001192092896 + }, + { + "candidate_nll": 3.6925485134124756, + "code_tokens": 70, + "cvar95_kl": 7.485743522644043, + "delta_nll": 3.1104116439819336, + "max_kl": 9.168230056762695, + "mean_kl": 3.297051191329956, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.34285715222358704 + }, + { + "candidate_nll": 4.080589294433594, + "code_tokens": 25, + "cvar95_kl": 7.498349189758301, + "delta_nll": 2.5923004150390625, + "max_kl": 7.706131458282471, + "mean_kl": 2.7235803604125977, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.448725700378418, + "code_tokens": 50, + "cvar95_kl": 5.668278217315674, + "delta_nll": 1.794342279434204, + "max_kl": 6.981934547424316, + "mean_kl": 2.255553960800171, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 3.650709390640259, + "code_tokens": 48, + "cvar95_kl": 6.992645263671875, + "delta_nll": 2.8526663780212402, + "max_kl": 7.673394203186035, + "mean_kl": 3.038642644882202, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.4218978881835938, + "code_tokens": 104, + "cvar95_kl": 9.552672386169434, + "delta_nll": 3.0778255462646484, + "max_kl": 12.807207107543945, + "mean_kl": 2.8962771892547607, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.4134615361690521 + }, + { + "candidate_nll": 3.882356882095337, + "code_tokens": 68, + "cvar95_kl": 7.189871788024902, + "delta_nll": 2.892362594604492, + "max_kl": 8.725593566894531, + "mean_kl": 2.949209690093994, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.22058823704719543 + }, + { + "candidate_nll": 3.552440643310547, + "code_tokens": 145, + "cvar95_kl": 6.776376724243164, + "delta_nll": 2.7975285053253174, + "max_kl": 7.916009902954102, + "mean_kl": 2.7792539596557617, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.3655172288417816 + }, + { + "candidate_nll": 3.1774444580078125, + "code_tokens": 41, + "cvar95_kl": 6.56836462020874, + "delta_nll": 2.33255672454834, + "max_kl": 6.839869022369385, + "mean_kl": 2.525017261505127, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.3658536672592163 + }, + { + "candidate_nll": 2.7343080043792725, + "code_tokens": 53, + "cvar95_kl": 6.715541362762451, + "delta_nll": 1.9731857776641846, + "max_kl": 7.327755928039551, + "mean_kl": 1.8158304691314697, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.5660377144813538 + }, + { + "candidate_nll": 4.000922203063965, + "code_tokens": 58, + "cvar95_kl": 8.806139945983887, + "delta_nll": 3.2086925506591797, + "max_kl": 12.909284591674805, + "mean_kl": 3.0824503898620605, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.36206895112991333 + }, + { + "candidate_nll": 4.295392990112305, + "code_tokens": 24, + "cvar95_kl": 10.201957702636719, + "delta_nll": 2.563333034515381, + "max_kl": 11.057034492492676, + "mean_kl": 3.0209271907806396, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.1740059852600098, + "code_tokens": 89, + "cvar95_kl": 7.694464683532715, + "delta_nll": 2.182152032852173, + "max_kl": 11.125519752502441, + "mean_kl": 2.412564992904663, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.449438214302063 + }, + { + "candidate_nll": 4.151259899139404, + "code_tokens": 101, + "cvar95_kl": 9.198267936706543, + "delta_nll": 3.6661837100982666, + "max_kl": 9.89823055267334, + "mean_kl": 3.644404411315918, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.2772277295589447 + }, + { + "candidate_nll": 5.237314224243164, + "code_tokens": 33, + "cvar95_kl": 10.572456359863281, + "delta_nll": 3.779756546020508, + "max_kl": 13.175298690795898, + "mean_kl": 3.830256462097168, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 3.844468593597412, + "code_tokens": 50, + "cvar95_kl": 9.045470237731934, + "delta_nll": 2.7978029251098633, + "max_kl": 10.026638984680176, + "mean_kl": 3.0674729347229004, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.41999998688697815 + }, + { + "candidate_nll": 4.652431488037109, + "code_tokens": 58, + "cvar95_kl": 9.423846244812012, + "delta_nll": 3.243678569793701, + "max_kl": 12.123363494873047, + "mean_kl": 3.4499175548553467, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 4.146509647369385, + "code_tokens": 62, + "cvar95_kl": 7.958052635192871, + "delta_nll": 3.1673507690429688, + "max_kl": 9.004565238952637, + "mean_kl": 3.2936770915985107, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 3.725612163543701, + "code_tokens": 42, + "cvar95_kl": 7.56003999710083, + "delta_nll": 3.0339221954345703, + "max_kl": 9.144930839538574, + "mean_kl": 3.0353474617004395, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.7730202674865723, + "code_tokens": 15, + "cvar95_kl": 7.126126766204834, + "delta_nll": 2.8482792377471924, + "max_kl": 7.126126766204834, + "mean_kl": 2.5166068077087402, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 4.33591890335083, + "code_tokens": 57, + "cvar95_kl": 7.460629940032959, + "delta_nll": 3.5971367359161377, + "max_kl": 7.739984035491943, + "mean_kl": 3.688610315322876, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.17543859779834747 + }, + { + "candidate_nll": 3.9573395252227783, + "code_tokens": 58, + "cvar95_kl": 6.414039611816406, + "delta_nll": 2.9138035774230957, + "max_kl": 6.740392208099365, + "mean_kl": 2.7571640014648438, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 4.072404384613037, + "code_tokens": 45, + "cvar95_kl": 7.058173656463623, + "delta_nll": 3.2004332542419434, + "max_kl": 8.029068946838379, + "mean_kl": 3.2035162448883057, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.2888889014720917 + }, + { + "candidate_nll": 4.614999771118164, + "code_tokens": 32, + "cvar95_kl": 8.222527503967285, + "delta_nll": 2.5857694149017334, + "max_kl": 8.520743370056152, + "mean_kl": 3.141068935394287, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 4.627252578735352, + "code_tokens": 39, + "cvar95_kl": 10.656576156616211, + "delta_nll": 3.306586265563965, + "max_kl": 12.31099796295166, + "mean_kl": 3.7498207092285156, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 3.0318408012390137, + "code_tokens": 136, + "cvar95_kl": 7.563023567199707, + "delta_nll": 2.621079444885254, + "max_kl": 9.105915069580078, + "mean_kl": 2.5602757930755615, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.597012996673584, + "code_tokens": 50, + "cvar95_kl": 9.727985382080078, + "delta_nll": 3.3762564659118652, + "max_kl": 10.2135009765625, + "mean_kl": 3.3762476444244385, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 3.3802402019500732, + "code_tokens": 34, + "cvar95_kl": 6.968450546264648, + "delta_nll": 2.514324426651001, + "max_kl": 7.170389652252197, + "mean_kl": 2.580500841140747, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 4.415182113647461, + "code_tokens": 44, + "cvar95_kl": 10.259132385253906, + "delta_nll": 3.236392021179199, + "max_kl": 13.127507209777832, + "mean_kl": 3.5458505153656006, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.2954545319080353 + }, + { + "candidate_nll": 3.01303768157959, + "code_tokens": 36, + "cvar95_kl": 5.864461898803711, + "delta_nll": 1.579615831375122, + "max_kl": 7.080041408538818, + "mean_kl": 1.877451777458191, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 4.452481269836426, + "code_tokens": 46, + "cvar95_kl": 8.636805534362793, + "delta_nll": 3.6493239402770996, + "max_kl": 9.519923210144043, + "mean_kl": 3.701899290084839, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 4.297454357147217, + "code_tokens": 48, + "cvar95_kl": 9.697834968566895, + "delta_nll": 3.117114543914795, + "max_kl": 10.317031860351562, + "mean_kl": 3.430734634399414, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.799924373626709, + "code_tokens": 25, + "cvar95_kl": 8.425783157348633, + "delta_nll": 2.904513359069824, + "max_kl": 9.99622917175293, + "mean_kl": 2.7032952308654785, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 4.638398170471191, + "code_tokens": 69, + "cvar95_kl": 10.05996322631836, + "delta_nll": 3.1341190338134766, + "max_kl": 11.09068775177002, + "mean_kl": 3.584963798522949, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 3.877368688583374, + "code_tokens": 60, + "cvar95_kl": 7.221673965454102, + "delta_nll": 2.9119110107421875, + "max_kl": 7.447531700134277, + "mean_kl": 3.1962149143218994, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 3.603076219558716, + "code_tokens": 49, + "cvar95_kl": 8.393372535705566, + "delta_nll": 2.6885986328125, + "max_kl": 12.548627853393555, + "mean_kl": 2.667205572128296, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.3469387888908386 + }, + { + "candidate_nll": 4.30967378616333, + "code_tokens": 44, + "cvar95_kl": 9.36144733428955, + "delta_nll": 2.6091456413269043, + "max_kl": 10.760303497314453, + "mean_kl": 2.496518135070801, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.9782590866088867, + "code_tokens": 81, + "cvar95_kl": 7.374444007873535, + "delta_nll": 2.8328447341918945, + "max_kl": 9.42188835144043, + "mean_kl": 2.902648448944092, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.32098764181137085 + }, + { + "candidate_nll": 4.2463202476501465, + "code_tokens": 26, + "cvar95_kl": 8.297276496887207, + "delta_nll": 2.8095850944519043, + "max_kl": 9.834696769714355, + "mean_kl": 2.8408288955688477, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.184657096862793, + "code_tokens": 32, + "cvar95_kl": 10.812742233276367, + "delta_nll": 3.3047361373901367, + "max_kl": 11.020708084106445, + "mean_kl": 3.685166597366333, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 4.62319803237915, + "code_tokens": 48, + "cvar95_kl": 8.55642318725586, + "delta_nll": 3.6754560470581055, + "max_kl": 9.253241539001465, + "mean_kl": 3.616130828857422, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 3.634413957595825, + "code_tokens": 180, + "cvar95_kl": 7.568722248077393, + "delta_nll": 2.9350929260253906, + "max_kl": 9.20962905883789, + "mean_kl": 2.918628692626953, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.31111112236976624 + }, + { + "candidate_nll": 4.078640460968018, + "code_tokens": 38, + "cvar95_kl": 8.620773315429688, + "delta_nll": 3.3568787574768066, + "max_kl": 8.822489738464355, + "mean_kl": 3.529296875, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 3.385084390640259, + "code_tokens": 90, + "cvar95_kl": 8.368600845336914, + "delta_nll": 2.6900227069854736, + "max_kl": 9.487518310546875, + "mean_kl": 2.726747751235962, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.3444444537162781 + }, + { + "candidate_nll": 4.352132797241211, + "code_tokens": 44, + "cvar95_kl": 8.470175743103027, + "delta_nll": 3.7775027751922607, + "max_kl": 8.807779312133789, + "mean_kl": 3.6915907859802246, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 2.9548068046569824, + "code_tokens": 60, + "cvar95_kl": 7.250366687774658, + "delta_nll": 2.210906505584717, + "max_kl": 7.435710906982422, + "mean_kl": 2.3435142040252686, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.388805627822876, + "code_tokens": 69, + "cvar95_kl": 6.37248420715332, + "delta_nll": 2.38600754737854, + "max_kl": 6.92216157913208, + "mean_kl": 2.2846014499664307, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.36231884360313416 + }, + { + "candidate_nll": 3.942918062210083, + "code_tokens": 71, + "cvar95_kl": 7.728221893310547, + "delta_nll": 3.425356388092041, + "max_kl": 8.67613697052002, + "mean_kl": 3.541234016418457, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.2535211145877838 + }, + { + "candidate_nll": 3.1326682567596436, + "code_tokens": 22, + "cvar95_kl": 5.988738059997559, + "delta_nll": 1.8899950981140137, + "max_kl": 6.211241722106934, + "mean_kl": 1.722172498703003, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 2.2095797061920166, + "code_tokens": 215, + "cvar95_kl": 5.899947643280029, + "delta_nll": 1.655800223350525, + "max_kl": 8.379737854003906, + "mean_kl": 1.748653769493103, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.5348837375640869 + }, + { + "candidate_nll": 4.782500743865967, + "code_tokens": 26, + "cvar95_kl": 9.233384132385254, + "delta_nll": 3.679711103439331, + "max_kl": 10.238029479980469, + "mean_kl": 3.4501099586486816, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 3.836142063140869, + "code_tokens": 33, + "cvar95_kl": 7.62031364440918, + "delta_nll": 2.9678955078125, + "max_kl": 9.01274585723877, + "mean_kl": 3.0100579261779785, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 2.5232362747192383, + "code_tokens": 18, + "cvar95_kl": 6.468929290771484, + "delta_nll": 1.5870881080627441, + "max_kl": 6.468929290771484, + "mean_kl": 1.8127943277359009, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.740409851074219, + "code_tokens": 71, + "cvar95_kl": 9.651758193969727, + "delta_nll": 3.460348129272461, + "max_kl": 11.862231254577637, + "mean_kl": 3.640049934387207, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.3380281627178192 + }, + { + "candidate_nll": 3.713538885116577, + "code_tokens": 107, + "cvar95_kl": 6.3289361000061035, + "delta_nll": 2.9321374893188477, + "max_kl": 6.799528121948242, + "mean_kl": 2.8074283599853516, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.38317757844924927 + }, + { + "candidate_nll": 4.882130146026611, + "code_tokens": 40, + "cvar95_kl": 9.50335693359375, + "delta_nll": 2.8271055221557617, + "max_kl": 10.18832015991211, + "mean_kl": 3.4957118034362793, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.9609386920928955, + "code_tokens": 32, + "cvar95_kl": 6.802403450012207, + "delta_nll": 2.596641778945923, + "max_kl": 8.018084526062012, + "mean_kl": 2.718657970428467, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 3.610692024230957, + "code_tokens": 130, + "cvar95_kl": 7.132121562957764, + "delta_nll": 2.911438226699829, + "max_kl": 9.209728240966797, + "mean_kl": 3.020761489868164, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.0167713165283203, + "code_tokens": 27, + "cvar95_kl": 5.241549015045166, + "delta_nll": 1.5453495979309082, + "max_kl": 5.3651204109191895, + "mean_kl": 1.834603190422058, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.227175235748291, + "code_tokens": 42, + "cvar95_kl": 9.007840156555176, + "delta_nll": 3.254234790802002, + "max_kl": 9.531181335449219, + "mean_kl": 3.1842803955078125, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.464443683624268, + "code_tokens": 30, + "cvar95_kl": 9.401666641235352, + "delta_nll": 3.71893048286438, + "max_kl": 11.09783935546875, + "mean_kl": 3.769158363342285, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.23333333432674408 + }, + { + "candidate_nll": 5.4136810302734375, + "code_tokens": 58, + "cvar95_kl": 9.937823295593262, + "delta_nll": 3.994448184967041, + "max_kl": 10.591158866882324, + "mean_kl": 3.890303611755371, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 4.289900779724121, + "code_tokens": 34, + "cvar95_kl": 8.02885627746582, + "delta_nll": 3.059673547744751, + "max_kl": 8.707138061523438, + "mean_kl": 2.960114002227783, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 2.516338348388672, + "code_tokens": 208, + "cvar95_kl": 6.204409599304199, + "delta_nll": 1.814875841140747, + "max_kl": 10.28381633758545, + "mean_kl": 1.8503748178482056, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.4759615361690521 + }, + { + "candidate_nll": 3.5952351093292236, + "code_tokens": 83, + "cvar95_kl": 6.517756462097168, + "delta_nll": 2.696329355239868, + "max_kl": 7.87760066986084, + "mean_kl": 2.745426893234253, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.3253012001514435 + }, + { + "candidate_nll": 3.6326754093170166, + "code_tokens": 170, + "cvar95_kl": 8.956191062927246, + "delta_nll": 3.1101198196411133, + "max_kl": 13.544168472290039, + "mean_kl": 3.0836331844329834, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.34117648005485535 + }, + { + "candidate_nll": 3.5393385887145996, + "code_tokens": 40, + "cvar95_kl": 8.574459075927734, + "delta_nll": 2.757918119430542, + "max_kl": 9.179316520690918, + "mean_kl": 2.9972853660583496, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.42500001192092896 + }, + { + "candidate_nll": 3.5509214401245117, + "code_tokens": 31, + "cvar95_kl": 7.284379482269287, + "delta_nll": 2.3042118549346924, + "max_kl": 7.4061174392700195, + "mean_kl": 2.394887685775757, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.4838709533214569 + }, + { + "candidate_nll": 4.5996246337890625, + "code_tokens": 42, + "cvar95_kl": 7.879964351654053, + "delta_nll": 3.80485463142395, + "max_kl": 9.98081111907959, + "mean_kl": 4.046616554260254, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.035093307495117, + "code_tokens": 27, + "cvar95_kl": 8.131765365600586, + "delta_nll": 2.2660274505615234, + "max_kl": 9.118736267089844, + "mean_kl": 3.0731663703918457, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 4.098763465881348, + "code_tokens": 25, + "cvar95_kl": 6.715123176574707, + "delta_nll": 1.8989529609680176, + "max_kl": 7.715762615203857, + "mean_kl": 2.242392063140869, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.2766313552856445, + "code_tokens": 40, + "cvar95_kl": 8.683692932128906, + "delta_nll": 3.708244800567627, + "max_kl": 8.910272598266602, + "mean_kl": 3.7970833778381348, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 3.7545711994171143, + "code_tokens": 24, + "cvar95_kl": 6.333379745483398, + "delta_nll": 2.3702166080474854, + "max_kl": 6.645071506500244, + "mean_kl": 2.2688064575195312, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.3002569675445557, + "code_tokens": 28, + "cvar95_kl": 5.767895221710205, + "delta_nll": 2.4175119400024414, + "max_kl": 6.335764408111572, + "mean_kl": 2.268183469772339, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.9560177326202393, + "code_tokens": 71, + "cvar95_kl": 8.539433479309082, + "delta_nll": 3.4771533012390137, + "max_kl": 9.45234203338623, + "mean_kl": 3.31797456741333, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.3239436745643616 + }, + { + "candidate_nll": 3.311817169189453, + "code_tokens": 24, + "cvar95_kl": 7.627372741699219, + "delta_nll": 2.782456398010254, + "max_kl": 7.726146697998047, + "mean_kl": 2.6556804180145264, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.224122524261475, + "code_tokens": 34, + "cvar95_kl": 9.577171325683594, + "delta_nll": 2.9525413513183594, + "max_kl": 9.764183044433594, + "mean_kl": 3.3949079513549805, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 4.539193153381348, + "code_tokens": 64, + "cvar95_kl": 8.857996940612793, + "delta_nll": 3.4198992252349854, + "max_kl": 9.875605583190918, + "mean_kl": 3.8293418884277344, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.296875 + }, + { + "candidate_nll": 2.9730865955352783, + "code_tokens": 45, + "cvar95_kl": 5.997976303100586, + "delta_nll": 2.2664952278137207, + "max_kl": 6.364012718200684, + "mean_kl": 2.07502818107605, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.5333333611488342 + }, + { + "candidate_nll": 4.283463954925537, + "code_tokens": 48, + "cvar95_kl": 9.267723083496094, + "delta_nll": 3.0847597122192383, + "max_kl": 9.357243537902832, + "mean_kl": 3.420189619064331, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.255723476409912, + "code_tokens": 21, + "cvar95_kl": 6.427079677581787, + "delta_nll": 2.5555224418640137, + "max_kl": 7.379303455352783, + "mean_kl": 2.608555316925049, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.070645809173584, + "code_tokens": 24, + "cvar95_kl": 10.380264282226562, + "delta_nll": 3.3162317276000977, + "max_kl": 11.08437442779541, + "mean_kl": 3.5986318588256836, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.350407600402832, + "code_tokens": 63, + "cvar95_kl": 7.074592113494873, + "delta_nll": 2.778480291366577, + "max_kl": 7.857761383056641, + "mean_kl": 2.9259045124053955, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.302929878234863, + "code_tokens": 22, + "cvar95_kl": 8.162576675415039, + "delta_nll": 4.024170875549316, + "max_kl": 8.676629066467285, + "mean_kl": 3.760995388031006, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.11238169670105, + "code_tokens": 25, + "cvar95_kl": 5.20335578918457, + "delta_nll": 1.9837193489074707, + "max_kl": 5.424187183380127, + "mean_kl": 2.195052146911621, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.181497097015381, + "code_tokens": 21, + "cvar95_kl": 5.720425605773926, + "delta_nll": 1.2211713790893555, + "max_kl": 5.771333694458008, + "mean_kl": 1.588106632232666, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.523809552192688 + }, + { + "candidate_nll": 2.7409961223602295, + "code_tokens": 24, + "cvar95_kl": 4.533476829528809, + "delta_nll": 1.274664044380188, + "max_kl": 5.177115440368652, + "mean_kl": 1.597269058227539, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 5.433968544006348, + "code_tokens": 15, + "cvar95_kl": 9.933222770690918, + "delta_nll": 3.6404807567596436, + "max_kl": 9.933222770690918, + "mean_kl": 3.8583593368530273, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.052343845367432, + "code_tokens": 59, + "cvar95_kl": 8.190730094909668, + "delta_nll": 3.2714874744415283, + "max_kl": 8.83748722076416, + "mean_kl": 3.2046327590942383, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.23728813230991364 + }, + { + "candidate_nll": 3.909467935562134, + "code_tokens": 54, + "cvar95_kl": 7.459274768829346, + "delta_nll": 2.9753496646881104, + "max_kl": 7.932649612426758, + "mean_kl": 2.953207015991211, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 4.881419658660889, + "code_tokens": 28, + "cvar95_kl": 7.737185955047607, + "delta_nll": 2.929560661315918, + "max_kl": 8.405150413513184, + "mean_kl": 3.300550699234009, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.4409170150756836, + "code_tokens": 46, + "cvar95_kl": 6.500015735626221, + "delta_nll": 2.3561911582946777, + "max_kl": 7.499691963195801, + "mean_kl": 2.5887656211853027, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 2.542072057723999, + "code_tokens": 27, + "cvar95_kl": 4.637248992919922, + "delta_nll": 1.0272496938705444, + "max_kl": 4.649721622467041, + "mean_kl": 1.6024127006530762, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.48148149251937866 + }, + { + "candidate_nll": 4.492657661437988, + "code_tokens": 97, + "cvar95_kl": 6.945944309234619, + "delta_nll": 3.235227108001709, + "max_kl": 7.702723503112793, + "mean_kl": 3.2835686206817627, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.24742268025875092 + }, + { + "candidate_nll": 4.186028480529785, + "code_tokens": 24, + "cvar95_kl": 7.056452751159668, + "delta_nll": 3.383795976638794, + "max_kl": 7.86702299118042, + "mean_kl": 3.1381378173828125, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 4.953714847564697, + "code_tokens": 30, + "cvar95_kl": 9.356110572814941, + "delta_nll": 3.688566207885742, + "max_kl": 10.736920356750488, + "mean_kl": 4.064682960510254, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.399885654449463, + "code_tokens": 38, + "cvar95_kl": 11.219636917114258, + "delta_nll": 2.966815948486328, + "max_kl": 12.033462524414062, + "mean_kl": 3.3428313732147217, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 2.527658462524414, + "code_tokens": 21, + "cvar95_kl": 4.034332275390625, + "delta_nll": 1.0398794412612915, + "max_kl": 4.2234883308410645, + "mean_kl": 1.312421202659607, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 4.8651227951049805, + "code_tokens": 65, + "cvar95_kl": 8.792977333068848, + "delta_nll": 4.328161239624023, + "max_kl": 9.87132453918457, + "mean_kl": 4.37290096282959, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.2153846174478531 + }, + { + "candidate_nll": 2.943725347518921, + "code_tokens": 27, + "cvar95_kl": 5.748538017272949, + "delta_nll": 2.1591989994049072, + "max_kl": 5.761634349822998, + "mean_kl": 2.209474563598633, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.48148149251937866 + }, + { + "candidate_nll": 3.7810826301574707, + "code_tokens": 35, + "cvar95_kl": 9.638334274291992, + "delta_nll": 3.0572664737701416, + "max_kl": 12.998612403869629, + "mean_kl": 2.9582808017730713, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 3.7833688259124756, + "code_tokens": 75, + "cvar95_kl": 7.276426315307617, + "delta_nll": 2.858844757080078, + "max_kl": 8.808890342712402, + "mean_kl": 2.9216365814208984, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 4.2360358238220215, + "code_tokens": 25, + "cvar95_kl": 6.793317794799805, + "delta_nll": 2.5943002700805664, + "max_kl": 6.859598636627197, + "mean_kl": 3.200976610183716, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.682727336883545, + "code_tokens": 52, + "cvar95_kl": 7.373510837554932, + "delta_nll": 2.7587385177612305, + "max_kl": 8.96750259399414, + "mean_kl": 2.744626522064209, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 3.1990785598754883, + "code_tokens": 110, + "cvar95_kl": 8.934125900268555, + "delta_nll": 2.588418483734131, + "max_kl": 10.271785736083984, + "mean_kl": 2.674553871154785, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.4363636374473572 + }, + { + "candidate_nll": 3.0804381370544434, + "code_tokens": 25, + "cvar95_kl": 5.624538421630859, + "delta_nll": 1.847616195678711, + "max_kl": 5.890575408935547, + "mean_kl": 1.9582011699676514, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 4.246603488922119, + "code_tokens": 56, + "cvar95_kl": 5.624715328216553, + "delta_nll": 3.2296175956726074, + "max_kl": 5.797147750854492, + "mean_kl": 3.055328607559204, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.1964285671710968 + }, + { + "candidate_nll": 4.8739776611328125, + "code_tokens": 108, + "cvar95_kl": 6.895989894866943, + "delta_nll": 3.32012939453125, + "max_kl": 7.466159343719482, + "mean_kl": 3.356740713119507, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.21296297013759613 + }, + { + "candidate_nll": 3.9160544872283936, + "code_tokens": 47, + "cvar95_kl": 8.400788307189941, + "delta_nll": 2.5443146228790283, + "max_kl": 9.554823875427246, + "mean_kl": 3.0489108562469482, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 4.170992851257324, + "code_tokens": 102, + "cvar95_kl": 8.52673625946045, + "delta_nll": 3.172584295272827, + "max_kl": 9.949728965759277, + "mean_kl": 3.3026273250579834, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 3.066173791885376, + "code_tokens": 18, + "cvar95_kl": 4.921903133392334, + "delta_nll": 1.4405481815338135, + "max_kl": 4.921903133392334, + "mean_kl": 1.6966681480407715, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 4.909525394439697, + "code_tokens": 28, + "cvar95_kl": 8.819828987121582, + "delta_nll": 3.809861183166504, + "max_kl": 10.029065132141113, + "mean_kl": 3.6821258068084717, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 4.373770236968994, + "code_tokens": 30, + "cvar95_kl": 6.806249618530273, + "delta_nll": 2.830369472503662, + "max_kl": 7.30444860458374, + "mean_kl": 2.9381496906280518, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.4357614517211914, + "code_tokens": 48, + "cvar95_kl": 8.305248260498047, + "delta_nll": 2.960644483566284, + "max_kl": 11.885842323303223, + "mean_kl": 2.9893558025360107, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.3839423656463623, + "code_tokens": 59, + "cvar95_kl": 6.833145618438721, + "delta_nll": 2.8469274044036865, + "max_kl": 7.505489826202393, + "mean_kl": 2.9032676219940186, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.35593220591545105 + }, + { + "candidate_nll": 4.080001354217529, + "code_tokens": 102, + "cvar95_kl": 10.09871768951416, + "delta_nll": 3.0378074645996094, + "max_kl": 12.677062034606934, + "mean_kl": 3.1004180908203125, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.37254902720451355 + }, + { + "candidate_nll": 3.505739688873291, + "code_tokens": 36, + "cvar95_kl": 6.494429588317871, + "delta_nll": 2.765374183654785, + "max_kl": 7.00405740737915, + "mean_kl": 2.796797037124634, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.3055555522441864 + }, + { + "candidate_nll": 5.214040279388428, + "code_tokens": 51, + "cvar95_kl": 11.483235359191895, + "delta_nll": 4.325108528137207, + "max_kl": 12.980380058288574, + "mean_kl": 4.102801322937012, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 3.57212233543396, + "code_tokens": 62, + "cvar95_kl": 6.420836925506592, + "delta_nll": 2.326443672180176, + "max_kl": 6.71972131729126, + "mean_kl": 2.745229959487915, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 4.46502161026001, + "code_tokens": 39, + "cvar95_kl": 10.751242637634277, + "delta_nll": 3.888739585876465, + "max_kl": 11.354065895080566, + "mean_kl": 3.807793617248535, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 3.463132858276367, + "code_tokens": 76, + "cvar95_kl": 7.023488998413086, + "delta_nll": 2.7234485149383545, + "max_kl": 7.7228803634643555, + "mean_kl": 2.7957801818847656, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.3947368562221527 + }, + { + "candidate_nll": 5.208499431610107, + "code_tokens": 58, + "cvar95_kl": 9.57836627960205, + "delta_nll": 3.703848361968994, + "max_kl": 10.014718055725098, + "mean_kl": 3.5587918758392334, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.18965516984462738 + }, + { + "candidate_nll": 4.007776260375977, + "code_tokens": 93, + "cvar95_kl": 9.1859769821167, + "delta_nll": 2.8803811073303223, + "max_kl": 13.655652046203613, + "mean_kl": 2.808589458465576, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.34408602118492126 + }, + { + "candidate_nll": 5.452725410461426, + "code_tokens": 45, + "cvar95_kl": 10.404799461364746, + "delta_nll": 4.097479820251465, + "max_kl": 11.611339569091797, + "mean_kl": 4.318462371826172, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 2.0689895153045654, + "code_tokens": 209, + "cvar95_kl": 5.499825477600098, + "delta_nll": 1.5120141506195068, + "max_kl": 6.501262664794922, + "mean_kl": 1.6445503234863281, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.5023923516273499 + }, + { + "candidate_nll": 3.853585958480835, + "code_tokens": 46, + "cvar95_kl": 7.373960018157959, + "delta_nll": 2.885118246078491, + "max_kl": 8.606781959533691, + "mean_kl": 2.565938949584961, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.54347825050354 + }, + { + "candidate_nll": 3.963355541229248, + "code_tokens": 39, + "cvar95_kl": 9.303937911987305, + "delta_nll": 3.2425684928894043, + "max_kl": 9.8635835647583, + "mean_kl": 3.1588406562805176, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 2.860501289367676, + "code_tokens": 25, + "cvar95_kl": 5.208372592926025, + "delta_nll": 1.9188058376312256, + "max_kl": 5.983712673187256, + "mean_kl": 1.972426176071167, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.086628437042236, + "code_tokens": 28, + "cvar95_kl": 12.776472091674805, + "delta_nll": 2.9114322662353516, + "max_kl": 15.157503128051758, + "mean_kl": 3.0446434020996094, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.412019729614258, + "code_tokens": 84, + "cvar95_kl": 7.3540496826171875, + "delta_nll": 3.158097267150879, + "max_kl": 7.961038589477539, + "mean_kl": 3.2920005321502686, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 3.5529890060424805, + "code_tokens": 34, + "cvar95_kl": 6.270872116088867, + "delta_nll": 2.740708827972412, + "max_kl": 6.67777681350708, + "mean_kl": 2.8429763317108154, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 3.3411056995391846, + "code_tokens": 28, + "cvar95_kl": 5.648660182952881, + "delta_nll": 2.0860586166381836, + "max_kl": 6.321291923522949, + "mean_kl": 2.3680806159973145, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.39884090423584, + "code_tokens": 92, + "cvar95_kl": 7.345637321472168, + "delta_nll": 2.2999954223632812, + "max_kl": 7.589704513549805, + "mean_kl": 2.5096960067749023, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.3695652186870575 + }, + { + "candidate_nll": 4.704717636108398, + "code_tokens": 185, + "cvar95_kl": 9.9719820022583, + "delta_nll": 3.9746029376983643, + "max_kl": 12.62096881866455, + "mean_kl": 4.071902751922607, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.24864864349365234 + }, + { + "candidate_nll": 3.7504160404205322, + "code_tokens": 106, + "cvar95_kl": 8.849160194396973, + "delta_nll": 3.397421360015869, + "max_kl": 11.1014404296875, + "mean_kl": 3.445896863937378, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 5.196961402893066, + "code_tokens": 64, + "cvar95_kl": 9.059039115905762, + "delta_nll": 3.654407024383545, + "max_kl": 10.817133903503418, + "mean_kl": 3.685386896133423, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 4.124581813812256, + "code_tokens": 74, + "cvar95_kl": 7.602740287780762, + "delta_nll": 3.0137977600097656, + "max_kl": 8.34546947479248, + "mean_kl": 3.0025901794433594, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.31081080436706543 + }, + { + "candidate_nll": 3.182344436645508, + "code_tokens": 27, + "cvar95_kl": 8.598033905029297, + "delta_nll": 2.1949405670166016, + "max_kl": 9.772436141967773, + "mean_kl": 2.463778257369995, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 4.91113805770874, + "code_tokens": 72, + "cvar95_kl": 9.119094848632812, + "delta_nll": 4.287300109863281, + "max_kl": 9.817774772644043, + "mean_kl": 4.383299827575684, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 3.614169120788574, + "code_tokens": 62, + "cvar95_kl": 5.930384635925293, + "delta_nll": 2.6808667182922363, + "max_kl": 6.263224124908447, + "mean_kl": 2.5697579383850098, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.4032258093357086 + }, + { + "candidate_nll": 3.6092565059661865, + "code_tokens": 42, + "cvar95_kl": 7.891903400421143, + "delta_nll": 2.911090135574341, + "max_kl": 10.603010177612305, + "mean_kl": 2.9351308345794678, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.933047294616699, + "code_tokens": 238, + "cvar95_kl": 9.391154289245605, + "delta_nll": 4.045714378356934, + "max_kl": 13.762989044189453, + "mean_kl": 4.029013156890869, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.1974789947271347 + }, + { + "candidate_nll": 4.240156650543213, + "code_tokens": 68, + "cvar95_kl": 8.848766326904297, + "delta_nll": 3.3256449699401855, + "max_kl": 11.729697227478027, + "mean_kl": 3.315869092941284, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 4.225320816040039, + "code_tokens": 83, + "cvar95_kl": 9.872417449951172, + "delta_nll": 3.241170883178711, + "max_kl": 12.391952514648438, + "mean_kl": 3.292370557785034, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.28915661573410034 + }, + { + "candidate_nll": 6.044972896575928, + "code_tokens": 64, + "cvar95_kl": 11.184221267700195, + "delta_nll": 4.781205177307129, + "max_kl": 13.111284255981445, + "mean_kl": 4.816034317016602, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.234375 + }, + { + "candidate_nll": 2.578829288482666, + "code_tokens": 37, + "cvar95_kl": 4.1751275062561035, + "delta_nll": 1.9220293760299683, + "max_kl": 4.312366485595703, + "mean_kl": 1.9250143766403198, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 4.377813339233398, + "code_tokens": 27, + "cvar95_kl": 8.471891403198242, + "delta_nll": 2.8678860664367676, + "max_kl": 8.582232475280762, + "mean_kl": 3.1090712547302246, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.8918464183807373, + "code_tokens": 40, + "cvar95_kl": 7.796198844909668, + "delta_nll": 3.0001301765441895, + "max_kl": 8.159327507019043, + "mean_kl": 3.2590489387512207, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 3.9594080448150635, + "code_tokens": 28, + "cvar95_kl": 8.583600997924805, + "delta_nll": 2.9616353511810303, + "max_kl": 9.931201934814453, + "mean_kl": 3.071216583251953, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.213467597961426, + "code_tokens": 59, + "cvar95_kl": 7.238033294677734, + "delta_nll": 2.3588380813598633, + "max_kl": 8.45518684387207, + "mean_kl": 2.5748836994171143, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.33898305892944336 + }, + { + "candidate_nll": 4.238046169281006, + "code_tokens": 57, + "cvar95_kl": 8.251842498779297, + "delta_nll": 3.668045997619629, + "max_kl": 9.662748336791992, + "mean_kl": 3.4731409549713135, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 4.250063896179199, + "code_tokens": 25, + "cvar95_kl": 6.444606304168701, + "delta_nll": 2.866123676300049, + "max_kl": 6.473931789398193, + "mean_kl": 2.9580185413360596, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 2.741061210632324, + "code_tokens": 48, + "cvar95_kl": 5.7980570793151855, + "delta_nll": 1.9939088821411133, + "max_kl": 6.53996467590332, + "mean_kl": 1.921798586845398, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.9514622688293457, + "code_tokens": 35, + "cvar95_kl": 6.274663925170898, + "delta_nll": 3.089686393737793, + "max_kl": 6.366103649139404, + "mean_kl": 2.905968189239502, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.2571428716182709 + }, + { + "candidate_nll": 3.100595712661743, + "code_tokens": 30, + "cvar95_kl": 6.987055778503418, + "delta_nll": 2.455888509750366, + "max_kl": 8.414538383483887, + "mean_kl": 2.4261810779571533, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.393273115158081, + "code_tokens": 49, + "cvar95_kl": 6.3227081298828125, + "delta_nll": 2.270195484161377, + "max_kl": 8.558971405029297, + "mean_kl": 2.463522434234619, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 4.796656608581543, + "code_tokens": 98, + "cvar95_kl": 9.191142082214355, + "delta_nll": 3.780439853668213, + "max_kl": 10.565492630004883, + "mean_kl": 3.7029075622558594, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.2448979616165161 + }, + { + "candidate_nll": 3.014880418777466, + "code_tokens": 25, + "cvar95_kl": 4.974124908447266, + "delta_nll": 1.634074091911316, + "max_kl": 5.026817321777344, + "mean_kl": 1.880496859550476, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 3.252445697784424, + "code_tokens": 27, + "cvar95_kl": 7.896254539489746, + "delta_nll": 2.474722385406494, + "max_kl": 8.878544807434082, + "mean_kl": 2.534348964691162, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 2.6816904544830322, + "code_tokens": 20, + "cvar95_kl": 6.965108394622803, + "delta_nll": 1.6366057395935059, + "max_kl": 6.965108394622803, + "mean_kl": 2.0809760093688965, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.672128677368164, + "code_tokens": 253, + "cvar95_kl": 9.408705711364746, + "delta_nll": 3.979458808898926, + "max_kl": 12.40748119354248, + "mean_kl": 4.048156261444092, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.20948617160320282 + }, + { + "candidate_nll": 4.204179763793945, + "code_tokens": 55, + "cvar95_kl": 8.692419052124023, + "delta_nll": 3.3750596046447754, + "max_kl": 10.154470443725586, + "mean_kl": 3.4830334186553955, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.3272727131843567 + }, + { + "candidate_nll": 3.697755813598633, + "code_tokens": 31, + "cvar95_kl": 5.830173492431641, + "delta_nll": 2.491257429122925, + "max_kl": 6.098862648010254, + "mean_kl": 2.6963882446289062, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 4.3731465339660645, + "code_tokens": 35, + "cvar95_kl": 11.2025146484375, + "delta_nll": 3.668900489807129, + "max_kl": 11.977642059326172, + "mean_kl": 3.638741970062256, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.1668896675109863, + "code_tokens": 61, + "cvar95_kl": 7.032824993133545, + "delta_nll": 2.3640213012695312, + "max_kl": 7.884407043457031, + "mean_kl": 2.4603168964385986, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.44262295961380005 + }, + { + "candidate_nll": 3.8008828163146973, + "code_tokens": 55, + "cvar95_kl": 8.241219520568848, + "delta_nll": 2.846165180206299, + "max_kl": 8.574978828430176, + "mean_kl": 2.8266921043395996, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.2545454502105713 + }, + { + "candidate_nll": 4.986948013305664, + "code_tokens": 108, + "cvar95_kl": 9.358540534973145, + "delta_nll": 3.479980945587158, + "max_kl": 15.873682022094727, + "mean_kl": 3.683058500289917, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 3.6927101612091064, + "code_tokens": 58, + "cvar95_kl": 7.097822666168213, + "delta_nll": 2.4054040908813477, + "max_kl": 8.701706886291504, + "mean_kl": 2.521507740020752, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.32758620381355286 + }, + { + "candidate_nll": 4.204352855682373, + "code_tokens": 91, + "cvar95_kl": 9.289053916931152, + "delta_nll": 3.3768975734710693, + "max_kl": 12.127582550048828, + "mean_kl": 3.353581428527832, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.2747252881526947 + }, + { + "candidate_nll": 3.412365198135376, + "code_tokens": 105, + "cvar95_kl": 7.072507858276367, + "delta_nll": 2.6486258506774902, + "max_kl": 8.674285888671875, + "mean_kl": 2.7963457107543945, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.3523809611797333 + }, + { + "candidate_nll": 3.926640033721924, + "code_tokens": 49, + "cvar95_kl": 8.380897521972656, + "delta_nll": 3.2793874740600586, + "max_kl": 9.11481761932373, + "mean_kl": 3.051339626312256, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.30612245202064514 + }, + { + "candidate_nll": 4.024323463439941, + "code_tokens": 27, + "cvar95_kl": 8.259329795837402, + "delta_nll": 2.5621604919433594, + "max_kl": 9.010907173156738, + "mean_kl": 2.826179265975952, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.188447952270508, + "code_tokens": 25, + "cvar95_kl": 6.4607038497924805, + "delta_nll": 2.1480712890625, + "max_kl": 6.915925025939941, + "mean_kl": 2.3986475467681885, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.6058545112609863, + "code_tokens": 98, + "cvar95_kl": 7.6280717849731445, + "delta_nll": 2.310800075531006, + "max_kl": 8.872424125671387, + "mean_kl": 2.590116024017334, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.33673468232154846 + }, + { + "candidate_nll": 3.2901604175567627, + "code_tokens": 23, + "cvar95_kl": 4.906269073486328, + "delta_nll": 2.640378713607788, + "max_kl": 4.908646106719971, + "mean_kl": 2.476661443710327, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 5.09556770324707, + "code_tokens": 31, + "cvar95_kl": 7.932300567626953, + "delta_nll": 3.230194091796875, + "max_kl": 8.289745330810547, + "mean_kl": 3.8530690670013428, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 3.6045477390289307, + "code_tokens": 36, + "cvar95_kl": 9.4942626953125, + "delta_nll": 2.856454610824585, + "max_kl": 11.059560775756836, + "mean_kl": 2.8060805797576904, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.808647871017456, + "code_tokens": 38, + "cvar95_kl": 6.275684356689453, + "delta_nll": 2.3749704360961914, + "max_kl": 6.330950736999512, + "mean_kl": 2.873269557952881, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 3.117582321166992, + "code_tokens": 25, + "cvar95_kl": 4.254770278930664, + "delta_nll": 1.5114924907684326, + "max_kl": 5.54752254486084, + "mean_kl": 1.5070675611495972, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 3.0869970321655273, + "code_tokens": 26, + "cvar95_kl": 5.468042373657227, + "delta_nll": 2.207547903060913, + "max_kl": 5.85869836807251, + "mean_kl": 2.67914080619812, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.309294700622559, + "code_tokens": 62, + "cvar95_kl": 7.74704647064209, + "delta_nll": 3.566852569580078, + "max_kl": 8.574357032775879, + "mean_kl": 3.409372091293335, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.24193547666072845 + }, + { + "candidate_nll": 4.376315593719482, + "code_tokens": 36, + "cvar95_kl": 6.554775238037109, + "delta_nll": 2.637796401977539, + "max_kl": 6.952280521392822, + "mean_kl": 3.1365318298339844, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.3055555522441864 + }, + { + "candidate_nll": 4.321113586425781, + "code_tokens": 119, + "cvar95_kl": 7.783020496368408, + "delta_nll": 2.9905247688293457, + "max_kl": 9.503257751464844, + "mean_kl": 3.1299853324890137, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.27731093764305115 + }, + { + "candidate_nll": 3.8838846683502197, + "code_tokens": 37, + "cvar95_kl": 9.056497573852539, + "delta_nll": 3.5711770057678223, + "max_kl": 10.669641494750977, + "mean_kl": 3.3467140197753906, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 4.167435646057129, + "code_tokens": 28, + "cvar95_kl": 9.043018341064453, + "delta_nll": 3.384702205657959, + "max_kl": 9.682417869567871, + "mean_kl": 3.64459490776062, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.665771961212158, + "code_tokens": 143, + "cvar95_kl": 9.077667236328125, + "delta_nll": 3.0994250774383545, + "max_kl": 12.568334579467773, + "mean_kl": 3.147160291671753, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.35664334893226624 + }, + { + "candidate_nll": 4.006197452545166, + "code_tokens": 29, + "cvar95_kl": 7.589213848114014, + "delta_nll": 2.223891258239746, + "max_kl": 7.701835632324219, + "mean_kl": 2.8458359241485596, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.4137931168079376 + }, + { + "candidate_nll": 4.742638111114502, + "code_tokens": 28, + "cvar95_kl": 9.52490234375, + "delta_nll": 3.670149803161621, + "max_kl": 9.831729888916016, + "mean_kl": 3.527925729751587, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 4.8098978996276855, + "code_tokens": 30, + "cvar95_kl": 10.828847885131836, + "delta_nll": 3.5458602905273438, + "max_kl": 12.182533264160156, + "mean_kl": 4.036829471588135, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.849641799926758, + "code_tokens": 85, + "cvar95_kl": 10.171728134155273, + "delta_nll": 3.4278316497802734, + "max_kl": 13.276893615722656, + "mean_kl": 3.3238062858581543, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.364705890417099 + }, + { + "candidate_nll": 4.5800018310546875, + "code_tokens": 44, + "cvar95_kl": 9.328857421875, + "delta_nll": 3.6027581691741943, + "max_kl": 9.848316192626953, + "mean_kl": 3.5245866775512695, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.34090909361839294 + }, + { + "candidate_nll": 4.285816669464111, + "code_tokens": 19, + "cvar95_kl": 9.134896278381348, + "delta_nll": 3.4817144870758057, + "max_kl": 9.134896278381348, + "mean_kl": 3.145864725112915, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 3.912722110748291, + "code_tokens": 113, + "cvar95_kl": 7.099109172821045, + "delta_nll": 3.1752991676330566, + "max_kl": 8.193395614624023, + "mean_kl": 3.0454819202423096, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.28318583965301514 + }, + { + "candidate_nll": 4.106246471405029, + "code_tokens": 40, + "cvar95_kl": 8.352333068847656, + "delta_nll": 2.631052017211914, + "max_kl": 10.010682106018066, + "mean_kl": 3.242873430252075, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 5.410520076751709, + "code_tokens": 51, + "cvar95_kl": 9.872074127197266, + "delta_nll": 3.8038601875305176, + "max_kl": 10.528875350952148, + "mean_kl": 4.198887348175049, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.2549019753932953 + }, + { + "candidate_nll": 4.575567722320557, + "code_tokens": 55, + "cvar95_kl": 8.452696800231934, + "delta_nll": 2.994535207748413, + "max_kl": 9.135016441345215, + "mean_kl": 3.274672269821167, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 2.962507724761963, + "code_tokens": 27, + "cvar95_kl": 5.111700057983398, + "delta_nll": 1.8324394226074219, + "max_kl": 5.440239906311035, + "mean_kl": 1.76588773727417, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.5925925970077515 + }, + { + "candidate_nll": 4.621575355529785, + "code_tokens": 32, + "cvar95_kl": 8.598491668701172, + "delta_nll": 3.781769275665283, + "max_kl": 9.611828804016113, + "mean_kl": 3.7001614570617676, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 3.2185544967651367, + "code_tokens": 64, + "cvar95_kl": 7.547697067260742, + "delta_nll": 2.566016435623169, + "max_kl": 8.060226440429688, + "mean_kl": 2.652555227279663, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.4375 + }, + { + "candidate_nll": 4.359724044799805, + "code_tokens": 35, + "cvar95_kl": 8.89474868774414, + "delta_nll": 3.3100697994232178, + "max_kl": 9.630475997924805, + "mean_kl": 3.3086342811584473, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 4.568037986755371, + "code_tokens": 152, + "cvar95_kl": 9.05842113494873, + "delta_nll": 4.116190433502197, + "max_kl": 10.315524101257324, + "mean_kl": 3.9189751148223877, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 4.277551651000977, + "code_tokens": 77, + "cvar95_kl": 7.441728115081787, + "delta_nll": 3.0162172317504883, + "max_kl": 8.896404266357422, + "mean_kl": 2.8964054584503174, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.33766233921051025 + }, + { + "candidate_nll": 3.463831901550293, + "code_tokens": 77, + "cvar95_kl": 8.390497207641602, + "delta_nll": 2.8955507278442383, + "max_kl": 10.474390029907227, + "mean_kl": 2.8361237049102783, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.350649356842041 + }, + { + "candidate_nll": 2.6657612323760986, + "code_tokens": 71, + "cvar95_kl": 6.557036876678467, + "delta_nll": 2.103672504425049, + "max_kl": 8.181038856506348, + "mean_kl": 2.1894874572753906, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.5070422291755676 + }, + { + "candidate_nll": 5.815581798553467, + "code_tokens": 112, + "cvar95_kl": 10.728046417236328, + "delta_nll": 4.937202453613281, + "max_kl": 11.971535682678223, + "mean_kl": 4.77695894241333, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.2232142835855484 + }, + { + "candidate_nll": 2.878714084625244, + "code_tokens": 111, + "cvar95_kl": 6.277914047241211, + "delta_nll": 2.1588103771209717, + "max_kl": 7.027322292327881, + "mean_kl": 2.1527445316314697, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.4954954981803894 + }, + { + "candidate_nll": 4.277730464935303, + "code_tokens": 62, + "cvar95_kl": 9.45264720916748, + "delta_nll": 3.4898343086242676, + "max_kl": 10.809815406799316, + "mean_kl": 3.559006929397583, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 2.3460211753845215, + "code_tokens": 52, + "cvar95_kl": 5.153203010559082, + "delta_nll": 1.7354412078857422, + "max_kl": 5.947084426879883, + "mean_kl": 1.7133265733718872, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.4423076808452606 + }, + { + "candidate_nll": 3.911740303039551, + "code_tokens": 24, + "cvar95_kl": 5.612708568572998, + "delta_nll": 2.0165138244628906, + "max_kl": 5.965774059295654, + "mean_kl": 2.4392597675323486, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.52886962890625, + "code_tokens": 48, + "cvar95_kl": 7.946741104125977, + "delta_nll": 2.6370997428894043, + "max_kl": 8.770952224731445, + "mean_kl": 3.0147228240966797, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.966698169708252, + "code_tokens": 98, + "cvar95_kl": 8.729766845703125, + "delta_nll": 2.916405200958252, + "max_kl": 9.87862491607666, + "mean_kl": 3.030724287033081, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.496561288833618, + "code_tokens": 48, + "cvar95_kl": 7.195143222808838, + "delta_nll": 2.503875732421875, + "max_kl": 7.844903945922852, + "mean_kl": 2.5769705772399902, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.3541666567325592 + }, + { + "candidate_nll": 5.020009517669678, + "code_tokens": 56, + "cvar95_kl": 11.314598083496094, + "delta_nll": 4.074014186859131, + "max_kl": 12.364473342895508, + "mean_kl": 4.241601467132568, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.2321428507566452 + }, + { + "candidate_nll": 4.690566539764404, + "code_tokens": 88, + "cvar95_kl": 9.952672958374023, + "delta_nll": 4.089289665222168, + "max_kl": 11.989094734191895, + "mean_kl": 4.010793685913086, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.564791202545166, + "code_tokens": 50, + "cvar95_kl": 8.245766639709473, + "delta_nll": 2.9312610626220703, + "max_kl": 9.124855995178223, + "mean_kl": 3.072469711303711, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 2.8104894161224365, + "code_tokens": 85, + "cvar95_kl": 6.738185882568359, + "delta_nll": 2.373502254486084, + "max_kl": 9.809783935546875, + "mean_kl": 2.2030184268951416, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.4941176474094391 + }, + { + "candidate_nll": 4.5062360763549805, + "code_tokens": 27, + "cvar95_kl": 8.200380325317383, + "delta_nll": 3.432023048400879, + "max_kl": 8.210859298706055, + "mean_kl": 3.766556739807129, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 4.4259443283081055, + "code_tokens": 48, + "cvar95_kl": 10.012646675109863, + "delta_nll": 3.30832576751709, + "max_kl": 11.942523002624512, + "mean_kl": 3.6365387439727783, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.038277626037598, + "code_tokens": 29, + "cvar95_kl": 9.458065032958984, + "delta_nll": 3.390430450439453, + "max_kl": 9.508381843566895, + "mean_kl": 3.5575807094573975, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 3.9995694160461426, + "code_tokens": 63, + "cvar95_kl": 8.50751781463623, + "delta_nll": 3.061354160308838, + "max_kl": 11.031837463378906, + "mean_kl": 3.296837568283081, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.256293296813965, + "code_tokens": 89, + "cvar95_kl": 7.547812461853027, + "delta_nll": 3.5812346935272217, + "max_kl": 9.303106307983398, + "mean_kl": 3.5128982067108154, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.26966291666030884 + }, + { + "candidate_nll": 4.771695613861084, + "code_tokens": 133, + "cvar95_kl": 7.95451021194458, + "delta_nll": 3.427417278289795, + "max_kl": 9.331835746765137, + "mean_kl": 3.4850919246673584, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.24060150980949402 + }, + { + "candidate_nll": 2.7622692584991455, + "code_tokens": 25, + "cvar95_kl": 4.412172317504883, + "delta_nll": 1.8137325048446655, + "max_kl": 4.456623554229736, + "mean_kl": 1.8832076787948608, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.180335760116577, + "code_tokens": 33, + "cvar95_kl": 5.987857818603516, + "delta_nll": 2.143035411834717, + "max_kl": 6.618498802185059, + "mean_kl": 2.4810080528259277, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.754101514816284, + "code_tokens": 93, + "cvar95_kl": 6.840904235839844, + "delta_nll": 1.9973266124725342, + "max_kl": 8.058186531066895, + "mean_kl": 2.2063448429107666, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.40860214829444885 + }, + { + "candidate_nll": 4.127386569976807, + "code_tokens": 101, + "cvar95_kl": 7.634439468383789, + "delta_nll": 3.158987045288086, + "max_kl": 9.276954650878906, + "mean_kl": 3.026571035385132, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.31683167815208435 + }, + { + "candidate_nll": 4.497574329376221, + "code_tokens": 152, + "cvar95_kl": 10.49519157409668, + "delta_nll": 3.9328110218048096, + "max_kl": 11.921772003173828, + "mean_kl": 3.9016318321228027, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.2697368562221527 + }, + { + "candidate_nll": 3.901298761367798, + "code_tokens": 33, + "cvar95_kl": 7.052495956420898, + "delta_nll": 2.50042724609375, + "max_kl": 7.0742998123168945, + "mean_kl": 2.8419086933135986, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.642364263534546, + "code_tokens": 29, + "cvar95_kl": 6.063979148864746, + "delta_nll": 2.3559012413024902, + "max_kl": 6.3303117752075195, + "mean_kl": 2.5631465911865234, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.481283187866211, + "code_tokens": 27, + "cvar95_kl": 10.117565155029297, + "delta_nll": 3.0232133865356445, + "max_kl": 11.29935073852539, + "mean_kl": 3.4284515380859375, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.938042402267456, + "code_tokens": 33, + "cvar95_kl": 7.956312656402588, + "delta_nll": 3.628002166748047, + "max_kl": 8.405702590942383, + "mean_kl": 3.525794506072998, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.522766590118408, + "code_tokens": 27, + "cvar95_kl": 7.053536415100098, + "delta_nll": 2.7263684272766113, + "max_kl": 7.330708980560303, + "mean_kl": 2.9130442142486572, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9427759647369385, + "code_tokens": 84, + "cvar95_kl": 9.097234725952148, + "delta_nll": 3.238882541656494, + "max_kl": 11.369553565979004, + "mean_kl": 3.3391425609588623, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.6074094772338867, + "code_tokens": 103, + "cvar95_kl": 6.3360137939453125, + "delta_nll": 2.5619397163391113, + "max_kl": 7.712724685668945, + "mean_kl": 2.542719841003418, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.35922330617904663 + }, + { + "candidate_nll": 4.2346978187561035, + "code_tokens": 88, + "cvar95_kl": 8.626641273498535, + "delta_nll": 3.3183505535125732, + "max_kl": 9.53078556060791, + "mean_kl": 3.2833635807037354, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.34090909361839294 + }, + { + "candidate_nll": 5.372900009155273, + "code_tokens": 62, + "cvar95_kl": 8.655327796936035, + "delta_nll": 4.063969135284424, + "max_kl": 8.92595100402832, + "mean_kl": 3.793776512145996, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.20967741310596466 + }, + { + "candidate_nll": 4.202354907989502, + "code_tokens": 24, + "cvar95_kl": 5.31739616394043, + "delta_nll": 2.6656460762023926, + "max_kl": 5.478182315826416, + "mean_kl": 2.8131754398345947, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.040273189544678, + "code_tokens": 34, + "cvar95_kl": 7.359140396118164, + "delta_nll": 2.7018210887908936, + "max_kl": 7.386143207550049, + "mean_kl": 2.608687162399292, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.318253040313721, + "code_tokens": 43, + "cvar95_kl": 7.312015056610107, + "delta_nll": 3.5985076427459717, + "max_kl": 7.86962890625, + "mean_kl": 3.5602009296417236, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 4.826839447021484, + "code_tokens": 54, + "cvar95_kl": 9.6726713180542, + "delta_nll": 3.337534189224243, + "max_kl": 11.28248119354248, + "mean_kl": 3.323072671890259, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.547386407852173, + "code_tokens": 50, + "cvar95_kl": 6.334344387054443, + "delta_nll": 2.8175010681152344, + "max_kl": 6.7493486404418945, + "mean_kl": 2.7747442722320557, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 4.478532314300537, + "code_tokens": 235, + "cvar95_kl": 8.955660820007324, + "delta_nll": 3.721021890640259, + "max_kl": 11.598328590393066, + "mean_kl": 3.7061121463775635, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.31489360332489014 + }, + { + "candidate_nll": 4.851076602935791, + "code_tokens": 33, + "cvar95_kl": 6.575536727905273, + "delta_nll": 2.8224215507507324, + "max_kl": 6.769711494445801, + "mean_kl": 3.3759591579437256, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 4.2025146484375, + "code_tokens": 45, + "cvar95_kl": 8.248625755310059, + "delta_nll": 3.2160122394561768, + "max_kl": 8.658052444458008, + "mean_kl": 3.17995023727417, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.458163261413574, + "code_tokens": 28, + "cvar95_kl": 12.57642650604248, + "delta_nll": 3.8352034091949463, + "max_kl": 14.199176788330078, + "mean_kl": 3.662245273590088, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.4840688705444336, + "code_tokens": 49, + "cvar95_kl": 6.884179592132568, + "delta_nll": 2.805804491043091, + "max_kl": 8.213479042053223, + "mean_kl": 2.8509838581085205, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 4.793592929840088, + "code_tokens": 107, + "cvar95_kl": 11.222647666931152, + "delta_nll": 4.544177532196045, + "max_kl": 13.950278282165527, + "mean_kl": 4.346545696258545, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.23364485800266266 + }, + { + "candidate_nll": 4.419187068939209, + "code_tokens": 48, + "cvar95_kl": 8.974957466125488, + "delta_nll": 3.198276996612549, + "max_kl": 9.799493789672852, + "mean_kl": 3.4761483669281006, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.156785726547241, + "code_tokens": 40, + "cvar95_kl": 5.251509666442871, + "delta_nll": 1.3172448873519897, + "max_kl": 5.278302192687988, + "mean_kl": 1.7669060230255127, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.1052956581115723, + "code_tokens": 46, + "cvar95_kl": 7.947576522827148, + "delta_nll": 2.6358120441436768, + "max_kl": 10.428979873657227, + "mean_kl": 2.8028836250305176, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.0673322677612305, + "code_tokens": 31, + "cvar95_kl": 7.17875862121582, + "delta_nll": 3.0248825550079346, + "max_kl": 7.2879180908203125, + "mean_kl": 3.0643117427825928, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 1.9583607912063599, + "code_tokens": 58, + "cvar95_kl": 5.68655252456665, + "delta_nll": 1.5418763160705566, + "max_kl": 6.715342044830322, + "mean_kl": 1.5491353273391724, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 3.504296064376831, + "code_tokens": 49, + "cvar95_kl": 7.780176162719727, + "delta_nll": 2.386526107788086, + "max_kl": 11.701725959777832, + "mean_kl": 2.504680633544922, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.3265306055545807 + }, + { + "candidate_nll": 3.0501060485839844, + "code_tokens": 18, + "cvar95_kl": 7.6696319580078125, + "delta_nll": 2.1035125255584717, + "max_kl": 7.6696319580078125, + "mean_kl": 2.1018288135528564, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 3.4255993366241455, + "code_tokens": 28, + "cvar95_kl": 5.353171348571777, + "delta_nll": 2.2661876678466797, + "max_kl": 5.617177486419678, + "mean_kl": 2.5348589420318604, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.990595579147339, + "code_tokens": 57, + "cvar95_kl": 6.969613552093506, + "delta_nll": 3.2485289573669434, + "max_kl": 7.130553245544434, + "mean_kl": 3.314882516860962, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.22807016968727112 + }, + { + "candidate_nll": 3.5289106369018555, + "code_tokens": 88, + "cvar95_kl": 7.733628749847412, + "delta_nll": 2.828800678253174, + "max_kl": 8.88117790222168, + "mean_kl": 2.944613456726074, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.295660495758057, + "code_tokens": 23, + "cvar95_kl": 8.864158630371094, + "delta_nll": 3.0968737602233887, + "max_kl": 9.001667022705078, + "mean_kl": 3.308743953704834, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 5.220477104187012, + "code_tokens": 40, + "cvar95_kl": 11.230514526367188, + "delta_nll": 3.96101975440979, + "max_kl": 12.51132583618164, + "mean_kl": 4.153676986694336, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.784026622772217, + "code_tokens": 20, + "cvar95_kl": 8.350561141967773, + "delta_nll": 2.6228115558624268, + "max_kl": 8.350561141967773, + "mean_kl": 3.109578847885132, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.009519100189209, + "code_tokens": 63, + "cvar95_kl": 6.970791816711426, + "delta_nll": 2.9937334060668945, + "max_kl": 7.521615982055664, + "mean_kl": 3.0395140647888184, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 3.810168504714966, + "code_tokens": 39, + "cvar95_kl": 8.596236228942871, + "delta_nll": 2.857487201690674, + "max_kl": 9.183858871459961, + "mean_kl": 2.940120220184326, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.597004413604736, + "code_tokens": 50, + "cvar95_kl": 8.738770484924316, + "delta_nll": 3.1851396560668945, + "max_kl": 9.213842391967773, + "mean_kl": 3.3788444995880127, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 4.038053512573242, + "code_tokens": 63, + "cvar95_kl": 8.812397003173828, + "delta_nll": 3.0705761909484863, + "max_kl": 9.29512882232666, + "mean_kl": 3.350170135498047, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 4.832842826843262, + "code_tokens": 46, + "cvar95_kl": 7.811056137084961, + "delta_nll": 3.860280752182007, + "max_kl": 8.40959644317627, + "mean_kl": 3.887354612350464, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 3.5227251052856445, + "code_tokens": 25, + "cvar95_kl": 8.145697593688965, + "delta_nll": 2.8438305854797363, + "max_kl": 8.947132110595703, + "mean_kl": 2.9731009006500244, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.5214409828186035, + "code_tokens": 111, + "cvar95_kl": 6.319309711456299, + "delta_nll": 2.7071750164031982, + "max_kl": 7.487135887145996, + "mean_kl": 2.7776269912719727, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.9788100719451904, + "code_tokens": 71, + "cvar95_kl": 6.970831394195557, + "delta_nll": 3.2502522468566895, + "max_kl": 9.042606353759766, + "mean_kl": 3.1130337715148926, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.28169015049934387 + }, + { + "candidate_nll": 5.11986780166626, + "code_tokens": 28, + "cvar95_kl": 9.08627986907959, + "delta_nll": 3.7491440773010254, + "max_kl": 10.635926246643066, + "mean_kl": 4.032223224639893, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 4.505980491638184, + "code_tokens": 25, + "cvar95_kl": 9.998075485229492, + "delta_nll": 3.3055245876312256, + "max_kl": 10.933615684509277, + "mean_kl": 3.464348554611206, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.627643585205078, + "code_tokens": 40, + "cvar95_kl": 6.655121803283691, + "delta_nll": 2.324871778488159, + "max_kl": 7.711423397064209, + "mean_kl": 2.3875560760498047, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.42500001192092896 + }, + { + "candidate_nll": 4.000006675720215, + "code_tokens": 177, + "cvar95_kl": 7.778572082519531, + "delta_nll": 2.9555625915527344, + "max_kl": 9.903688430786133, + "mean_kl": 3.2165238857269287, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.3050847351551056 + }, + { + "candidate_nll": 3.7381317615509033, + "code_tokens": 51, + "cvar95_kl": 6.526729583740234, + "delta_nll": 2.6469531059265137, + "max_kl": 7.048407077789307, + "mean_kl": 2.665651559829712, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 2.7420108318328857, + "code_tokens": 69, + "cvar95_kl": 7.368505001068115, + "delta_nll": 2.287550449371338, + "max_kl": 8.561427116394043, + "mean_kl": 2.2227847576141357, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.5072463750839233 + }, + { + "candidate_nll": 3.376431465148926, + "code_tokens": 26, + "cvar95_kl": 7.039741516113281, + "delta_nll": 2.628171443939209, + "max_kl": 7.531854629516602, + "mean_kl": 2.6394076347351074, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.4444005489349365, + "code_tokens": 24, + "cvar95_kl": 4.952383995056152, + "delta_nll": 1.9260612726211548, + "max_kl": 5.796665668487549, + "mean_kl": 2.114863157272339, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.170371055603027, + "code_tokens": 15, + "cvar95_kl": 8.946414947509766, + "delta_nll": 3.019895076751709, + "max_kl": 8.946414947509766, + "mean_kl": 2.777305841445923, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.026669025421143, + "code_tokens": 43, + "cvar95_kl": 9.476262092590332, + "delta_nll": 2.8773200511932373, + "max_kl": 10.458745956420898, + "mean_kl": 3.202457904815674, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.3720930218696594 + }, + { + "candidate_nll": 3.586723804473877, + "code_tokens": 18, + "cvar95_kl": 5.073284149169922, + "delta_nll": 2.4943981170654297, + "max_kl": 5.073284149169922, + "mean_kl": 2.38215708732605, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.41379714012146, + "code_tokens": 49, + "cvar95_kl": 7.442148685455322, + "delta_nll": 2.6858036518096924, + "max_kl": 8.056628227233887, + "mean_kl": 2.685866117477417, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.3469387888908386 + }, + { + "candidate_nll": 3.7072064876556396, + "code_tokens": 59, + "cvar95_kl": 7.134310245513916, + "delta_nll": 2.8394503593444824, + "max_kl": 8.511360168457031, + "mean_kl": 2.911407470703125, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.2881355881690979 + }, + { + "candidate_nll": 2.849698781967163, + "code_tokens": 221, + "cvar95_kl": 7.032480716705322, + "delta_nll": 2.264432668685913, + "max_kl": 8.932659149169922, + "mean_kl": 2.3375356197357178, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 4.765591621398926, + "code_tokens": 25, + "cvar95_kl": 8.979406356811523, + "delta_nll": 3.563718557357788, + "max_kl": 9.218477249145508, + "mean_kl": 3.6696109771728516, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 3.606206178665161, + "code_tokens": 36, + "cvar95_kl": 8.280641555786133, + "delta_nll": 2.8822855949401855, + "max_kl": 8.799342155456543, + "mean_kl": 3.086287498474121, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.378762722015381, + "code_tokens": 158, + "cvar95_kl": 10.287166595458984, + "delta_nll": 3.7346179485321045, + "max_kl": 15.137598991394043, + "mean_kl": 3.7028303146362305, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.1582278460264206 + }, + { + "candidate_nll": 4.426680088043213, + "code_tokens": 92, + "cvar95_kl": 8.70273494720459, + "delta_nll": 3.11114501953125, + "max_kl": 10.160143852233887, + "mean_kl": 3.1159887313842773, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 3.0916574001312256, + "code_tokens": 102, + "cvar95_kl": 5.793104648590088, + "delta_nll": 2.177614688873291, + "max_kl": 6.351934909820557, + "mean_kl": 2.149204969406128, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.4215686321258545 + }, + { + "candidate_nll": 4.106569290161133, + "code_tokens": 171, + "cvar95_kl": 9.104877471923828, + "delta_nll": 3.4965991973876953, + "max_kl": 11.10040283203125, + "mean_kl": 3.4872078895568848, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.28654971718788147 + }, + { + "candidate_nll": 4.138050556182861, + "code_tokens": 40, + "cvar95_kl": 10.129643440246582, + "delta_nll": 2.9331865310668945, + "max_kl": 11.144844055175781, + "mean_kl": 3.171175241470337, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.304347515106201, + "code_tokens": 48, + "cvar95_kl": 7.4492621421813965, + "delta_nll": 3.068333625793457, + "max_kl": 8.699409484863281, + "mean_kl": 3.026773691177368, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.738336086273193, + "code_tokens": 24, + "cvar95_kl": 8.817025184631348, + "delta_nll": 3.133415699005127, + "max_kl": 9.876923561096191, + "mean_kl": 3.7210094928741455, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.6522138118743896, + "code_tokens": 48, + "cvar95_kl": 6.938449382781982, + "delta_nll": 2.665109634399414, + "max_kl": 7.2476887702941895, + "mean_kl": 2.496703624725342, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.1392018795013428, + "code_tokens": 25, + "cvar95_kl": 6.961149215698242, + "delta_nll": 1.7523187398910522, + "max_kl": 7.007984638214111, + "mean_kl": 1.9330166578292847, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 4.688647747039795, + "code_tokens": 30, + "cvar95_kl": 8.320935249328613, + "delta_nll": 3.1899027824401855, + "max_kl": 8.825041770935059, + "mean_kl": 3.074737071990967, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.524921178817749, + "code_tokens": 19, + "cvar95_kl": 7.019408702850342, + "delta_nll": 2.0981032848358154, + "max_kl": 7.019408702850342, + "mean_kl": 2.317767381668091, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 2.8198719024658203, + "code_tokens": 12, + "cvar95_kl": 4.2842912673950195, + "delta_nll": 1.7407994270324707, + "max_kl": 4.2842912673950195, + "mean_kl": 1.9953627586364746, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.1652984619140625, + "code_tokens": 37, + "cvar95_kl": 9.8702392578125, + "delta_nll": 2.824988842010498, + "max_kl": 9.927691459655762, + "mean_kl": 3.121124744415283, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.175842046737671, + "code_tokens": 33, + "cvar95_kl": 5.078061580657959, + "delta_nll": 1.970412254333496, + "max_kl": 5.5442399978637695, + "mean_kl": 2.1133458614349365, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 4.94415283203125, + "code_tokens": 94, + "cvar95_kl": 9.39877986907959, + "delta_nll": 4.092042922973633, + "max_kl": 10.789039611816406, + "mean_kl": 4.264394760131836, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.22340425848960876 + }, + { + "candidate_nll": 4.06624698638916, + "code_tokens": 90, + "cvar95_kl": 9.609384536743164, + "delta_nll": 3.4414820671081543, + "max_kl": 11.955211639404297, + "mean_kl": 3.4828882217407227, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.5439648628234863, + "code_tokens": 47, + "cvar95_kl": 6.173450946807861, + "delta_nll": 2.096200942993164, + "max_kl": 6.8264946937561035, + "mean_kl": 2.4570376873016357, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.27659574151039124 + }, + { + "candidate_nll": 3.6723358631134033, + "code_tokens": 61, + "cvar95_kl": 7.782769203186035, + "delta_nll": 2.748501777648926, + "max_kl": 10.081992149353027, + "mean_kl": 3.0773355960845947, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.31147539615631104 + }, + { + "candidate_nll": 3.8465607166290283, + "code_tokens": 39, + "cvar95_kl": 6.134505271911621, + "delta_nll": 2.6230459213256836, + "max_kl": 6.436993598937988, + "mean_kl": 2.7356226444244385, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 4.073806285858154, + "code_tokens": 152, + "cvar95_kl": 8.318432807922363, + "delta_nll": 3.1416444778442383, + "max_kl": 9.643752098083496, + "mean_kl": 3.2430248260498047, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.29605263471603394 + }, + { + "candidate_nll": 4.495396614074707, + "code_tokens": 100, + "cvar95_kl": 9.591527938842773, + "delta_nll": 3.526414155960083, + "max_kl": 10.836614608764648, + "mean_kl": 3.4542932510375977, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 3.3511228561401367, + "code_tokens": 89, + "cvar95_kl": 7.707283020019531, + "delta_nll": 2.7301950454711914, + "max_kl": 9.219827651977539, + "mean_kl": 2.478746175765991, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.449438214302063 + }, + { + "candidate_nll": 4.433993339538574, + "code_tokens": 40, + "cvar95_kl": 9.194869995117188, + "delta_nll": 3.1229920387268066, + "max_kl": 10.003822326660156, + "mean_kl": 3.3421051502227783, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 3.5940515995025635, + "code_tokens": 76, + "cvar95_kl": 7.493351459503174, + "delta_nll": 2.892484664916992, + "max_kl": 7.840336322784424, + "mean_kl": 2.8892064094543457, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.30263158679008484 + }, + { + "candidate_nll": 3.666146993637085, + "code_tokens": 56, + "cvar95_kl": 7.183818817138672, + "delta_nll": 2.4171595573425293, + "max_kl": 9.006553649902344, + "mean_kl": 2.4676945209503174, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 5.117517471313477, + "code_tokens": 34, + "cvar95_kl": 9.496617317199707, + "delta_nll": 3.628087043762207, + "max_kl": 9.809468269348145, + "mean_kl": 3.484168529510498, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 5.092311382293701, + "code_tokens": 96, + "cvar95_kl": 11.266045570373535, + "delta_nll": 4.4141998291015625, + "max_kl": 12.046244621276855, + "mean_kl": 4.292557716369629, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.2709245681762695, + "code_tokens": 451, + "cvar95_kl": 8.783744812011719, + "delta_nll": 2.5722293853759766, + "max_kl": 15.453869819641113, + "mean_kl": 2.539677381515503, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.3725055456161499 + }, + { + "candidate_nll": 4.767435073852539, + "code_tokens": 37, + "cvar95_kl": 9.331571578979492, + "delta_nll": 3.155125379562378, + "max_kl": 11.520435333251953, + "mean_kl": 3.3730828762054443, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.975503921508789, + "code_tokens": 42, + "cvar95_kl": 8.23976993560791, + "delta_nll": 2.7339060306549072, + "max_kl": 10.853692054748535, + "mean_kl": 3.0615336894989014, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.261904776096344 + }, + { + "candidate_nll": 3.7510242462158203, + "code_tokens": 33, + "cvar95_kl": 6.723182201385498, + "delta_nll": 2.7342681884765625, + "max_kl": 6.795413017272949, + "mean_kl": 2.707012176513672, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 3.9662234783172607, + "code_tokens": 43, + "cvar95_kl": 8.022358894348145, + "delta_nll": 3.0401718616485596, + "max_kl": 8.546387672424316, + "mean_kl": 2.9873099327087402, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.3255814015865326 + }, + { + "candidate_nll": 3.946845531463623, + "code_tokens": 73, + "cvar95_kl": 7.9548749923706055, + "delta_nll": 3.210409164428711, + "max_kl": 8.460836410522461, + "mean_kl": 3.231168508529663, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.2876712381839752 + }, + { + "candidate_nll": 3.332064628601074, + "code_tokens": 15, + "cvar95_kl": 5.96636962890625, + "delta_nll": 2.419588565826416, + "max_kl": 5.96636962890625, + "mean_kl": 2.1968038082122803, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.478579521179199, + "code_tokens": 34, + "cvar95_kl": 8.126416206359863, + "delta_nll": 3.2194995880126953, + "max_kl": 8.181243896484375, + "mean_kl": 3.783479928970337, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 4.3332977294921875, + "code_tokens": 136, + "cvar95_kl": 8.097620010375977, + "delta_nll": 3.2248144149780273, + "max_kl": 10.211221694946289, + "mean_kl": 3.1222288608551025, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.24264705181121826 + }, + { + "candidate_nll": 2.8249597549438477, + "code_tokens": 17, + "cvar95_kl": 4.028849124908447, + "delta_nll": 1.5788036584854126, + "max_kl": 4.028849124908447, + "mean_kl": 1.8737424612045288, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.228294372558594, + "code_tokens": 33, + "cvar95_kl": 10.715858459472656, + "delta_nll": 3.435239791870117, + "max_kl": 11.688087463378906, + "mean_kl": 3.3324081897735596, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.500195503234863, + "code_tokens": 38, + "cvar95_kl": 7.226444244384766, + "delta_nll": 3.6577486991882324, + "max_kl": 7.749030590057373, + "mean_kl": 3.5171196460723877, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 3.748439073562622, + "code_tokens": 61, + "cvar95_kl": 9.04489803314209, + "delta_nll": 3.350914478302002, + "max_kl": 10.577219009399414, + "mean_kl": 3.4321322441101074, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.21311475336551666 + }, + { + "candidate_nll": 3.4495086669921875, + "code_tokens": 143, + "cvar95_kl": 7.34695291519165, + "delta_nll": 2.713789224624634, + "max_kl": 9.661739349365234, + "mean_kl": 2.826554298400879, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.3986014127731323 + }, + { + "candidate_nll": 3.3510959148406982, + "code_tokens": 41, + "cvar95_kl": 8.310803413391113, + "delta_nll": 2.2620301246643066, + "max_kl": 8.8368558883667, + "mean_kl": 2.577833890914917, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.3414634168148041 + }, + { + "candidate_nll": 2.663095235824585, + "code_tokens": 67, + "cvar95_kl": 7.950413227081299, + "delta_nll": 2.2984135150909424, + "max_kl": 9.024316787719727, + "mean_kl": 2.542635917663574, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.447761207818985 + }, + { + "candidate_nll": 4.8798747062683105, + "code_tokens": 76, + "cvar95_kl": 9.457457542419434, + "delta_nll": 3.489614725112915, + "max_kl": 10.809778213500977, + "mean_kl": 3.4250736236572266, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 3.7634739875793457, + "code_tokens": 137, + "cvar95_kl": 7.908554553985596, + "delta_nll": 3.1276214122772217, + "max_kl": 10.755269050598145, + "mean_kl": 3.051689863204956, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.37956205010414124 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l18_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 18 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 4.989647388458252, + "median": 2.928609609603882, + "minimum": 1.0272496938705444, + "q25": 2.497844934463501, + "q75": 3.3764167428016663 + }, + "task_macro": { + "candidate_nll": 3.94275648355484, + "delta_nll": 2.9287099046707152, + "mean_kl": 3.013556678533554, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.3349523193389177 + }, + "token_weighted": { + "candidate_nll": 3.981560230255127, + "cvar95_kl": 8.911604881286621, + "delta_nll": 3.0647225379943848, + "max_kl": 16.496646881103516, + "mean_kl": 3.120887279510498, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.3239981532096863 + } + }, + "random_l4_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.319992745399475, + "macro_mean_kl": 2.4297506403923035, + "macro_top1_agreement": 0.4132299636602402, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 2.753326528549194, + "macro_mean_kl": 2.8903471670150758, + "macro_top1_agreement": 0.3467226977348328, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 2.8762920045852662, + "macro_mean_kl": 2.9375559701919554, + "macro_top1_agreement": 0.3404135471582413, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 3.011355362892151, + "macro_mean_kl": 3.0352238759994505, + "macro_top1_agreement": 0.3418193063735962, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 3.4407646656036377, + "code_tokens": 98, + "cvar95_kl": 8.395683288574219, + "delta_nll": 2.7139089107513428, + "max_kl": 9.4617338180542, + "mean_kl": 3.0815675258636475, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 4.110629558563232, + "code_tokens": 19, + "cvar95_kl": 7.980707168579102, + "delta_nll": 2.951216459274292, + "max_kl": 7.980707168579102, + "mean_kl": 3.0705418586730957, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.753441333770752, + "code_tokens": 39, + "cvar95_kl": 10.231269836425781, + "delta_nll": 2.7380919456481934, + "max_kl": 13.851889610290527, + "mean_kl": 3.0674631595611572, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 4.483571529388428, + "code_tokens": 24, + "cvar95_kl": 7.426213264465332, + "delta_nll": 3.0372819900512695, + "max_kl": 7.529287338256836, + "mean_kl": 2.8463830947875977, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.784829616546631, + "code_tokens": 25, + "cvar95_kl": 6.913713455200195, + "delta_nll": 2.3770713806152344, + "max_kl": 7.904742240905762, + "mean_kl": 2.2217562198638916, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 4.159047603607178, + "code_tokens": 55, + "cvar95_kl": 8.612700462341309, + "delta_nll": 3.2634360790252686, + "max_kl": 10.99951457977295, + "mean_kl": 3.1563193798065186, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.3272727131843567 + }, + { + "candidate_nll": 2.6867146492004395, + "code_tokens": 16, + "cvar95_kl": 6.322206974029541, + "delta_nll": 1.857274055480957, + "max_kl": 6.322206974029541, + "mean_kl": 1.6705739498138428, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 4.8356499671936035, + "code_tokens": 199, + "cvar95_kl": 10.2648286819458, + "delta_nll": 3.6137566566467285, + "max_kl": 12.219894409179688, + "mean_kl": 3.6905579566955566, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.28140702843666077 + }, + { + "candidate_nll": 3.7189533710479736, + "code_tokens": 28, + "cvar95_kl": 4.888851642608643, + "delta_nll": 2.1528754234313965, + "max_kl": 5.371127605438232, + "mean_kl": 1.9718939065933228, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.1745901107788086, + "code_tokens": 87, + "cvar95_kl": 5.867096900939941, + "delta_nll": 2.301607608795166, + "max_kl": 6.530790328979492, + "mean_kl": 2.3309028148651123, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.39080458879470825 + }, + { + "candidate_nll": 3.853524684906006, + "code_tokens": 35, + "cvar95_kl": 8.97774600982666, + "delta_nll": 2.620560646057129, + "max_kl": 9.439665794372559, + "mean_kl": 3.3201422691345215, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.34285715222358704 + }, + { + "candidate_nll": 3.6874399185180664, + "code_tokens": 62, + "cvar95_kl": 8.863676071166992, + "delta_nll": 3.0481483936309814, + "max_kl": 9.761646270751953, + "mean_kl": 2.9456310272216797, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 4.5126495361328125, + "code_tokens": 58, + "cvar95_kl": 9.186108589172363, + "delta_nll": 3.4075918197631836, + "max_kl": 9.72826099395752, + "mean_kl": 3.3570501804351807, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.2586206793785095 + }, + { + "candidate_nll": 3.943188190460205, + "code_tokens": 79, + "cvar95_kl": 7.254729270935059, + "delta_nll": 2.784140110015869, + "max_kl": 7.7720232009887695, + "mean_kl": 3.030134677886963, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.29113924503326416 + }, + { + "candidate_nll": 3.37683367729187, + "code_tokens": 58, + "cvar95_kl": 6.9810967445373535, + "delta_nll": 2.2837138175964355, + "max_kl": 7.369553089141846, + "mean_kl": 2.527056932449341, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.36206895112991333 + }, + { + "candidate_nll": 4.257217884063721, + "code_tokens": 48, + "cvar95_kl": 7.147885799407959, + "delta_nll": 3.095020055770874, + "max_kl": 7.4577317237854, + "mean_kl": 3.1893155574798584, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 4.634400844573975, + "code_tokens": 37, + "cvar95_kl": 9.590239524841309, + "delta_nll": 3.5089683532714844, + "max_kl": 10.275575637817383, + "mean_kl": 3.368974208831787, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 2.9487063884735107, + "code_tokens": 61, + "cvar95_kl": 7.136788368225098, + "delta_nll": 2.2412991523742676, + "max_kl": 8.031384468078613, + "mean_kl": 2.389334201812744, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.39344263076782227 + }, + { + "candidate_nll": 3.987739086151123, + "code_tokens": 77, + "cvar95_kl": 7.483656883239746, + "delta_nll": 3.218019485473633, + "max_kl": 8.323501586914062, + "mean_kl": 3.0812506675720215, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.376420497894287, + "code_tokens": 98, + "cvar95_kl": 8.504822731018066, + "delta_nll": 3.354717254638672, + "max_kl": 9.233647346496582, + "mean_kl": 3.173337936401367, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.3469387888908386 + }, + { + "candidate_nll": 4.857462406158447, + "code_tokens": 162, + "cvar95_kl": 10.565969467163086, + "delta_nll": 4.272311210632324, + "max_kl": 13.234207153320312, + "mean_kl": 4.311803817749023, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.22839505970478058 + }, + { + "candidate_nll": 4.260203838348389, + "code_tokens": 103, + "cvar95_kl": 8.725066184997559, + "delta_nll": 3.515737771987915, + "max_kl": 9.41884708404541, + "mean_kl": 3.578217029571533, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.25242719054222107 + }, + { + "candidate_nll": 4.9237060546875, + "code_tokens": 73, + "cvar95_kl": 7.925129413604736, + "delta_nll": 3.6047356128692627, + "max_kl": 8.223356246948242, + "mean_kl": 3.4910688400268555, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.2602739632129669 + }, + { + "candidate_nll": 4.5240654945373535, + "code_tokens": 119, + "cvar95_kl": 9.83855152130127, + "delta_nll": 3.5410919189453125, + "max_kl": 15.148799896240234, + "mean_kl": 3.7315475940704346, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.2689075767993927 + }, + { + "candidate_nll": 2.055091142654419, + "code_tokens": 17, + "cvar95_kl": 7.289971351623535, + "delta_nll": 0.8655678033828735, + "max_kl": 7.289971351623535, + "mean_kl": 1.9895042181015015, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.529411792755127 + }, + { + "candidate_nll": 3.6969916820526123, + "code_tokens": 56, + "cvar95_kl": 7.461082935333252, + "delta_nll": 2.271289825439453, + "max_kl": 7.66545295715332, + "mean_kl": 2.3632915019989014, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.1333560943603516, + "code_tokens": 57, + "cvar95_kl": 8.181538581848145, + "delta_nll": 2.6659955978393555, + "max_kl": 8.518342018127441, + "mean_kl": 2.6278743743896484, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.4912280738353729 + }, + { + "candidate_nll": 3.491610527038574, + "code_tokens": 63, + "cvar95_kl": 8.193458557128906, + "delta_nll": 2.858865261077881, + "max_kl": 9.659364700317383, + "mean_kl": 2.886281728744507, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 5.228814601898193, + "code_tokens": 200, + "cvar95_kl": 11.342015266418457, + "delta_nll": 4.390879154205322, + "max_kl": 13.422186851501465, + "mean_kl": 4.402688026428223, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.23000000417232513 + }, + { + "candidate_nll": 4.029473304748535, + "code_tokens": 30, + "cvar95_kl": 7.54890775680542, + "delta_nll": 2.945270538330078, + "max_kl": 8.157188415527344, + "mean_kl": 2.966036081314087, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.3336703777313232, + "code_tokens": 28, + "cvar95_kl": 9.93556022644043, + "delta_nll": 2.7173192501068115, + "max_kl": 10.881975173950195, + "mean_kl": 2.5609405040740967, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.2298696041107178, + "code_tokens": 26, + "cvar95_kl": 6.810540199279785, + "delta_nll": 2.4589128494262695, + "max_kl": 7.184243202209473, + "mean_kl": 2.3746492862701416, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.2171518802642822, + "code_tokens": 54, + "cvar95_kl": 8.217673301696777, + "delta_nll": 2.3842105865478516, + "max_kl": 11.342459678649902, + "mean_kl": 2.4209399223327637, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 2.8532519340515137, + "code_tokens": 47, + "cvar95_kl": 5.747776031494141, + "delta_nll": 1.8965158462524414, + "max_kl": 6.756322383880615, + "mean_kl": 2.048374891281128, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.38297873735427856 + }, + { + "candidate_nll": 4.035120964050293, + "code_tokens": 93, + "cvar95_kl": 8.841249465942383, + "delta_nll": 3.300591468811035, + "max_kl": 9.841086387634277, + "mean_kl": 3.0871903896331787, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.37634408473968506 + }, + { + "candidate_nll": 3.1843173503875732, + "code_tokens": 30, + "cvar95_kl": 7.47149658203125, + "delta_nll": 2.3249428272247314, + "max_kl": 8.655618667602539, + "mean_kl": 2.5724570751190186, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 4.2105278968811035, + "code_tokens": 89, + "cvar95_kl": 7.655592441558838, + "delta_nll": 3.142024517059326, + "max_kl": 9.517688751220703, + "mean_kl": 3.0600833892822266, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.26966291666030884 + }, + { + "candidate_nll": 4.225256443023682, + "code_tokens": 70, + "cvar95_kl": 9.354639053344727, + "delta_nll": 3.261007070541382, + "max_kl": 9.66158390045166, + "mean_kl": 3.4221255779266357, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.3285714387893677 + }, + { + "candidate_nll": 4.282116413116455, + "code_tokens": 26, + "cvar95_kl": 6.330634117126465, + "delta_nll": 2.668889284133911, + "max_kl": 6.374765396118164, + "mean_kl": 3.09578537940979, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.535594940185547, + "code_tokens": 48, + "cvar95_kl": 8.032740592956543, + "delta_nll": 2.577888011932373, + "max_kl": 9.686149597167969, + "mean_kl": 2.6800191402435303, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.3958333432674408 + }, + { + "candidate_nll": 2.677454710006714, + "code_tokens": 29, + "cvar95_kl": 4.724508285522461, + "delta_nll": 1.8083066940307617, + "max_kl": 4.738491535186768, + "mean_kl": 1.8836694955825806, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.4137931168079376 + }, + { + "candidate_nll": 4.525897979736328, + "code_tokens": 17, + "cvar95_kl": 7.67483377456665, + "delta_nll": 3.4127349853515625, + "max_kl": 7.67483377456665, + "mean_kl": 2.933849573135376, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 3.1669304370880127, + "code_tokens": 37, + "cvar95_kl": 6.230550765991211, + "delta_nll": 2.2467143535614014, + "max_kl": 6.563260078430176, + "mean_kl": 2.3265950679779053, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 4.600305080413818, + "code_tokens": 121, + "cvar95_kl": 10.327154159545898, + "delta_nll": 4.088903427124023, + "max_kl": 17.669700622558594, + "mean_kl": 4.053330421447754, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.22314049303531647 + }, + { + "candidate_nll": 3.69681978225708, + "code_tokens": 30, + "cvar95_kl": 7.252873420715332, + "delta_nll": 2.297976016998291, + "max_kl": 7.349137306213379, + "mean_kl": 2.4538111686706543, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.033553123474121, + "code_tokens": 74, + "cvar95_kl": 8.282072067260742, + "delta_nll": 3.1350812911987305, + "max_kl": 9.365809440612793, + "mean_kl": 3.1464929580688477, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.353900909423828, + "code_tokens": 53, + "cvar95_kl": 6.337156295776367, + "delta_nll": 2.519477128982544, + "max_kl": 6.717959880828857, + "mean_kl": 2.469224452972412, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.3396226465702057 + }, + { + "candidate_nll": 3.810359001159668, + "code_tokens": 21, + "cvar95_kl": 5.415643215179443, + "delta_nll": 1.9509632587432861, + "max_kl": 5.461890697479248, + "mean_kl": 2.326721668243408, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.665421485900879, + "code_tokens": 20, + "cvar95_kl": 6.44553804397583, + "delta_nll": 1.3030078411102295, + "max_kl": 6.44553804397583, + "mean_kl": 1.4029724597930908, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 3.2908949851989746, + "code_tokens": 122, + "cvar95_kl": 7.49111795425415, + "delta_nll": 2.4718966484069824, + "max_kl": 10.474172592163086, + "mean_kl": 2.436627149581909, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.3852458894252777 + }, + { + "candidate_nll": 4.055283069610596, + "code_tokens": 94, + "cvar95_kl": 8.100759506225586, + "delta_nll": 3.2151432037353516, + "max_kl": 8.599501609802246, + "mean_kl": 3.2595391273498535, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.3085106313228607 + }, + { + "candidate_nll": 2.8870394229888916, + "code_tokens": 12, + "cvar95_kl": 5.268968105316162, + "delta_nll": 1.8217378854751587, + "max_kl": 5.268968105316162, + "mean_kl": 1.6851097345352173, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.589277744293213, + "code_tokens": 38, + "cvar95_kl": 7.729896068572998, + "delta_nll": 3.0828776359558105, + "max_kl": 7.922307968139648, + "mean_kl": 3.5724189281463623, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 2.1772570610046387, + "code_tokens": 58, + "cvar95_kl": 5.982794284820557, + "delta_nll": 1.2724699974060059, + "max_kl": 6.65003776550293, + "mean_kl": 1.2967588901519775, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.568965494632721 + }, + { + "candidate_nll": 4.310214042663574, + "code_tokens": 54, + "cvar95_kl": 8.85581111907959, + "delta_nll": 3.7139501571655273, + "max_kl": 9.198653221130371, + "mean_kl": 3.589865207672119, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.7489256858825684, + "code_tokens": 40, + "cvar95_kl": 7.271544456481934, + "delta_nll": 2.817385673522949, + "max_kl": 8.024086952209473, + "mean_kl": 3.078301191329956, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.1897823810577393, + "code_tokens": 115, + "cvar95_kl": 7.380054950714111, + "delta_nll": 2.677236557006836, + "max_kl": 9.167708396911621, + "mean_kl": 2.597524642944336, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.426086962223053 + }, + { + "candidate_nll": 4.059971809387207, + "code_tokens": 60, + "cvar95_kl": 8.729886054992676, + "delta_nll": 3.535358190536499, + "max_kl": 9.858399391174316, + "mean_kl": 3.574852466583252, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.097425937652588, + "code_tokens": 124, + "cvar95_kl": 7.983603000640869, + "delta_nll": 3.3274388313293457, + "max_kl": 9.545974731445312, + "mean_kl": 3.267894983291626, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 5.028902530670166, + "code_tokens": 91, + "cvar95_kl": 10.50126838684082, + "delta_nll": 4.258153915405273, + "max_kl": 11.695731163024902, + "mean_kl": 4.190471172332764, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 5.475979328155518, + "code_tokens": 125, + "cvar95_kl": 12.090612411499023, + "delta_nll": 4.656062126159668, + "max_kl": 13.186424255371094, + "mean_kl": 4.8274922370910645, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 2.3060433864593506, + "code_tokens": 26, + "cvar95_kl": 3.690910816192627, + "delta_nll": 1.1701668500900269, + "max_kl": 3.8381435871124268, + "mean_kl": 1.7229379415512085, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 4.414707660675049, + "code_tokens": 24, + "cvar95_kl": 9.080997467041016, + "delta_nll": 2.1742825508117676, + "max_kl": 9.238471031188965, + "mean_kl": 2.6116130352020264, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.3175811767578125, + "code_tokens": 180, + "cvar95_kl": 8.167052268981934, + "delta_nll": 2.4633994102478027, + "max_kl": 10.722310066223145, + "mean_kl": 2.5211284160614014, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.4055555462837219 + }, + { + "candidate_nll": 4.577696800231934, + "code_tokens": 40, + "cvar95_kl": 10.613689422607422, + "delta_nll": 3.202888250350952, + "max_kl": 13.49239444732666, + "mean_kl": 3.393576145172119, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.677060127258301, + "code_tokens": 72, + "cvar95_kl": 7.655529499053955, + "delta_nll": 2.7020633220672607, + "max_kl": 8.503034591674805, + "mean_kl": 2.8197944164276123, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.078174591064453, + "code_tokens": 20, + "cvar95_kl": 4.961228847503662, + "delta_nll": 1.500603437423706, + "max_kl": 4.961228847503662, + "mean_kl": 2.021690845489502, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.8069045543670654, + "code_tokens": 76, + "cvar95_kl": 7.209811687469482, + "delta_nll": 2.81626033782959, + "max_kl": 8.337162017822266, + "mean_kl": 2.8446216583251953, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 4.1854777336120605, + "code_tokens": 42, + "cvar95_kl": 7.376758575439453, + "delta_nll": 2.8137364387512207, + "max_kl": 9.79542064666748, + "mean_kl": 2.8953349590301514, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.3095238208770752 + }, + { + "candidate_nll": 2.2177894115448, + "code_tokens": 28, + "cvar95_kl": 6.042934417724609, + "delta_nll": 1.634566307067871, + "max_kl": 6.064579010009766, + "mean_kl": 1.465990662574768, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 4.44095516204834, + "code_tokens": 58, + "cvar95_kl": 10.215277671813965, + "delta_nll": 3.4074783325195312, + "max_kl": 10.966558456420898, + "mean_kl": 3.476846933364868, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 3.009279489517212, + "code_tokens": 25, + "cvar95_kl": 6.038324356079102, + "delta_nll": 2.140630006790161, + "max_kl": 6.265566349029541, + "mean_kl": 2.275395631790161, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.895822525024414, + "code_tokens": 88, + "cvar95_kl": 8.813055038452148, + "delta_nll": 3.930814743041992, + "max_kl": 9.6246976852417, + "mean_kl": 3.9981322288513184, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.20454545319080353 + }, + { + "candidate_nll": 3.463404417037964, + "code_tokens": 42, + "cvar95_kl": 6.603689670562744, + "delta_nll": 2.3358871936798096, + "max_kl": 7.230937957763672, + "mean_kl": 2.227414846420288, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.6407744884490967, + "code_tokens": 24, + "cvar95_kl": 6.924835205078125, + "delta_nll": 2.821223735809326, + "max_kl": 7.36505651473999, + "mean_kl": 2.9118385314941406, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.3786511421203613, + "code_tokens": 24, + "cvar95_kl": 4.854696750640869, + "delta_nll": 1.693779706954956, + "max_kl": 5.009117126464844, + "mean_kl": 1.7894150018692017, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.822678327560425, + "code_tokens": 43, + "cvar95_kl": 8.010366439819336, + "delta_nll": 2.7554993629455566, + "max_kl": 9.274551391601562, + "mean_kl": 2.6113083362579346, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.4651162922382355 + }, + { + "candidate_nll": 3.1232128143310547, + "code_tokens": 24, + "cvar95_kl": 8.409770011901855, + "delta_nll": 2.2369930744171143, + "max_kl": 8.512411117553711, + "mean_kl": 2.3921945095062256, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.0936808586120605, + "code_tokens": 14, + "cvar95_kl": 4.676770210266113, + "delta_nll": 1.781055212020874, + "max_kl": 4.676770210266113, + "mean_kl": 1.9094706773757935, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.5764052867889404, + "code_tokens": 41, + "cvar95_kl": 7.2562994956970215, + "delta_nll": 3.085361957550049, + "max_kl": 8.500829696655273, + "mean_kl": 2.7432944774627686, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.39024388790130615 + }, + { + "candidate_nll": 4.303398609161377, + "code_tokens": 32, + "cvar95_kl": 8.731542587280273, + "delta_nll": 3.315058708190918, + "max_kl": 9.335165023803711, + "mean_kl": 3.3956761360168457, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.34375 + }, + { + "candidate_nll": 2.948302745819092, + "code_tokens": 53, + "cvar95_kl": 6.198042392730713, + "delta_nll": 2.186607837677002, + "max_kl": 6.636029243469238, + "mean_kl": 2.025085687637329, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.4528301954269409 + }, + { + "candidate_nll": 3.2644708156585693, + "code_tokens": 51, + "cvar95_kl": 7.864051818847656, + "delta_nll": 2.8861796855926514, + "max_kl": 8.599980354309082, + "mean_kl": 2.839130401611328, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.270907878875732, + "code_tokens": 36, + "cvar95_kl": 6.858707427978516, + "delta_nll": 2.5787525177001953, + "max_kl": 7.300185203552246, + "mean_kl": 2.7990503311157227, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.0762693881988525, + "code_tokens": 24, + "cvar95_kl": 5.875152111053467, + "delta_nll": 1.5155171155929565, + "max_kl": 7.209710121154785, + "mean_kl": 1.5648916959762573, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 4.528422832489014, + "code_tokens": 39, + "cvar95_kl": 7.8542985916137695, + "delta_nll": 3.2140884399414062, + "max_kl": 7.873805999755859, + "mean_kl": 3.5492889881134033, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 4.779646873474121, + "code_tokens": 84, + "cvar95_kl": 9.181705474853516, + "delta_nll": 4.081234455108643, + "max_kl": 10.549416542053223, + "mean_kl": 4.060744762420654, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 3.0870752334594727, + "code_tokens": 32, + "cvar95_kl": 5.042028427124023, + "delta_nll": 2.164307117462158, + "max_kl": 5.140247344970703, + "mean_kl": 1.9686429500579834, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 4.429274082183838, + "code_tokens": 19, + "cvar95_kl": 10.158134460449219, + "delta_nll": 3.119110584259033, + "max_kl": 10.158134460449219, + "mean_kl": 3.122197151184082, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.8681833744049072, + "code_tokens": 50, + "cvar95_kl": 10.293785095214844, + "delta_nll": 3.0502066612243652, + "max_kl": 13.332862854003906, + "mean_kl": 3.1076135635375977, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 2.8356409072875977, + "code_tokens": 71, + "cvar95_kl": 5.595458984375, + "delta_nll": 2.335827350616455, + "max_kl": 6.223273754119873, + "mean_kl": 2.212303400039673, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.3802816867828369 + }, + { + "candidate_nll": 4.693419456481934, + "code_tokens": 28, + "cvar95_kl": 8.955413818359375, + "delta_nll": 3.305450916290283, + "max_kl": 9.179688453674316, + "mean_kl": 3.3770244121551514, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 2.605400562286377, + "code_tokens": 76, + "cvar95_kl": 6.373559951782227, + "delta_nll": 1.736139178276062, + "max_kl": 8.878124237060547, + "mean_kl": 1.7189409732818604, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.5526315569877625 + }, + { + "candidate_nll": 4.1864142417907715, + "code_tokens": 34, + "cvar95_kl": 8.532797813415527, + "delta_nll": 3.2238290309906006, + "max_kl": 8.693868637084961, + "mean_kl": 3.1888413429260254, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 4.037524223327637, + "code_tokens": 11, + "cvar95_kl": 3.2495007514953613, + "delta_nll": 1.8901324272155762, + "max_kl": 3.2495007514953613, + "mean_kl": 1.3615838289260864, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 3.229701280593872, + "code_tokens": 31, + "cvar95_kl": 7.958824157714844, + "delta_nll": 1.8134337663650513, + "max_kl": 9.999542236328125, + "mean_kl": 2.435001850128174, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.4193548262119293 + }, + { + "candidate_nll": 3.120560884475708, + "code_tokens": 118, + "cvar95_kl": 6.402284622192383, + "delta_nll": 2.273784637451172, + "max_kl": 7.890582084655762, + "mean_kl": 2.300467014312744, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.3813559412956238 + }, + { + "candidate_nll": 2.843842029571533, + "code_tokens": 60, + "cvar95_kl": 8.496126174926758, + "delta_nll": 2.068146228790283, + "max_kl": 9.799067497253418, + "mean_kl": 2.330653190612793, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 4.112344264984131, + "code_tokens": 47, + "cvar95_kl": 7.617570877075195, + "delta_nll": 3.2168943881988525, + "max_kl": 7.786830902099609, + "mean_kl": 3.286752223968506, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.25531914830207825 + }, + { + "candidate_nll": 3.7675323486328125, + "code_tokens": 84, + "cvar95_kl": 8.94970703125, + "delta_nll": 2.615518093109131, + "max_kl": 13.214193344116211, + "mean_kl": 2.660922050476074, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.4404761791229248 + }, + { + "candidate_nll": 4.356907844543457, + "code_tokens": 27, + "cvar95_kl": 9.560443878173828, + "delta_nll": 3.362881660461426, + "max_kl": 10.49138069152832, + "mean_kl": 3.3379323482513428, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 3.411135673522949, + "code_tokens": 19, + "cvar95_kl": 7.640441417694092, + "delta_nll": 1.799228549003601, + "max_kl": 7.640441417694092, + "mean_kl": 1.7695822715759277, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.4736842215061188 + }, + { + "candidate_nll": 4.15464973449707, + "code_tokens": 114, + "cvar95_kl": 9.051828384399414, + "delta_nll": 3.3816370964050293, + "max_kl": 10.839929580688477, + "mean_kl": 3.4776699542999268, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 4.9047417640686035, + "code_tokens": 40, + "cvar95_kl": 8.463356971740723, + "delta_nll": 3.1768484115600586, + "max_kl": 9.066914558410645, + "mean_kl": 3.2110133171081543, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 3.948779344558716, + "code_tokens": 25, + "cvar95_kl": 6.837372779846191, + "delta_nll": 2.4631011486053467, + "max_kl": 7.639347076416016, + "mean_kl": 2.389054536819458, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 3.9288339614868164, + "code_tokens": 23, + "cvar95_kl": 8.911283493041992, + "delta_nll": 3.158144950866699, + "max_kl": 8.943307876586914, + "mean_kl": 2.8482892513275146, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 4.342625141143799, + "code_tokens": 77, + "cvar95_kl": 9.538057327270508, + "delta_nll": 3.326275587081909, + "max_kl": 11.16035270690918, + "mean_kl": 3.360565185546875, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.31168830394744873 + }, + { + "candidate_nll": 3.3392844200134277, + "code_tokens": 19, + "cvar95_kl": 7.497058868408203, + "delta_nll": 2.231618881225586, + "max_kl": 7.497058868408203, + "mean_kl": 2.0859572887420654, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 3.89487361907959, + "code_tokens": 37, + "cvar95_kl": 6.812231540679932, + "delta_nll": 2.9116299152374268, + "max_kl": 7.568211555480957, + "mean_kl": 2.918987274169922, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 3.4911439418792725, + "code_tokens": 33, + "cvar95_kl": 7.030263900756836, + "delta_nll": 2.1079790592193604, + "max_kl": 8.132512092590332, + "mean_kl": 2.4782121181488037, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.110599040985107, + "code_tokens": 95, + "cvar95_kl": 10.020315170288086, + "delta_nll": 3.4507575035095215, + "max_kl": 11.439229965209961, + "mean_kl": 3.4095866680145264, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.35789474844932556 + }, + { + "candidate_nll": 3.3819265365600586, + "code_tokens": 125, + "cvar95_kl": 6.5478949546813965, + "delta_nll": 2.3687663078308105, + "max_kl": 8.271309852600098, + "mean_kl": 2.513434648513794, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.35199999809265137 + }, + { + "candidate_nll": 5.129704475402832, + "code_tokens": 168, + "cvar95_kl": 11.871784210205078, + "delta_nll": 4.416656017303467, + "max_kl": 16.96334457397461, + "mean_kl": 4.411320209503174, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.2738095223903656 + }, + { + "candidate_nll": 2.9085190296173096, + "code_tokens": 31, + "cvar95_kl": 11.760282516479492, + "delta_nll": 1.5931168794631958, + "max_kl": 14.549941062927246, + "mean_kl": 2.2277450561523438, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.5806451439857483 + }, + { + "candidate_nll": 3.6504721641540527, + "code_tokens": 83, + "cvar95_kl": 8.39433479309082, + "delta_nll": 2.888765335083008, + "max_kl": 8.987948417663574, + "mean_kl": 2.923570394515991, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.3614457845687866 + }, + { + "candidate_nll": 4.671442985534668, + "code_tokens": 50, + "cvar95_kl": 9.498276710510254, + "delta_nll": 3.9715003967285156, + "max_kl": 11.182723999023438, + "mean_kl": 3.9764084815979004, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 2.9282174110412598, + "code_tokens": 69, + "cvar95_kl": 6.503773212432861, + "delta_nll": 2.561035394668579, + "max_kl": 7.450150012969971, + "mean_kl": 2.6150074005126953, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.36231884360313416 + }, + { + "candidate_nll": 4.368228912353516, + "code_tokens": 50, + "cvar95_kl": 8.894834518432617, + "delta_nll": 3.2833328247070312, + "max_kl": 10.267316818237305, + "mean_kl": 3.4753427505493164, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.4905500411987305, + "code_tokens": 161, + "cvar95_kl": 7.81485652923584, + "delta_nll": 3.4867682456970215, + "max_kl": 8.414484977722168, + "mean_kl": 3.3988115787506104, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.3229813575744629 + }, + { + "candidate_nll": 4.535617351531982, + "code_tokens": 53, + "cvar95_kl": 8.729435920715332, + "delta_nll": 3.8798413276672363, + "max_kl": 9.275604248046875, + "mean_kl": 3.7702438831329346, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 4.248714447021484, + "code_tokens": 97, + "cvar95_kl": 9.93364143371582, + "delta_nll": 3.8031864166259766, + "max_kl": 10.938989639282227, + "mean_kl": 3.5976455211639404, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.39175257086753845 + }, + { + "candidate_nll": 3.084653377532959, + "code_tokens": 22, + "cvar95_kl": 7.4995527267456055, + "delta_nll": 1.7708404064178467, + "max_kl": 9.547601699829102, + "mean_kl": 2.0864760875701904, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 2.5555009841918945, + "code_tokens": 34, + "cvar95_kl": 9.617895126342773, + "delta_nll": 1.6865968704223633, + "max_kl": 11.98992919921875, + "mean_kl": 2.138594150543213, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 4.241522312164307, + "code_tokens": 81, + "cvar95_kl": 8.817001342773438, + "delta_nll": 3.3828790187835693, + "max_kl": 10.241300582885742, + "mean_kl": 3.709165573120117, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.2469135820865631 + }, + { + "candidate_nll": 1.9811334609985352, + "code_tokens": 17, + "cvar95_kl": 3.787569522857666, + "delta_nll": 0.7190301418304443, + "max_kl": 3.787569522857666, + "mean_kl": 0.8510425686836243, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 4.04729700088501, + "code_tokens": 181, + "cvar95_kl": 8.1381254196167, + "delta_nll": 3.2220396995544434, + "max_kl": 10.285239219665527, + "mean_kl": 3.221968650817871, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.32596686482429504 + }, + { + "candidate_nll": 4.380119323730469, + "code_tokens": 63, + "cvar95_kl": 8.004924774169922, + "delta_nll": 3.3997108936309814, + "max_kl": 8.56183910369873, + "mean_kl": 3.4327638149261475, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 1.6961581707000732, + "code_tokens": 35, + "cvar95_kl": 3.961942434310913, + "delta_nll": 1.0026350021362305, + "max_kl": 4.355330944061279, + "mean_kl": 1.220361590385437, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.5142857432365417 + }, + { + "candidate_nll": 3.5334537029266357, + "code_tokens": 25, + "cvar95_kl": 5.616385459899902, + "delta_nll": 2.728666305541992, + "max_kl": 5.871410369873047, + "mean_kl": 2.4216620922088623, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.395860195159912, + "code_tokens": 60, + "cvar95_kl": 8.956561088562012, + "delta_nll": 3.307494640350342, + "max_kl": 9.874591827392578, + "mean_kl": 3.647490978240967, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.406399250030518, + "code_tokens": 86, + "cvar95_kl": 9.250007629394531, + "delta_nll": 3.4947524070739746, + "max_kl": 11.965775489807129, + "mean_kl": 3.4473631381988525, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.3255814015865326 + }, + { + "candidate_nll": 4.173226356506348, + "code_tokens": 40, + "cvar95_kl": 7.676867485046387, + "delta_nll": 3.247060537338257, + "max_kl": 7.746138095855713, + "mean_kl": 3.5496838092803955, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.2757861614227295, + "code_tokens": 27, + "cvar95_kl": 4.721895217895508, + "delta_nll": 1.7122259140014648, + "max_kl": 4.884432792663574, + "mean_kl": 1.8583115339279175, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 3.7252371311187744, + "code_tokens": 52, + "cvar95_kl": 7.034523010253906, + "delta_nll": 2.6356542110443115, + "max_kl": 7.96168851852417, + "mean_kl": 2.9083611965179443, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.2884615361690521 + }, + { + "candidate_nll": 2.934007167816162, + "code_tokens": 71, + "cvar95_kl": 7.911287784576416, + "delta_nll": 2.1921329498291016, + "max_kl": 9.55640697479248, + "mean_kl": 2.3912999629974365, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.39436620473861694 + }, + { + "candidate_nll": 3.3404324054718018, + "code_tokens": 25, + "cvar95_kl": 7.766396522521973, + "delta_nll": 2.2385807037353516, + "max_kl": 7.943704128265381, + "mean_kl": 2.4244368076324463, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 2.3890910148620605, + "code_tokens": 103, + "cvar95_kl": 5.729055404663086, + "delta_nll": 1.7945953607559204, + "max_kl": 8.48880672454834, + "mean_kl": 1.8649879693984985, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.49514561891555786 + }, + { + "candidate_nll": 3.6426243782043457, + "code_tokens": 116, + "cvar95_kl": 7.688932418823242, + "delta_nll": 2.5850863456726074, + "max_kl": 8.32945442199707, + "mean_kl": 2.6183300018310547, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.37068966031074524 + }, + { + "candidate_nll": 3.441427707672119, + "code_tokens": 107, + "cvar95_kl": 7.003168106079102, + "delta_nll": 2.793081521987915, + "max_kl": 9.271197319030762, + "mean_kl": 2.8226356506347656, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.3644859790802002 + }, + { + "candidate_nll": 3.1485018730163574, + "code_tokens": 54, + "cvar95_kl": 5.933460235595703, + "delta_nll": 2.2054641246795654, + "max_kl": 6.246464252471924, + "mean_kl": 2.186356782913208, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.35185185074806213 + }, + { + "candidate_nll": 4.1259870529174805, + "code_tokens": 46, + "cvar95_kl": 8.372899055480957, + "delta_nll": 3.4925947189331055, + "max_kl": 9.721776962280273, + "mean_kl": 3.3028109073638916, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.239130437374115 + }, + { + "candidate_nll": 3.7323126792907715, + "code_tokens": 170, + "cvar95_kl": 8.861197471618652, + "delta_nll": 3.403165817260742, + "max_kl": 14.285181045532227, + "mean_kl": 3.3104865550994873, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.3588235378265381 + }, + { + "candidate_nll": 3.513535261154175, + "code_tokens": 45, + "cvar95_kl": 6.482486724853516, + "delta_nll": 2.0868430137634277, + "max_kl": 7.568443298339844, + "mean_kl": 2.416188955307007, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9668214321136475, + "code_tokens": 24, + "cvar95_kl": 6.027217864990234, + "delta_nll": 2.558518409729004, + "max_kl": 6.157658576965332, + "mean_kl": 2.8667566776275635, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.5934040546417236, + "code_tokens": 78, + "cvar95_kl": 7.083639621734619, + "delta_nll": 2.4314441680908203, + "max_kl": 7.857134819030762, + "mean_kl": 2.4960765838623047, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.43589743971824646 + }, + { + "candidate_nll": 3.8650009632110596, + "code_tokens": 33, + "cvar95_kl": 7.632802963256836, + "delta_nll": 3.2344439029693604, + "max_kl": 7.829099178314209, + "mean_kl": 3.087615966796875, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.545436859130859, + "code_tokens": 32, + "cvar95_kl": 7.731673240661621, + "delta_nll": 3.42976713180542, + "max_kl": 8.387221336364746, + "mean_kl": 3.3554153442382812, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 4.868621349334717, + "code_tokens": 77, + "cvar95_kl": 8.456477165222168, + "delta_nll": 3.8798716068267822, + "max_kl": 9.227700233459473, + "mean_kl": 3.764155387878418, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.19480518996715546 + }, + { + "candidate_nll": 1.9431709051132202, + "code_tokens": 222, + "cvar95_kl": 6.334272861480713, + "delta_nll": 1.5209202766418457, + "max_kl": 9.031747817993164, + "mean_kl": 1.6066526174545288, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.6036036014556885 + }, + { + "candidate_nll": 3.9161603450775146, + "code_tokens": 86, + "cvar95_kl": 8.051687240600586, + "delta_nll": 2.867565631866455, + "max_kl": 9.388579368591309, + "mean_kl": 2.919541358947754, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.3255814015865326 + }, + { + "candidate_nll": 2.543691873550415, + "code_tokens": 31, + "cvar95_kl": 7.297207832336426, + "delta_nll": 2.2096424102783203, + "max_kl": 8.315489768981934, + "mean_kl": 2.2118918895721436, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.5483871102333069 + }, + { + "candidate_nll": 3.170351266860962, + "code_tokens": 34, + "cvar95_kl": 7.073329925537109, + "delta_nll": 2.6587512493133545, + "max_kl": 7.992109775543213, + "mean_kl": 2.6797683238983154, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 3.917029619216919, + "code_tokens": 38, + "cvar95_kl": 6.8988471031188965, + "delta_nll": 2.529569149017334, + "max_kl": 7.126597881317139, + "mean_kl": 2.6202237606048584, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 3.5316457748413086, + "code_tokens": 93, + "cvar95_kl": 7.706381797790527, + "delta_nll": 2.568601131439209, + "max_kl": 9.159712791442871, + "mean_kl": 2.619619131088257, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.40860214829444885 + }, + { + "candidate_nll": 4.307596683502197, + "code_tokens": 70, + "cvar95_kl": 8.647835731506348, + "delta_nll": 3.5289909839630127, + "max_kl": 10.939667701721191, + "mean_kl": 3.5024640560150146, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.863576412200928, + "code_tokens": 67, + "cvar95_kl": 9.315767288208008, + "delta_nll": 4.053084373474121, + "max_kl": 9.505389213562012, + "mean_kl": 4.078247547149658, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.23880596458911896 + }, + { + "candidate_nll": 3.795036554336548, + "code_tokens": 66, + "cvar95_kl": 6.880446434020996, + "delta_nll": 2.508909225463867, + "max_kl": 8.252176284790039, + "mean_kl": 2.4648380279541016, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.3787878751754761 + }, + { + "candidate_nll": 3.843451976776123, + "code_tokens": 34, + "cvar95_kl": 5.730145454406738, + "delta_nll": 2.831157684326172, + "max_kl": 5.922737121582031, + "mean_kl": 2.857491970062256, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 3.2991018295288086, + "code_tokens": 64, + "cvar95_kl": 7.117890357971191, + "delta_nll": 2.3890764713287354, + "max_kl": 8.028105735778809, + "mean_kl": 2.494619607925415, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.6313648223876953, + "code_tokens": 56, + "cvar95_kl": 5.885019302368164, + "delta_nll": 2.3210091590881348, + "max_kl": 6.186561107635498, + "mean_kl": 2.5534610748291016, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.1901204586029053, + "code_tokens": 21, + "cvar95_kl": 5.082888603210449, + "delta_nll": 2.0650382041931152, + "max_kl": 5.483434677124023, + "mean_kl": 2.194352626800537, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.5586791038513184, + "code_tokens": 60, + "cvar95_kl": 7.697170257568359, + "delta_nll": 2.893338680267334, + "max_kl": 8.01723575592041, + "mean_kl": 2.966423988342285, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.216549396514893, + "code_tokens": 29, + "cvar95_kl": 8.150657653808594, + "delta_nll": 2.8627734184265137, + "max_kl": 8.158995628356934, + "mean_kl": 2.7058210372924805, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 5.561500072479248, + "code_tokens": 34, + "cvar95_kl": 10.6506986618042, + "delta_nll": 4.8394551277160645, + "max_kl": 10.65950870513916, + "mean_kl": 4.840099334716797, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 4.814663887023926, + "code_tokens": 88, + "cvar95_kl": 10.505480766296387, + "delta_nll": 3.8955564498901367, + "max_kl": 12.19345474243164, + "mean_kl": 4.048790454864502, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 2.445730209350586, + "code_tokens": 17, + "cvar95_kl": 5.962122440338135, + "delta_nll": 1.9025065898895264, + "max_kl": 5.962122440338135, + "mean_kl": 1.7500712871551514, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.529411792755127 + }, + { + "candidate_nll": 3.745011568069458, + "code_tokens": 35, + "cvar95_kl": 4.670696258544922, + "delta_nll": 1.8544644117355347, + "max_kl": 5.710427761077881, + "mean_kl": 1.9720380306243896, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 3.0036122798919678, + "code_tokens": 48, + "cvar95_kl": 5.625959396362305, + "delta_nll": 2.1138358116149902, + "max_kl": 6.324275493621826, + "mean_kl": 2.180941343307495, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.6546835899353027, + "code_tokens": 130, + "cvar95_kl": 7.099216938018799, + "delta_nll": 2.6644110679626465, + "max_kl": 8.616958618164062, + "mean_kl": 2.63737154006958, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.4076923131942749 + }, + { + "candidate_nll": 5.1502885818481445, + "code_tokens": 65, + "cvar95_kl": 8.719244956970215, + "delta_nll": 4.055349349975586, + "max_kl": 9.243741035461426, + "mean_kl": 4.1994709968566895, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 4.506704330444336, + "code_tokens": 145, + "cvar95_kl": 10.46946907043457, + "delta_nll": 3.6316893100738525, + "max_kl": 12.085600852966309, + "mean_kl": 3.6629176139831543, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.29655173420906067 + }, + { + "candidate_nll": 2.8146510124206543, + "code_tokens": 80, + "cvar95_kl": 7.542943477630615, + "delta_nll": 2.098951816558838, + "max_kl": 10.02967643737793, + "mean_kl": 2.052034378051758, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.5249999761581421 + }, + { + "candidate_nll": 3.7483925819396973, + "code_tokens": 70, + "cvar95_kl": 8.017694473266602, + "delta_nll": 3.1662557125091553, + "max_kl": 9.384512901306152, + "mean_kl": 3.359759569168091, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.0431928634643555, + "code_tokens": 25, + "cvar95_kl": 6.595137596130371, + "delta_nll": 2.554903984069824, + "max_kl": 6.713994979858398, + "mean_kl": 2.69447660446167, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.5364127159118652, + "code_tokens": 50, + "cvar95_kl": 6.4982428550720215, + "delta_nll": 1.8820292949676514, + "max_kl": 7.190588474273682, + "mean_kl": 2.463148593902588, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 3.456355094909668, + "code_tokens": 48, + "cvar95_kl": 6.533738613128662, + "delta_nll": 2.6583120822906494, + "max_kl": 7.051448822021484, + "mean_kl": 2.727060079574585, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.358283758163452, + "code_tokens": 104, + "cvar95_kl": 10.020508766174316, + "delta_nll": 3.014211416244507, + "max_kl": 12.173360824584961, + "mean_kl": 2.9106686115264893, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.45192307233810425 + }, + { + "candidate_nll": 3.5162296295166016, + "code_tokens": 68, + "cvar95_kl": 5.539032459259033, + "delta_nll": 2.526235342025757, + "max_kl": 6.174436092376709, + "mean_kl": 2.5384726524353027, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 4.008400917053223, + "code_tokens": 145, + "cvar95_kl": 7.546788215637207, + "delta_nll": 3.253488779067993, + "max_kl": 8.635436058044434, + "mean_kl": 3.205841541290283, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.33103448152542114 + }, + { + "candidate_nll": 2.5855178833007812, + "code_tokens": 41, + "cvar95_kl": 6.3692169189453125, + "delta_nll": 1.7406301498413086, + "max_kl": 9.056057929992676, + "mean_kl": 1.8905287981033325, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.4146341383457184 + }, + { + "candidate_nll": 2.4019360542297363, + "code_tokens": 53, + "cvar95_kl": 5.26568078994751, + "delta_nll": 1.6408138275146484, + "max_kl": 6.3756914138793945, + "mean_kl": 1.5155529975891113, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.6226415038108826 + }, + { + "candidate_nll": 4.3968586921691895, + "code_tokens": 58, + "cvar95_kl": 9.31470012664795, + "delta_nll": 3.6046290397644043, + "max_kl": 10.053613662719727, + "mean_kl": 3.562600612640381, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 3.8363687992095947, + "code_tokens": 24, + "cvar95_kl": 8.519122123718262, + "delta_nll": 2.104308605194092, + "max_kl": 9.622902870178223, + "mean_kl": 2.4314630031585693, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.3778750896453857, + "code_tokens": 89, + "cvar95_kl": 9.059659957885742, + "delta_nll": 2.386021137237549, + "max_kl": 10.464463233947754, + "mean_kl": 2.679948329925537, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.43820226192474365 + }, + { + "candidate_nll": 3.761221408843994, + "code_tokens": 101, + "cvar95_kl": 8.417855262756348, + "delta_nll": 3.2761452198028564, + "max_kl": 10.57144832611084, + "mean_kl": 3.2960147857666016, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.3366336524486542 + }, + { + "candidate_nll": 4.742405414581299, + "code_tokens": 33, + "cvar95_kl": 8.115131378173828, + "delta_nll": 3.2848477363586426, + "max_kl": 8.865796089172363, + "mean_kl": 3.5107390880584717, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 3.939426898956299, + "code_tokens": 50, + "cvar95_kl": 9.488926887512207, + "delta_nll": 2.89276123046875, + "max_kl": 11.048735618591309, + "mean_kl": 3.11356782913208, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 4.091323375701904, + "code_tokens": 58, + "cvar95_kl": 7.597135543823242, + "delta_nll": 2.682570457458496, + "max_kl": 9.167119979858398, + "mean_kl": 2.800201177597046, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.2586206793785095 + }, + { + "candidate_nll": 3.733919858932495, + "code_tokens": 62, + "cvar95_kl": 7.750706672668457, + "delta_nll": 2.754760980606079, + "max_kl": 9.829529762268066, + "mean_kl": 2.8233768939971924, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 3.464540481567383, + "code_tokens": 42, + "cvar95_kl": 7.918123722076416, + "delta_nll": 2.772850513458252, + "max_kl": 10.787105560302734, + "mean_kl": 2.8694355487823486, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.428173303604126, + "code_tokens": 15, + "cvar95_kl": 8.408536911010742, + "delta_nll": 2.503432273864746, + "max_kl": 8.408536911010742, + "mean_kl": 2.372941255569458, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 3.974079132080078, + "code_tokens": 57, + "cvar95_kl": 7.864706516265869, + "delta_nll": 3.2352969646453857, + "max_kl": 8.976761817932129, + "mean_kl": 3.29451847076416, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.2982456088066101 + }, + { + "candidate_nll": 3.794093608856201, + "code_tokens": 58, + "cvar95_kl": 6.430487155914307, + "delta_nll": 2.7505576610565186, + "max_kl": 6.882565975189209, + "mean_kl": 2.717407703399658, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.32758620381355286 + }, + { + "candidate_nll": 4.210790157318115, + "code_tokens": 45, + "cvar95_kl": 8.430335998535156, + "delta_nll": 3.3388190269470215, + "max_kl": 11.073572158813477, + "mean_kl": 3.1634256839752197, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.31111112236976624 + }, + { + "candidate_nll": 4.587059497833252, + "code_tokens": 32, + "cvar95_kl": 9.105957984924316, + "delta_nll": 2.5578291416168213, + "max_kl": 9.186572074890137, + "mean_kl": 2.969259023666382, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 4.176034927368164, + "code_tokens": 39, + "cvar95_kl": 10.467733383178711, + "delta_nll": 2.8553686141967773, + "max_kl": 12.46886157989502, + "mean_kl": 3.2741754055023193, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.03128719329834, + "code_tokens": 136, + "cvar95_kl": 7.54624080657959, + "delta_nll": 2.62052583694458, + "max_kl": 10.528792381286621, + "mean_kl": 2.5103399753570557, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.3970588147640228 + }, + { + "candidate_nll": 4.830615997314453, + "code_tokens": 50, + "cvar95_kl": 11.216290473937988, + "delta_nll": 3.6098594665527344, + "max_kl": 13.253210067749023, + "mean_kl": 3.5952277183532715, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 3.505692481994629, + "code_tokens": 34, + "cvar95_kl": 7.151232719421387, + "delta_nll": 2.6397767066955566, + "max_kl": 7.796603679656982, + "mean_kl": 2.6122665405273438, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 3.948845148086548, + "code_tokens": 44, + "cvar95_kl": 10.075748443603516, + "delta_nll": 2.770054817199707, + "max_kl": 13.82807731628418, + "mean_kl": 2.9971582889556885, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 2.91092586517334, + "code_tokens": 36, + "cvar95_kl": 4.4540696144104, + "delta_nll": 1.477504014968872, + "max_kl": 4.558900833129883, + "mean_kl": 1.743957281112671, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.219226837158203, + "code_tokens": 46, + "cvar95_kl": 8.483409881591797, + "delta_nll": 3.416069507598877, + "max_kl": 9.954256057739258, + "mean_kl": 3.444394588470459, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.28260868787765503 + }, + { + "candidate_nll": 4.148343086242676, + "code_tokens": 48, + "cvar95_kl": 10.036842346191406, + "delta_nll": 2.968003273010254, + "max_kl": 10.871546745300293, + "mean_kl": 3.3295834064483643, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.705350875854492, + "code_tokens": 25, + "cvar95_kl": 7.825708866119385, + "delta_nll": 2.8099398612976074, + "max_kl": 9.175088882446289, + "mean_kl": 2.543952465057373, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.5583343505859375, + "code_tokens": 69, + "cvar95_kl": 8.680874824523926, + "delta_nll": 3.0540552139282227, + "max_kl": 10.316757202148438, + "mean_kl": 3.292302131652832, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.24637681245803833 + }, + { + "candidate_nll": 3.6658172607421875, + "code_tokens": 60, + "cvar95_kl": 7.41693639755249, + "delta_nll": 2.700359582901001, + "max_kl": 9.094948768615723, + "mean_kl": 2.982229471206665, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.7148044109344482, + "code_tokens": 49, + "cvar95_kl": 8.266252517700195, + "delta_nll": 2.8003268241882324, + "max_kl": 13.496810913085938, + "mean_kl": 2.7804577350616455, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.3469387888908386 + }, + { + "candidate_nll": 3.7294609546661377, + "code_tokens": 44, + "cvar95_kl": 8.898777961730957, + "delta_nll": 2.028932571411133, + "max_kl": 10.923503875732422, + "mean_kl": 2.08388614654541, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.5227272510528564 + }, + { + "candidate_nll": 3.751727819442749, + "code_tokens": 81, + "cvar95_kl": 7.830969333648682, + "delta_nll": 2.606313705444336, + "max_kl": 9.86623764038086, + "mean_kl": 2.656686782836914, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.34567901492118835 + }, + { + "candidate_nll": 4.649376392364502, + "code_tokens": 26, + "cvar95_kl": 8.76923942565918, + "delta_nll": 3.2126412391662598, + "max_kl": 10.861197471618652, + "mean_kl": 3.121267557144165, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.9570059776306152, + "code_tokens": 32, + "cvar95_kl": 8.94990348815918, + "delta_nll": 3.077085018157959, + "max_kl": 9.318476676940918, + "mean_kl": 3.404103994369507, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 4.495823860168457, + "code_tokens": 48, + "cvar95_kl": 8.342767715454102, + "delta_nll": 3.548081874847412, + "max_kl": 9.234992980957031, + "mean_kl": 3.4835989475250244, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 3.580782413482666, + "code_tokens": 180, + "cvar95_kl": 8.369616508483887, + "delta_nll": 2.8814611434936523, + "max_kl": 10.316535949707031, + "mean_kl": 2.8775696754455566, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.7583916187286377, + "code_tokens": 38, + "cvar95_kl": 7.711099147796631, + "delta_nll": 3.036630153656006, + "max_kl": 7.863528251647949, + "mean_kl": 3.15216064453125, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 3.494119167327881, + "code_tokens": 90, + "cvar95_kl": 8.912022590637207, + "delta_nll": 2.7990574836730957, + "max_kl": 10.3503999710083, + "mean_kl": 2.840824842453003, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.35555556416511536 + }, + { + "candidate_nll": 4.185794830322266, + "code_tokens": 44, + "cvar95_kl": 8.075642585754395, + "delta_nll": 3.6111648082733154, + "max_kl": 8.176257133483887, + "mean_kl": 3.455526828765869, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.34090909361839294 + }, + { + "candidate_nll": 2.7201764583587646, + "code_tokens": 60, + "cvar95_kl": 6.0188140869140625, + "delta_nll": 1.9762762784957886, + "max_kl": 7.444685459136963, + "mean_kl": 2.1088457107543945, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.621342658996582, + "code_tokens": 69, + "cvar95_kl": 6.934355735778809, + "delta_nll": 2.618544578552246, + "max_kl": 8.948725700378418, + "mean_kl": 2.5402939319610596, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.37681159377098083 + }, + { + "candidate_nll": 3.2073442935943604, + "code_tokens": 71, + "cvar95_kl": 7.3817315101623535, + "delta_nll": 2.6897826194763184, + "max_kl": 9.649024963378906, + "mean_kl": 2.822244882583618, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.3802816867828369 + }, + { + "candidate_nll": 2.7739381790161133, + "code_tokens": 22, + "cvar95_kl": 4.3464250564575195, + "delta_nll": 1.5312650203704834, + "max_kl": 4.5338616371154785, + "mean_kl": 1.498108983039856, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 2.1388208866119385, + "code_tokens": 215, + "cvar95_kl": 5.807420253753662, + "delta_nll": 1.5850414037704468, + "max_kl": 6.997461795806885, + "mean_kl": 1.6443361043930054, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.5348837375640869 + }, + { + "candidate_nll": 4.405669212341309, + "code_tokens": 26, + "cvar95_kl": 8.47658920288086, + "delta_nll": 3.302879571914673, + "max_kl": 8.915496826171875, + "mean_kl": 3.0582377910614014, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.816885471343994, + "code_tokens": 33, + "cvar95_kl": 7.3485517501831055, + "delta_nll": 2.948638916015625, + "max_kl": 8.149500846862793, + "mean_kl": 2.8772668838500977, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 3.0335144996643066, + "code_tokens": 18, + "cvar95_kl": 6.743980884552002, + "delta_nll": 2.0973663330078125, + "max_kl": 6.743980884552002, + "mean_kl": 2.3112173080444336, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.877918720245361, + "code_tokens": 71, + "cvar95_kl": 9.484857559204102, + "delta_nll": 3.5978569984436035, + "max_kl": 10.840962409973145, + "mean_kl": 3.7738096714019775, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.3380281627178192 + }, + { + "candidate_nll": 3.39499831199646, + "code_tokens": 107, + "cvar95_kl": 6.050165176391602, + "delta_nll": 2.6135969161987305, + "max_kl": 6.481265068054199, + "mean_kl": 2.5661065578460693, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.38317757844924927 + }, + { + "candidate_nll": 4.769425392150879, + "code_tokens": 40, + "cvar95_kl": 9.648136138916016, + "delta_nll": 2.7144007682800293, + "max_kl": 9.868269920349121, + "mean_kl": 3.2190845012664795, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.557016372680664, + "code_tokens": 32, + "cvar95_kl": 7.5413408279418945, + "delta_nll": 2.1927194595336914, + "max_kl": 8.963384628295898, + "mean_kl": 2.4257476329803467, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.627199649810791, + "code_tokens": 130, + "cvar95_kl": 7.528897285461426, + "delta_nll": 2.927945852279663, + "max_kl": 11.442055702209473, + "mean_kl": 3.0085995197296143, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.3692307770252228 + }, + { + "candidate_nll": 3.0222277641296387, + "code_tokens": 27, + "cvar95_kl": 4.7468976974487305, + "delta_nll": 1.5508060455322266, + "max_kl": 5.20893669128418, + "mean_kl": 1.9552397727966309, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 3.110520124435425, + "code_tokens": 42, + "cvar95_kl": 6.6008381843566895, + "delta_nll": 2.137579917907715, + "max_kl": 6.933841228485107, + "mean_kl": 2.3204240798950195, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.4523809552192688 + }, + { + "candidate_nll": 3.9863107204437256, + "code_tokens": 30, + "cvar95_kl": 6.732073783874512, + "delta_nll": 3.240797519683838, + "max_kl": 7.972780227661133, + "mean_kl": 3.155766248703003, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.4197282791137695, + "code_tokens": 58, + "cvar95_kl": 10.052996635437012, + "delta_nll": 4.000495433807373, + "max_kl": 11.302114486694336, + "mean_kl": 3.947944402694702, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 3.9213805198669434, + "code_tokens": 34, + "cvar95_kl": 7.819709300994873, + "delta_nll": 2.6911532878875732, + "max_kl": 8.624363899230957, + "mean_kl": 2.564340353012085, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 2.587153911590576, + "code_tokens": 208, + "cvar95_kl": 6.228973388671875, + "delta_nll": 1.8856914043426514, + "max_kl": 7.722765922546387, + "mean_kl": 1.9609217643737793, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.4951923191547394 + }, + { + "candidate_nll": 3.3145711421966553, + "code_tokens": 83, + "cvar95_kl": 6.266745567321777, + "delta_nll": 2.4156653881073, + "max_kl": 6.9935102462768555, + "mean_kl": 2.4428820610046387, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.34939759969711304 + }, + { + "candidate_nll": 3.416029214859009, + "code_tokens": 170, + "cvar95_kl": 8.30118465423584, + "delta_nll": 2.8934736251831055, + "max_kl": 10.069465637207031, + "mean_kl": 2.8860487937927246, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.3588235378265381 + }, + { + "candidate_nll": 2.9022092819213867, + "code_tokens": 40, + "cvar95_kl": 7.155121803283691, + "delta_nll": 2.120788812637329, + "max_kl": 7.679198265075684, + "mean_kl": 2.318382740020752, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.4749999940395355 + }, + { + "candidate_nll": 2.8925251960754395, + "code_tokens": 31, + "cvar95_kl": 6.549638748168945, + "delta_nll": 1.6458156108856201, + "max_kl": 6.734190940856934, + "mean_kl": 1.8406556844711304, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.5806451439857483 + }, + { + "candidate_nll": 3.510512113571167, + "code_tokens": 42, + "cvar95_kl": 7.420232772827148, + "delta_nll": 2.7157421112060547, + "max_kl": 9.22470760345459, + "mean_kl": 2.9088664054870605, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.3145432472229, + "code_tokens": 27, + "cvar95_kl": 7.07840633392334, + "delta_nll": 2.5454773902893066, + "max_kl": 8.076467514038086, + "mean_kl": 3.1192562580108643, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 3.1728692054748535, + "code_tokens": 25, + "cvar95_kl": 5.791269302368164, + "delta_nll": 0.9730587005615234, + "max_kl": 6.637829780578613, + "mean_kl": 1.527357816696167, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 4.533588409423828, + "code_tokens": 40, + "cvar95_kl": 9.462213516235352, + "delta_nll": 3.9652018547058105, + "max_kl": 9.495834350585938, + "mean_kl": 4.001735687255859, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.628875732421875, + "code_tokens": 24, + "cvar95_kl": 6.140568256378174, + "delta_nll": 2.244521141052246, + "max_kl": 7.515410423278809, + "mean_kl": 2.11141037940979, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.5125062465667725, + "code_tokens": 28, + "cvar95_kl": 5.978395462036133, + "delta_nll": 2.629761219024658, + "max_kl": 6.046397686004639, + "mean_kl": 2.558197498321533, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.7217445373535156, + "code_tokens": 71, + "cvar95_kl": 8.32164478302002, + "delta_nll": 3.24288010597229, + "max_kl": 8.665556907653809, + "mean_kl": 3.091193199157715, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.28169015049934387 + }, + { + "candidate_nll": 2.9918339252471924, + "code_tokens": 24, + "cvar95_kl": 5.967455863952637, + "delta_nll": 2.462473154067993, + "max_kl": 6.062401294708252, + "mean_kl": 2.2539970874786377, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.90136981010437, + "code_tokens": 34, + "cvar95_kl": 11.711913108825684, + "delta_nll": 2.629788875579834, + "max_kl": 15.894153594970703, + "mean_kl": 3.314577341079712, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 4.06912088394165, + "code_tokens": 64, + "cvar95_kl": 8.398988723754883, + "delta_nll": 2.949826955795288, + "max_kl": 9.092184066772461, + "mean_kl": 3.370577335357666, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.296875 + }, + { + "candidate_nll": 2.4627630710601807, + "code_tokens": 45, + "cvar95_kl": 5.327789783477783, + "delta_nll": 1.756171703338623, + "max_kl": 6.697144985198975, + "mean_kl": 1.724286437034607, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.5777778029441833 + }, + { + "candidate_nll": 4.244382381439209, + "code_tokens": 48, + "cvar95_kl": 9.216679573059082, + "delta_nll": 3.04567813873291, + "max_kl": 11.03330135345459, + "mean_kl": 3.4016313552856445, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.961827278137207, + "code_tokens": 21, + "cvar95_kl": 6.676607608795166, + "delta_nll": 2.2616262435913086, + "max_kl": 6.729164123535156, + "mean_kl": 2.351311445236206, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.572000741958618, + "code_tokens": 24, + "cvar95_kl": 9.46061897277832, + "delta_nll": 2.817586660385132, + "max_kl": 11.015374183654785, + "mean_kl": 3.1406078338623047, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9371817111968994, + "code_tokens": 63, + "cvar95_kl": 6.393807411193848, + "delta_nll": 2.3652544021606445, + "max_kl": 6.961467742919922, + "mean_kl": 2.4342525005340576, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.3968254029750824 + }, + { + "candidate_nll": 5.412199974060059, + "code_tokens": 22, + "cvar95_kl": 9.228409767150879, + "delta_nll": 4.133440971374512, + "max_kl": 10.825698852539062, + "mean_kl": 3.996243953704834, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 3.3191421031951904, + "code_tokens": 25, + "cvar95_kl": 6.444607734680176, + "delta_nll": 2.1904797554016113, + "max_kl": 7.091843128204346, + "mean_kl": 2.4929943084716797, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.1289873123168945, + "code_tokens": 21, + "cvar95_kl": 5.421415328979492, + "delta_nll": 1.1686615943908691, + "max_kl": 6.086781024932861, + "mean_kl": 1.6518672704696655, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.523809552192688 + }, + { + "candidate_nll": 2.9180004596710205, + "code_tokens": 24, + "cvar95_kl": 4.173370838165283, + "delta_nll": 1.451668381690979, + "max_kl": 4.3799262046813965, + "mean_kl": 1.5894056558609009, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 5.340699672698975, + "code_tokens": 15, + "cvar95_kl": 10.361346244812012, + "delta_nll": 3.5472118854522705, + "max_kl": 10.361346244812012, + "mean_kl": 3.639540433883667, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.857062578201294, + "code_tokens": 59, + "cvar95_kl": 8.401025772094727, + "delta_nll": 3.0762062072753906, + "max_kl": 9.269640922546387, + "mean_kl": 2.9882190227508545, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.2881355881690979 + }, + { + "candidate_nll": 3.7222979068756104, + "code_tokens": 54, + "cvar95_kl": 7.73170804977417, + "delta_nll": 2.788179636001587, + "max_kl": 8.136899948120117, + "mean_kl": 2.9561476707458496, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.928691387176514, + "code_tokens": 28, + "cvar95_kl": 11.21743106842041, + "delta_nll": 2.976832389831543, + "max_kl": 15.113962173461914, + "mean_kl": 3.4653894901275635, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.340106725692749, + "code_tokens": 46, + "cvar95_kl": 6.707757949829102, + "delta_nll": 2.255380630493164, + "max_kl": 7.7443437576293945, + "mean_kl": 2.503124952316284, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 2.4066531658172607, + "code_tokens": 27, + "cvar95_kl": 4.3115997314453125, + "delta_nll": 0.8918308019638062, + "max_kl": 4.368620872497559, + "mean_kl": 1.4688606262207031, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 4.061816215515137, + "code_tokens": 97, + "cvar95_kl": 7.044244289398193, + "delta_nll": 2.8043856620788574, + "max_kl": 9.193522453308105, + "mean_kl": 2.7458624839782715, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.3505154550075531 + }, + { + "candidate_nll": 4.17683744430542, + "code_tokens": 24, + "cvar95_kl": 7.340678691864014, + "delta_nll": 3.3746049404144287, + "max_kl": 7.397850513458252, + "mean_kl": 3.256829023361206, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 4.638741493225098, + "code_tokens": 30, + "cvar95_kl": 8.69499397277832, + "delta_nll": 3.3735928535461426, + "max_kl": 9.048994064331055, + "mean_kl": 3.7171666622161865, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.023921966552734, + "code_tokens": 38, + "cvar95_kl": 8.473710060119629, + "delta_nll": 2.5908522605895996, + "max_kl": 8.887125015258789, + "mean_kl": 2.8157119750976562, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 2.487950325012207, + "code_tokens": 21, + "cvar95_kl": 3.3869502544403076, + "delta_nll": 1.0001713037490845, + "max_kl": 4.552275657653809, + "mean_kl": 1.0700349807739258, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 4.692441463470459, + "code_tokens": 65, + "cvar95_kl": 8.035333633422852, + "delta_nll": 4.155479431152344, + "max_kl": 8.219464302062988, + "mean_kl": 4.121510982513428, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 2.8718414306640625, + "code_tokens": 27, + "cvar95_kl": 7.106766700744629, + "delta_nll": 2.087315082550049, + "max_kl": 7.852028846740723, + "mean_kl": 2.194620370864868, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.5185185074806213 + }, + { + "candidate_nll": 3.273282051086426, + "code_tokens": 35, + "cvar95_kl": 6.7131547927856445, + "delta_nll": 2.5494658946990967, + "max_kl": 7.1769843101501465, + "mean_kl": 2.4815635681152344, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.6967484951019287, + "code_tokens": 75, + "cvar95_kl": 7.75349235534668, + "delta_nll": 2.7722244262695312, + "max_kl": 8.503680229187012, + "mean_kl": 2.7987215518951416, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.240535736083984, + "code_tokens": 25, + "cvar95_kl": 6.785511016845703, + "delta_nll": 2.5988001823425293, + "max_kl": 6.951158046722412, + "mean_kl": 3.1780428886413574, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 3.517472505569458, + "code_tokens": 52, + "cvar95_kl": 6.169475555419922, + "delta_nll": 2.5934834480285645, + "max_kl": 6.294086456298828, + "mean_kl": 2.5967979431152344, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.785571813583374, + "code_tokens": 110, + "cvar95_kl": 9.35903263092041, + "delta_nll": 3.1749117374420166, + "max_kl": 10.968042373657227, + "mean_kl": 3.286620855331421, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.3272727131843567 + }, + { + "candidate_nll": 3.1330697536468506, + "code_tokens": 25, + "cvar95_kl": 6.927232265472412, + "delta_nll": 1.9002478122711182, + "max_kl": 6.949608325958252, + "mean_kl": 2.1568431854248047, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.6519131660461426, + "code_tokens": 56, + "cvar95_kl": 5.473778247833252, + "delta_nll": 2.634927272796631, + "max_kl": 5.884529113769531, + "mean_kl": 2.36083984375, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.607174396514893, + "code_tokens": 108, + "cvar95_kl": 6.693226337432861, + "delta_nll": 3.05332612991333, + "max_kl": 8.139405250549316, + "mean_kl": 3.024751901626587, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.23148147761821747 + }, + { + "candidate_nll": 3.816936492919922, + "code_tokens": 47, + "cvar95_kl": 6.780214786529541, + "delta_nll": 2.4451966285705566, + "max_kl": 7.060251712799072, + "mean_kl": 2.904417037963867, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 3.867399215698242, + "code_tokens": 102, + "cvar95_kl": 8.409201622009277, + "delta_nll": 2.868990659713745, + "max_kl": 9.267026901245117, + "mean_kl": 3.0304906368255615, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.36274510622024536 + }, + { + "candidate_nll": 2.449172258377075, + "code_tokens": 18, + "cvar95_kl": 4.304207801818848, + "delta_nll": 0.8235466480255127, + "max_kl": 4.304207801818848, + "mean_kl": 1.2002654075622559, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 4.575943946838379, + "code_tokens": 28, + "cvar95_kl": 8.842262268066406, + "delta_nll": 3.4762797355651855, + "max_kl": 9.346183776855469, + "mean_kl": 3.3634085655212402, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.42509651184082, + "code_tokens": 30, + "cvar95_kl": 8.579370498657227, + "delta_nll": 2.8816957473754883, + "max_kl": 8.8291015625, + "mean_kl": 2.922745704650879, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.6258811950683594, + "code_tokens": 48, + "cvar95_kl": 8.601994514465332, + "delta_nll": 3.150764226913452, + "max_kl": 14.13879108428955, + "mean_kl": 3.1368274688720703, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.463285207748413, + "code_tokens": 59, + "cvar95_kl": 8.075398445129395, + "delta_nll": 2.9262702465057373, + "max_kl": 9.042973518371582, + "mean_kl": 3.006894111633301, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.38983049988746643 + }, + { + "candidate_nll": 3.721503734588623, + "code_tokens": 102, + "cvar95_kl": 9.11414623260498, + "delta_nll": 2.679309844970703, + "max_kl": 12.910987854003906, + "mean_kl": 2.7947518825531006, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.3921568691730499 + }, + { + "candidate_nll": 3.6146926879882812, + "code_tokens": 36, + "cvar95_kl": 7.365846633911133, + "delta_nll": 2.8743271827697754, + "max_kl": 7.866745948791504, + "mean_kl": 2.8635153770446777, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.3055555522441864 + }, + { + "candidate_nll": 4.682724475860596, + "code_tokens": 51, + "cvar95_kl": 9.045112609863281, + "delta_nll": 3.793792963027954, + "max_kl": 9.344708442687988, + "mean_kl": 3.629426956176758, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.2549019753932953 + }, + { + "candidate_nll": 3.701286554336548, + "code_tokens": 62, + "cvar95_kl": 7.2578349113464355, + "delta_nll": 2.4556078910827637, + "max_kl": 10.096802711486816, + "mean_kl": 2.8017680644989014, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 4.399543762207031, + "code_tokens": 39, + "cvar95_kl": 12.550058364868164, + "delta_nll": 3.8232617378234863, + "max_kl": 13.174121856689453, + "mean_kl": 3.731145143508911, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 3.250718832015991, + "code_tokens": 76, + "cvar95_kl": 7.8104352951049805, + "delta_nll": 2.5110344886779785, + "max_kl": 10.168678283691406, + "mean_kl": 2.536100149154663, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.3815789520740509 + }, + { + "candidate_nll": 5.211346626281738, + "code_tokens": 58, + "cvar95_kl": 8.751136779785156, + "delta_nll": 3.706695556640625, + "max_kl": 9.58469009399414, + "mean_kl": 3.758669137954712, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.18965516984462738 + }, + { + "candidate_nll": 3.6235334873199463, + "code_tokens": 93, + "cvar95_kl": 7.7349138259887695, + "delta_nll": 2.496138095855713, + "max_kl": 10.724502563476562, + "mean_kl": 2.3727011680603027, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.3655914068222046 + }, + { + "candidate_nll": 5.379446506500244, + "code_tokens": 45, + "cvar95_kl": 11.058159828186035, + "delta_nll": 4.024200916290283, + "max_kl": 12.993038177490234, + "mean_kl": 4.295466899871826, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 1.8507490158081055, + "code_tokens": 209, + "cvar95_kl": 5.21623420715332, + "delta_nll": 1.2937736511230469, + "max_kl": 6.580127716064453, + "mean_kl": 1.4802192449569702, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 3.4344024658203125, + "code_tokens": 46, + "cvar95_kl": 5.96929931640625, + "delta_nll": 2.4659347534179688, + "max_kl": 6.470644950866699, + "mean_kl": 2.1613988876342773, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.5652173757553101 + }, + { + "candidate_nll": 4.393549919128418, + "code_tokens": 39, + "cvar95_kl": 8.726612091064453, + "delta_nll": 3.672762870788574, + "max_kl": 8.767369270324707, + "mean_kl": 3.532088041305542, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 2.4108996391296387, + "code_tokens": 25, + "cvar95_kl": 4.591597080230713, + "delta_nll": 1.4692041873931885, + "max_kl": 5.23858118057251, + "mean_kl": 1.4417450428009033, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 3.743459463119507, + "code_tokens": 28, + "cvar95_kl": 11.994539260864258, + "delta_nll": 2.568263053894043, + "max_kl": 14.72020435333252, + "mean_kl": 2.6289427280426025, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.23140811920166, + "code_tokens": 84, + "cvar95_kl": 8.094209671020508, + "delta_nll": 2.9774856567382812, + "max_kl": 9.191812515258789, + "mean_kl": 3.0437381267547607, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.360527992248535, + "code_tokens": 34, + "cvar95_kl": 6.923655986785889, + "delta_nll": 2.548247814178467, + "max_kl": 8.517317771911621, + "mean_kl": 2.7212843894958496, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.38235294818878174 + }, + { + "candidate_nll": 2.5904314517974854, + "code_tokens": 28, + "cvar95_kl": 4.668458461761475, + "delta_nll": 1.3353843688964844, + "max_kl": 5.680989742279053, + "mean_kl": 1.6663668155670166, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 3.498181104660034, + "code_tokens": 92, + "cvar95_kl": 6.778264045715332, + "delta_nll": 2.3993356227874756, + "max_kl": 8.526177406311035, + "mean_kl": 2.4366238117218018, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.4021739065647125 + }, + { + "candidate_nll": 4.641665458679199, + "code_tokens": 185, + "cvar95_kl": 8.909616470336914, + "delta_nll": 3.911550760269165, + "max_kl": 12.88686466217041, + "mean_kl": 3.985501289367676, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.24864864349365234 + }, + { + "candidate_nll": 3.4310078620910645, + "code_tokens": 106, + "cvar95_kl": 8.78117847442627, + "delta_nll": 3.0780134201049805, + "max_kl": 9.950835227966309, + "mean_kl": 3.164438009262085, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.2924528419971466 + }, + { + "candidate_nll": 5.3583478927612305, + "code_tokens": 64, + "cvar95_kl": 9.040023803710938, + "delta_nll": 3.815793514251709, + "max_kl": 11.019899368286133, + "mean_kl": 3.796947956085205, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.203125 + }, + { + "candidate_nll": 4.329941749572754, + "code_tokens": 74, + "cvar95_kl": 8.299824714660645, + "delta_nll": 3.2191576957702637, + "max_kl": 9.191939353942871, + "mean_kl": 3.092604398727417, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.0482470989227295, + "code_tokens": 27, + "cvar95_kl": 8.96905517578125, + "delta_nll": 2.0608432292938232, + "max_kl": 9.666780471801758, + "mean_kl": 2.289508819580078, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.48148149251937866 + }, + { + "candidate_nll": 4.027962684631348, + "code_tokens": 72, + "cvar95_kl": 9.150960922241211, + "delta_nll": 3.4041247367858887, + "max_kl": 10.438668251037598, + "mean_kl": 3.4929840564727783, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.3055555522441864 + }, + { + "candidate_nll": 3.712430953979492, + "code_tokens": 62, + "cvar95_kl": 6.563958168029785, + "delta_nll": 2.7791285514831543, + "max_kl": 8.275555610656738, + "mean_kl": 2.5980618000030518, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 3.2290635108947754, + "code_tokens": 42, + "cvar95_kl": 8.823695182800293, + "delta_nll": 2.5308971405029297, + "max_kl": 10.773087501525879, + "mean_kl": 2.6422605514526367, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.252036094665527, + "code_tokens": 238, + "cvar95_kl": 8.972415924072266, + "delta_nll": 3.3647031784057617, + "max_kl": 11.683576583862305, + "mean_kl": 3.3879470825195312, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.2478991597890854 + }, + { + "candidate_nll": 3.598727226257324, + "code_tokens": 68, + "cvar95_kl": 7.354119777679443, + "delta_nll": 2.684215545654297, + "max_kl": 9.498540878295898, + "mean_kl": 2.682123899459839, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.38235294818878174 + }, + { + "candidate_nll": 3.7374300956726074, + "code_tokens": 83, + "cvar95_kl": 8.733644485473633, + "delta_nll": 2.7532801628112793, + "max_kl": 12.727655410766602, + "mean_kl": 2.8281452655792236, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.33734938502311707 + }, + { + "candidate_nll": 5.957076072692871, + "code_tokens": 64, + "cvar95_kl": 11.616240501403809, + "delta_nll": 4.6933088302612305, + "max_kl": 12.555733680725098, + "mean_kl": 4.799130439758301, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.203125 + }, + { + "candidate_nll": 2.464704990386963, + "code_tokens": 37, + "cvar95_kl": 5.744595527648926, + "delta_nll": 1.8079050779342651, + "max_kl": 6.664927959442139, + "mean_kl": 1.889661192893982, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.45945945382118225 + }, + { + "candidate_nll": 3.849961757659912, + "code_tokens": 27, + "cvar95_kl": 6.441207408905029, + "delta_nll": 2.3400344848632812, + "max_kl": 6.443090438842773, + "mean_kl": 2.6341745853424072, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 3.6284499168395996, + "code_tokens": 40, + "cvar95_kl": 8.381583213806152, + "delta_nll": 2.7367334365844727, + "max_kl": 8.649762153625488, + "mean_kl": 3.1330177783966064, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.1446709632873535, + "code_tokens": 28, + "cvar95_kl": 8.297554969787598, + "delta_nll": 3.1468982696533203, + "max_kl": 8.378475189208984, + "mean_kl": 3.1968557834625244, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.4352738857269287, + "code_tokens": 59, + "cvar95_kl": 7.262424468994141, + "delta_nll": 2.580644130706787, + "max_kl": 7.7946457862854, + "mean_kl": 2.755635976791382, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.35593220591545105 + }, + { + "candidate_nll": 4.029713153839111, + "code_tokens": 57, + "cvar95_kl": 9.58071517944336, + "delta_nll": 3.4597129821777344, + "max_kl": 10.053236961364746, + "mean_kl": 3.367490291595459, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.2982456088066101 + }, + { + "candidate_nll": 4.863898754119873, + "code_tokens": 25, + "cvar95_kl": 7.526782512664795, + "delta_nll": 3.4799585342407227, + "max_kl": 7.599419116973877, + "mean_kl": 3.543893337249756, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.1832854747772217, + "code_tokens": 48, + "cvar95_kl": 7.559814453125, + "delta_nll": 2.4361331462860107, + "max_kl": 9.37398624420166, + "mean_kl": 2.4640285968780518, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.49774432182312, + "code_tokens": 35, + "cvar95_kl": 6.058320045471191, + "delta_nll": 2.6359684467315674, + "max_kl": 6.156240940093994, + "mean_kl": 2.5697519779205322, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 2.4001283645629883, + "code_tokens": 30, + "cvar95_kl": 4.914880752563477, + "delta_nll": 1.7554211616516113, + "max_kl": 5.886845588684082, + "mean_kl": 1.740856409072876, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.5333333611488342 + }, + { + "candidate_nll": 3.287508010864258, + "code_tokens": 49, + "cvar95_kl": 6.584587097167969, + "delta_nll": 2.164430618286133, + "max_kl": 8.596285820007324, + "mean_kl": 2.289127826690674, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 4.944652557373047, + "code_tokens": 98, + "cvar95_kl": 8.760003089904785, + "delta_nll": 3.928435802459717, + "max_kl": 11.371747970581055, + "mean_kl": 3.7533905506134033, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.22448979318141937 + }, + { + "candidate_nll": 2.613804817199707, + "code_tokens": 25, + "cvar95_kl": 5.185585975646973, + "delta_nll": 1.2329984903335571, + "max_kl": 6.13198709487915, + "mean_kl": 1.5985134840011597, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 3.257115125656128, + "code_tokens": 27, + "cvar95_kl": 8.504387855529785, + "delta_nll": 2.4793920516967773, + "max_kl": 8.843860626220703, + "mean_kl": 2.4232122898101807, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.48148149251937866 + }, + { + "candidate_nll": 2.873076915740967, + "code_tokens": 20, + "cvar95_kl": 7.2205328941345215, + "delta_nll": 1.8279922008514404, + "max_kl": 7.2205328941345215, + "mean_kl": 2.317434787750244, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 4.452757358551025, + "code_tokens": 253, + "cvar95_kl": 9.511903762817383, + "delta_nll": 3.760087490081787, + "max_kl": 11.42101001739502, + "mean_kl": 3.826676607131958, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.24110671877861023 + }, + { + "candidate_nll": 4.114780902862549, + "code_tokens": 55, + "cvar95_kl": 9.184792518615723, + "delta_nll": 3.285660743713379, + "max_kl": 11.100996017456055, + "mean_kl": 3.4006502628326416, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.6492292881011963, + "code_tokens": 31, + "cvar95_kl": 6.580265522003174, + "delta_nll": 2.4427309036254883, + "max_kl": 8.10331916809082, + "mean_kl": 2.5637476444244385, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 4.461638927459717, + "code_tokens": 35, + "cvar95_kl": 11.83626651763916, + "delta_nll": 3.7573928833007812, + "max_kl": 13.411162376403809, + "mean_kl": 3.5979697704315186, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 2.603724241256714, + "code_tokens": 61, + "cvar95_kl": 5.680471897125244, + "delta_nll": 1.8008557558059692, + "max_kl": 6.543490886688232, + "mean_kl": 1.9184491634368896, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.5081967115402222 + }, + { + "candidate_nll": 3.7554030418395996, + "code_tokens": 55, + "cvar95_kl": 7.916075229644775, + "delta_nll": 2.800685405731201, + "max_kl": 8.500347137451172, + "mean_kl": 2.7741270065307617, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.3272727131843567 + }, + { + "candidate_nll": 4.777730464935303, + "code_tokens": 108, + "cvar95_kl": 8.723578453063965, + "delta_nll": 3.270763397216797, + "max_kl": 13.214776992797852, + "mean_kl": 3.3865773677825928, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.21296297013759613 + }, + { + "candidate_nll": 3.6932857036590576, + "code_tokens": 58, + "cvar95_kl": 6.244317531585693, + "delta_nll": 2.405979633331299, + "max_kl": 7.017670154571533, + "mean_kl": 2.452244997024536, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.3965517282485962 + }, + { + "candidate_nll": 4.598300933837891, + "code_tokens": 91, + "cvar95_kl": 8.874094009399414, + "delta_nll": 3.770845651626587, + "max_kl": 13.794089317321777, + "mean_kl": 3.7385404109954834, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.20879121124744415 + }, + { + "candidate_nll": 3.024914503097534, + "code_tokens": 105, + "cvar95_kl": 6.6581501960754395, + "delta_nll": 2.2611751556396484, + "max_kl": 9.38401985168457, + "mean_kl": 2.365142345428467, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 3.7623531818389893, + "code_tokens": 49, + "cvar95_kl": 7.561730861663818, + "delta_nll": 3.115100622177124, + "max_kl": 8.315485000610352, + "mean_kl": 2.8673367500305176, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 3.435776948928833, + "code_tokens": 27, + "cvar95_kl": 7.680008888244629, + "delta_nll": 1.973613977432251, + "max_kl": 9.198391914367676, + "mean_kl": 2.2784276008605957, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.253293991088867, + "code_tokens": 25, + "cvar95_kl": 6.315712928771973, + "delta_nll": 2.2129173278808594, + "max_kl": 6.341517925262451, + "mean_kl": 2.568274736404419, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.5200252532958984, + "code_tokens": 98, + "cvar95_kl": 8.322338104248047, + "delta_nll": 2.224970817565918, + "max_kl": 11.208114624023438, + "mean_kl": 2.465564727783203, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.3979591727256775 + }, + { + "candidate_nll": 2.8579018115997314, + "code_tokens": 23, + "cvar95_kl": 5.175960540771484, + "delta_nll": 2.208120107650757, + "max_kl": 6.150533676147461, + "mean_kl": 2.0777335166931152, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.47826087474823 + }, + { + "candidate_nll": 4.1821441650390625, + "code_tokens": 31, + "cvar95_kl": 5.768164157867432, + "delta_nll": 2.316770553588867, + "max_kl": 6.358329772949219, + "mean_kl": 2.777165412902832, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 3.0484564304351807, + "code_tokens": 36, + "cvar95_kl": 9.975723266601562, + "delta_nll": 2.300363302230835, + "max_kl": 13.133899688720703, + "mean_kl": 2.265131950378418, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 3.1531221866607666, + "code_tokens": 38, + "cvar95_kl": 6.3751325607299805, + "delta_nll": 1.7194446325302124, + "max_kl": 7.184912204742432, + "mean_kl": 2.1859967708587646, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.4736842215061188 + }, + { + "candidate_nll": 3.296121835708618, + "code_tokens": 25, + "cvar95_kl": 5.208107948303223, + "delta_nll": 1.6900320053100586, + "max_kl": 6.7628302574157715, + "mean_kl": 1.5968748331069946, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 2.406923532485962, + "code_tokens": 26, + "cvar95_kl": 5.385037422180176, + "delta_nll": 1.5274744033813477, + "max_kl": 6.335101127624512, + "mean_kl": 1.9274842739105225, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.9416356086730957, + "code_tokens": 62, + "cvar95_kl": 8.297627449035645, + "delta_nll": 3.1991934776306152, + "max_kl": 8.75294017791748, + "mean_kl": 3.194898843765259, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 4.053144454956055, + "code_tokens": 36, + "cvar95_kl": 5.521603107452393, + "delta_nll": 2.3146252632141113, + "max_kl": 5.6844801902771, + "mean_kl": 2.6196916103363037, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9294285774230957, + "code_tokens": 119, + "cvar95_kl": 7.543321132659912, + "delta_nll": 2.59883975982666, + "max_kl": 9.315592765808105, + "mean_kl": 2.757453680038452, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.32773110270500183 + }, + { + "candidate_nll": 3.5789506435394287, + "code_tokens": 37, + "cvar95_kl": 7.623802185058594, + "delta_nll": 3.2662429809570312, + "max_kl": 7.982696056365967, + "mean_kl": 3.029280185699463, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.892798662185669, + "code_tokens": 28, + "cvar95_kl": 10.075801849365234, + "delta_nll": 3.110065221786499, + "max_kl": 11.267196655273438, + "mean_kl": 3.438781261444092, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.554297924041748, + "code_tokens": 143, + "cvar95_kl": 8.540397644042969, + "delta_nll": 2.9879510402679443, + "max_kl": 10.170547485351562, + "mean_kl": 3.003401041030884, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.37062937021255493 + }, + { + "candidate_nll": 3.780261754989624, + "code_tokens": 29, + "cvar95_kl": 8.288434982299805, + "delta_nll": 1.9979554414749146, + "max_kl": 8.347906112670898, + "mean_kl": 2.6659326553344727, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.4137931168079376 + }, + { + "candidate_nll": 4.048064708709717, + "code_tokens": 28, + "cvar95_kl": 8.53363037109375, + "delta_nll": 2.975576400756836, + "max_kl": 10.346918106079102, + "mean_kl": 2.8234760761260986, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 4.627407550811768, + "code_tokens": 30, + "cvar95_kl": 10.177643775939941, + "delta_nll": 3.363369941711426, + "max_kl": 11.449633598327637, + "mean_kl": 3.771623373031616, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.1979434490203857, + "code_tokens": 85, + "cvar95_kl": 8.429342269897461, + "delta_nll": 2.7761332988739014, + "max_kl": 9.837424278259277, + "mean_kl": 2.6841530799865723, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.842589855194092, + "code_tokens": 44, + "cvar95_kl": 9.044302940368652, + "delta_nll": 3.8653461933135986, + "max_kl": 9.364442825317383, + "mean_kl": 3.6473450660705566, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.476077079772949, + "code_tokens": 19, + "cvar95_kl": 8.716670036315918, + "delta_nll": 3.6719748973846436, + "max_kl": 8.716670036315918, + "mean_kl": 3.327810525894165, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.720449209213257, + "code_tokens": 113, + "cvar95_kl": 7.658159255981445, + "delta_nll": 2.9830262660980225, + "max_kl": 9.4535551071167, + "mean_kl": 2.8954362869262695, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.3008849620819092 + }, + { + "candidate_nll": 4.012915134429932, + "code_tokens": 40, + "cvar95_kl": 7.324538230895996, + "delta_nll": 2.5377206802368164, + "max_kl": 7.378426551818848, + "mean_kl": 3.2049400806427, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.212809085845947, + "code_tokens": 51, + "cvar95_kl": 8.90218448638916, + "delta_nll": 3.606149196624756, + "max_kl": 10.133849143981934, + "mean_kl": 3.9139678478240967, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 3.888690233230591, + "code_tokens": 55, + "cvar95_kl": 7.297056198120117, + "delta_nll": 2.3076577186584473, + "max_kl": 9.407303810119629, + "mean_kl": 2.4581949710845947, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.290909081697464 + }, + { + "candidate_nll": 2.9382143020629883, + "code_tokens": 27, + "cvar95_kl": 6.092146873474121, + "delta_nll": 1.8081459999084473, + "max_kl": 6.523981094360352, + "mean_kl": 1.8058662414550781, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.5185185074806213 + }, + { + "candidate_nll": 4.69281005859375, + "code_tokens": 32, + "cvar95_kl": 8.238748550415039, + "delta_nll": 3.853003978729248, + "max_kl": 8.288086891174316, + "mean_kl": 3.6764862537384033, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.4955649375915527, + "code_tokens": 64, + "cvar95_kl": 7.732203960418701, + "delta_nll": 2.843026876449585, + "max_kl": 8.173371315002441, + "mean_kl": 2.934462785720825, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.637184143066406, + "code_tokens": 35, + "cvar95_kl": 9.09541130065918, + "delta_nll": 3.5875298976898193, + "max_kl": 9.266764640808105, + "mean_kl": 3.6102874279022217, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.278226375579834, + "code_tokens": 152, + "cvar95_kl": 9.801025390625, + "delta_nll": 3.82637882232666, + "max_kl": 11.120529174804688, + "mean_kl": 3.7182459831237793, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.21710526943206787 + }, + { + "candidate_nll": 3.9998180866241455, + "code_tokens": 77, + "cvar95_kl": 7.468335151672363, + "delta_nll": 2.7384839057922363, + "max_kl": 8.289471626281738, + "mean_kl": 2.839928388595581, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.33766233921051025 + }, + { + "candidate_nll": 3.4017512798309326, + "code_tokens": 77, + "cvar95_kl": 7.717950344085693, + "delta_nll": 2.833470344543457, + "max_kl": 8.27444076538086, + "mean_kl": 2.7378673553466797, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.33766233921051025 + }, + { + "candidate_nll": 2.4468212127685547, + "code_tokens": 71, + "cvar95_kl": 6.358656883239746, + "delta_nll": 1.8847324848175049, + "max_kl": 9.205120086669922, + "mean_kl": 2.011066436767578, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.5352112650871277 + }, + { + "candidate_nll": 5.147459506988525, + "code_tokens": 112, + "cvar95_kl": 10.12137508392334, + "delta_nll": 4.26908016204834, + "max_kl": 11.103832244873047, + "mean_kl": 4.066687107086182, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.1624417304992676, + "code_tokens": 111, + "cvar95_kl": 7.258369445800781, + "delta_nll": 2.442538022994995, + "max_kl": 8.386937141418457, + "mean_kl": 2.4441978931427, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.45045045018196106 + }, + { + "candidate_nll": 3.6832528114318848, + "code_tokens": 62, + "cvar95_kl": 8.392156600952148, + "delta_nll": 2.8953566551208496, + "max_kl": 9.659762382507324, + "mean_kl": 2.9650321006774902, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.30645161867141724 + }, + { + "candidate_nll": 2.4575674533843994, + "code_tokens": 52, + "cvar95_kl": 5.037691116333008, + "delta_nll": 1.8469874858856201, + "max_kl": 5.529775619506836, + "mean_kl": 1.7954169511795044, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.5192307829856873 + }, + { + "candidate_nll": 4.0768938064575195, + "code_tokens": 24, + "cvar95_kl": 5.169074535369873, + "delta_nll": 2.1816673278808594, + "max_kl": 5.565786361694336, + "mean_kl": 2.216773748397827, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.44521427154541, + "code_tokens": 48, + "cvar95_kl": 7.5163092613220215, + "delta_nll": 2.5534443855285645, + "max_kl": 8.200068473815918, + "mean_kl": 2.810157537460327, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.3541666567325592 + }, + { + "candidate_nll": 3.9385058879852295, + "code_tokens": 98, + "cvar95_kl": 9.095285415649414, + "delta_nll": 2.8882129192352295, + "max_kl": 9.42080307006836, + "mean_kl": 3.1358134746551514, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 3.9891061782836914, + "code_tokens": 48, + "cvar95_kl": 7.37077522277832, + "delta_nll": 2.9964206218719482, + "max_kl": 9.001171112060547, + "mean_kl": 2.937566041946411, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.3958333432674408 + }, + { + "candidate_nll": 4.660609722137451, + "code_tokens": 56, + "cvar95_kl": 10.471293449401855, + "delta_nll": 3.7146143913269043, + "max_kl": 13.090400695800781, + "mean_kl": 3.8102002143859863, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.3392857015132904 + }, + { + "candidate_nll": 4.628371238708496, + "code_tokens": 88, + "cvar95_kl": 9.895233154296875, + "delta_nll": 4.02709436416626, + "max_kl": 11.693394660949707, + "mean_kl": 3.9245522022247314, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.28409090638160706 + }, + { + "candidate_nll": 4.357773303985596, + "code_tokens": 50, + "cvar95_kl": 7.431310176849365, + "delta_nll": 2.7242431640625, + "max_kl": 8.981611251831055, + "mean_kl": 2.9213974475860596, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 2.133760452270508, + "code_tokens": 85, + "cvar95_kl": 6.545371055603027, + "delta_nll": 1.6967732906341553, + "max_kl": 9.965310096740723, + "mean_kl": 1.6536115407943726, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.5176470875740051 + }, + { + "candidate_nll": 4.229367733001709, + "code_tokens": 27, + "cvar95_kl": 6.951198577880859, + "delta_nll": 3.1551547050476074, + "max_kl": 7.191601276397705, + "mean_kl": 3.35949444770813, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 4.351576328277588, + "code_tokens": 48, + "cvar95_kl": 9.540066719055176, + "delta_nll": 3.2339577674865723, + "max_kl": 10.53604507446289, + "mean_kl": 3.5224015712738037, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.967167854309082, + "code_tokens": 29, + "cvar95_kl": 9.825973510742188, + "delta_nll": 3.3193206787109375, + "max_kl": 10.013962745666504, + "mean_kl": 3.346827507019043, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 4.054534435272217, + "code_tokens": 63, + "cvar95_kl": 9.404399871826172, + "delta_nll": 3.116319179534912, + "max_kl": 10.655838966369629, + "mean_kl": 3.372054100036621, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 3.8640449047088623, + "code_tokens": 89, + "cvar95_kl": 7.726492404937744, + "delta_nll": 3.188986301422119, + "max_kl": 9.547503471374512, + "mean_kl": 3.1549112796783447, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.24719101190567017 + }, + { + "candidate_nll": 4.564461708068848, + "code_tokens": 133, + "cvar95_kl": 8.381112098693848, + "delta_nll": 3.2201833724975586, + "max_kl": 9.435029029846191, + "mean_kl": 3.3695788383483887, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 2.3332083225250244, + "code_tokens": 25, + "cvar95_kl": 3.980456829071045, + "delta_nll": 1.3846715688705444, + "max_kl": 4.191792011260986, + "mean_kl": 1.3963950872421265, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.9526474475860596, + "code_tokens": 33, + "cvar95_kl": 6.077731609344482, + "delta_nll": 1.9153470993041992, + "max_kl": 7.219076633453369, + "mean_kl": 2.2871880531311035, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 2.470386266708374, + "code_tokens": 93, + "cvar95_kl": 8.550630569458008, + "delta_nll": 1.713611364364624, + "max_kl": 11.522867202758789, + "mean_kl": 2.047464609146118, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.5376344323158264 + }, + { + "candidate_nll": 3.8294644355773926, + "code_tokens": 101, + "cvar95_kl": 7.646758556365967, + "delta_nll": 2.861064910888672, + "max_kl": 8.897254943847656, + "mean_kl": 2.801790714263916, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.3465346395969391 + }, + { + "candidate_nll": 4.395529270172119, + "code_tokens": 152, + "cvar95_kl": 10.331520080566406, + "delta_nll": 3.830765962600708, + "max_kl": 14.368103981018066, + "mean_kl": 3.796889543533325, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.2763157784938812 + }, + { + "candidate_nll": 4.123966217041016, + "code_tokens": 33, + "cvar95_kl": 7.886044502258301, + "delta_nll": 2.7230947017669678, + "max_kl": 8.883645057678223, + "mean_kl": 2.967686891555786, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.896610975265503, + "code_tokens": 29, + "cvar95_kl": 9.067842483520508, + "delta_nll": 2.6101479530334473, + "max_kl": 10.840971946716309, + "mean_kl": 2.5699996948242188, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 5.24039888381958, + "code_tokens": 27, + "cvar95_kl": 9.442991256713867, + "delta_nll": 3.7823290824890137, + "max_kl": 9.701892852783203, + "mean_kl": 3.6483707427978516, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.7958059310913086, + "code_tokens": 33, + "cvar95_kl": 8.151897430419922, + "delta_nll": 3.4857659339904785, + "max_kl": 8.635640144348145, + "mean_kl": 3.3594775199890137, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 3.9492952823638916, + "code_tokens": 27, + "cvar95_kl": 7.255944728851318, + "delta_nll": 3.1528968811035156, + "max_kl": 7.302423000335693, + "mean_kl": 3.3751938343048096, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.7383296489715576, + "code_tokens": 84, + "cvar95_kl": 7.733315944671631, + "delta_nll": 3.0344362258911133, + "max_kl": 8.568126678466797, + "mean_kl": 3.124691963195801, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.367125988006592, + "code_tokens": 103, + "cvar95_kl": 6.6267409324646, + "delta_nll": 2.3216562271118164, + "max_kl": 7.413001537322998, + "mean_kl": 2.425525665283203, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.35922330617904663 + }, + { + "candidate_nll": 4.009372234344482, + "code_tokens": 88, + "cvar95_kl": 8.378414154052734, + "delta_nll": 3.093024969100952, + "max_kl": 9.57367992401123, + "mean_kl": 3.052720546722412, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.3295454680919647 + }, + { + "candidate_nll": 5.316073894500732, + "code_tokens": 62, + "cvar95_kl": 8.68299674987793, + "delta_nll": 4.007143020629883, + "max_kl": 9.942553520202637, + "mean_kl": 3.6983752250671387, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.24193547666072845 + }, + { + "candidate_nll": 3.805356740951538, + "code_tokens": 24, + "cvar95_kl": 4.877958297729492, + "delta_nll": 2.2686479091644287, + "max_kl": 5.443836212158203, + "mean_kl": 2.401197671890259, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.881122350692749, + "code_tokens": 34, + "cvar95_kl": 7.954127311706543, + "delta_nll": 2.542670249938965, + "max_kl": 8.747828483581543, + "mean_kl": 2.429611921310425, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 3.709561347961426, + "code_tokens": 43, + "cvar95_kl": 8.171048164367676, + "delta_nll": 2.9898159503936768, + "max_kl": 8.752848625183105, + "mean_kl": 2.9767186641693115, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.3488371968269348 + }, + { + "candidate_nll": 4.754175662994385, + "code_tokens": 54, + "cvar95_kl": 9.042387962341309, + "delta_nll": 3.2648704051971436, + "max_kl": 10.892945289611816, + "mean_kl": 3.3526880741119385, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.3592913150787354, + "code_tokens": 50, + "cvar95_kl": 6.712831974029541, + "delta_nll": 2.629405975341797, + "max_kl": 8.419926643371582, + "mean_kl": 2.601346015930176, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.948741912841797, + "code_tokens": 235, + "cvar95_kl": 8.417548179626465, + "delta_nll": 3.1912314891815186, + "max_kl": 10.731773376464844, + "mean_kl": 3.248368501663208, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.34893617033958435 + }, + { + "candidate_nll": 4.601778984069824, + "code_tokens": 33, + "cvar95_kl": 7.335174560546875, + "delta_nll": 2.5731239318847656, + "max_kl": 7.775979042053223, + "mean_kl": 3.238719940185547, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 4.394473075866699, + "code_tokens": 45, + "cvar95_kl": 6.8901519775390625, + "delta_nll": 3.407970666885376, + "max_kl": 7.255045413970947, + "mean_kl": 3.2794535160064697, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.8064799308776855, + "code_tokens": 28, + "cvar95_kl": 9.319903373718262, + "delta_nll": 3.1835200786590576, + "max_kl": 10.510403633117676, + "mean_kl": 2.880378246307373, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.247403144836426, + "code_tokens": 49, + "cvar95_kl": 6.657670974731445, + "delta_nll": 2.569138765335083, + "max_kl": 6.964437007904053, + "mean_kl": 2.6532626152038574, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 4.330038070678711, + "code_tokens": 107, + "cvar95_kl": 10.237415313720703, + "delta_nll": 4.080622673034668, + "max_kl": 12.403519630432129, + "mean_kl": 3.946079730987549, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.28037384152412415 + }, + { + "candidate_nll": 4.043099403381348, + "code_tokens": 48, + "cvar95_kl": 8.809823036193848, + "delta_nll": 2.8221893310546875, + "max_kl": 10.037240982055664, + "mean_kl": 3.2272891998291016, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.4212546348571777, + "code_tokens": 40, + "cvar95_kl": 6.169406890869141, + "delta_nll": 1.5817137956619263, + "max_kl": 6.67949104309082, + "mean_kl": 2.0212268829345703, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 2.691220760345459, + "code_tokens": 46, + "cvar95_kl": 7.287107944488525, + "delta_nll": 2.2217371463775635, + "max_kl": 10.924832344055176, + "mean_kl": 2.4411017894744873, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.3695652186870575 + }, + { + "candidate_nll": 3.410109043121338, + "code_tokens": 31, + "cvar95_kl": 6.994067668914795, + "delta_nll": 2.367659330368042, + "max_kl": 7.11026668548584, + "mean_kl": 2.5915443897247314, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.4516128897666931 + }, + { + "candidate_nll": 1.6579121351242065, + "code_tokens": 58, + "cvar95_kl": 4.721085071563721, + "delta_nll": 1.2414276599884033, + "max_kl": 5.622999668121338, + "mean_kl": 1.2438324689865112, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.6379310488700867 + }, + { + "candidate_nll": 2.7720680236816406, + "code_tokens": 49, + "cvar95_kl": 5.41569709777832, + "delta_nll": 1.654298186302185, + "max_kl": 6.525915622711182, + "mean_kl": 1.6616969108581543, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.4897959232330322 + }, + { + "candidate_nll": 2.3200161457061768, + "code_tokens": 18, + "cvar95_kl": 4.495962619781494, + "delta_nll": 1.373422622680664, + "max_kl": 4.495962619781494, + "mean_kl": 1.4955506324768066, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 2.641019344329834, + "code_tokens": 28, + "cvar95_kl": 5.117300033569336, + "delta_nll": 1.4816077947616577, + "max_kl": 5.200903415679932, + "mean_kl": 1.8559989929199219, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.9092659950256348, + "code_tokens": 57, + "cvar95_kl": 7.500428676605225, + "delta_nll": 3.16719913482666, + "max_kl": 7.891904830932617, + "mean_kl": 3.320647716522217, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 3.2418830394744873, + "code_tokens": 88, + "cvar95_kl": 7.877524375915527, + "delta_nll": 2.5417730808258057, + "max_kl": 8.817804336547852, + "mean_kl": 2.648747444152832, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.7178139686584473, + "code_tokens": 23, + "cvar95_kl": 7.821326732635498, + "delta_nll": 2.5190272331237793, + "max_kl": 8.198034286499023, + "mean_kl": 2.682481527328491, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 5.36529541015625, + "code_tokens": 40, + "cvar95_kl": 11.479629516601562, + "delta_nll": 4.105837821960449, + "max_kl": 13.50699520111084, + "mean_kl": 4.264955520629883, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.329496383666992, + "code_tokens": 20, + "cvar95_kl": 8.286314964294434, + "delta_nll": 3.168281316757202, + "max_kl": 8.286314964294434, + "mean_kl": 3.3525009155273438, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.7788889408111572, + "code_tokens": 63, + "cvar95_kl": 7.112260818481445, + "delta_nll": 2.763103485107422, + "max_kl": 8.032666206359863, + "mean_kl": 2.7400145530700684, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.259169340133667, + "code_tokens": 39, + "cvar95_kl": 6.249277114868164, + "delta_nll": 2.306488037109375, + "max_kl": 6.594289779663086, + "mean_kl": 2.469358205795288, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 4.1213507652282715, + "code_tokens": 50, + "cvar95_kl": 8.664580345153809, + "delta_nll": 2.7094860076904297, + "max_kl": 9.16715145111084, + "mean_kl": 2.851956367492676, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.11886739730835, + "code_tokens": 63, + "cvar95_kl": 9.55199909210205, + "delta_nll": 3.1513900756835938, + "max_kl": 10.404027938842773, + "mean_kl": 3.427302122116089, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 4.939764976501465, + "code_tokens": 46, + "cvar95_kl": 9.18823528289795, + "delta_nll": 3.96720290184021, + "max_kl": 10.020614624023438, + "mean_kl": 3.997657537460327, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 3.341188907623291, + "code_tokens": 25, + "cvar95_kl": 7.895838737487793, + "delta_nll": 2.662294387817383, + "max_kl": 8.256479263305664, + "mean_kl": 2.945880174636841, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.2066853046417236, + "code_tokens": 111, + "cvar95_kl": 6.041900634765625, + "delta_nll": 2.3924193382263184, + "max_kl": 6.8594069480896, + "mean_kl": 2.434662103652954, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.4144144058227539 + }, + { + "candidate_nll": 3.7465226650238037, + "code_tokens": 71, + "cvar95_kl": 7.439123153686523, + "delta_nll": 3.0179648399353027, + "max_kl": 8.571677207946777, + "mean_kl": 2.912043571472168, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.28169015049934387 + }, + { + "candidate_nll": 4.356515407562256, + "code_tokens": 28, + "cvar95_kl": 6.8552069664001465, + "delta_nll": 2.9857916831970215, + "max_kl": 7.563982963562012, + "mean_kl": 3.260376214981079, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 4.505384922027588, + "code_tokens": 25, + "cvar95_kl": 9.784930229187012, + "delta_nll": 3.30492901802063, + "max_kl": 10.460108757019043, + "mean_kl": 3.4610822200775146, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.8851656913757324, + "code_tokens": 40, + "cvar95_kl": 6.057552337646484, + "delta_nll": 2.5823938846588135, + "max_kl": 6.5749945640563965, + "mean_kl": 2.2288498878479004, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.9322733879089355, + "code_tokens": 177, + "cvar95_kl": 8.382525444030762, + "delta_nll": 2.887829303741455, + "max_kl": 11.084892272949219, + "mean_kl": 3.209721565246582, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.2881355881690979 + }, + { + "candidate_nll": 3.111236333847046, + "code_tokens": 51, + "cvar95_kl": 5.178574085235596, + "delta_nll": 2.0200576782226562, + "max_kl": 5.77569055557251, + "mean_kl": 2.13496732711792, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 2.322484016418457, + "code_tokens": 69, + "cvar95_kl": 7.168124198913574, + "delta_nll": 1.8680236339569092, + "max_kl": 8.469680786132812, + "mean_kl": 1.7818139791488647, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.49275362491607666 + }, + { + "candidate_nll": 3.3269529342651367, + "code_tokens": 26, + "cvar95_kl": 8.011857032775879, + "delta_nll": 2.57869291305542, + "max_kl": 8.404487609863281, + "mean_kl": 2.594367742538452, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 2.9714787006378174, + "code_tokens": 24, + "cvar95_kl": 4.919551372528076, + "delta_nll": 1.4531394243240356, + "max_kl": 6.140954494476318, + "mean_kl": 1.7134672403335571, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.832440137863159, + "code_tokens": 15, + "cvar95_kl": 8.885388374328613, + "delta_nll": 2.68196439743042, + "max_kl": 8.885388374328613, + "mean_kl": 2.485142230987549, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.590231418609619, + "code_tokens": 43, + "cvar95_kl": 9.595413208007812, + "delta_nll": 3.440882444381714, + "max_kl": 10.449369430541992, + "mean_kl": 3.726484775543213, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.3488371968269348 + }, + { + "candidate_nll": 4.5126729011535645, + "code_tokens": 18, + "cvar95_kl": 6.44251823425293, + "delta_nll": 3.420347213745117, + "max_kl": 6.44251823425293, + "mean_kl": 3.295889139175415, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.4354448318481445, + "code_tokens": 49, + "cvar95_kl": 7.109437465667725, + "delta_nll": 2.707451343536377, + "max_kl": 7.913554668426514, + "mean_kl": 2.794438600540161, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.3877550959587097 + }, + { + "candidate_nll": 3.5701193809509277, + "code_tokens": 59, + "cvar95_kl": 8.252632141113281, + "delta_nll": 2.7023632526397705, + "max_kl": 10.563410758972168, + "mean_kl": 2.8220455646514893, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.32203391194343567 + }, + { + "candidate_nll": 3.0136754512786865, + "code_tokens": 221, + "cvar95_kl": 8.0324125289917, + "delta_nll": 2.4284093379974365, + "max_kl": 10.90007495880127, + "mean_kl": 2.5285027027130127, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.4072398245334625 + }, + { + "candidate_nll": 3.9536080360412598, + "code_tokens": 25, + "cvar95_kl": 9.473272323608398, + "delta_nll": 2.751734972000122, + "max_kl": 12.233989715576172, + "mean_kl": 2.966763496398926, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.362787961959839, + "code_tokens": 36, + "cvar95_kl": 7.786057949066162, + "delta_nll": 2.6388673782348633, + "max_kl": 7.92656135559082, + "mean_kl": 2.8869779109954834, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.208718776702881, + "code_tokens": 158, + "cvar95_kl": 8.831171035766602, + "delta_nll": 3.5645740032196045, + "max_kl": 10.29761791229248, + "mean_kl": 3.5405492782592773, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.18987341225147247 + }, + { + "candidate_nll": 4.496691703796387, + "code_tokens": 92, + "cvar95_kl": 8.217584609985352, + "delta_nll": 3.181156635284424, + "max_kl": 8.621423721313477, + "mean_kl": 3.2604563236236572, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 3.202434539794922, + "code_tokens": 102, + "cvar95_kl": 6.592876434326172, + "delta_nll": 2.2883918285369873, + "max_kl": 7.980042457580566, + "mean_kl": 2.258336067199707, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.4313725531101227 + }, + { + "candidate_nll": 3.913938283920288, + "code_tokens": 171, + "cvar95_kl": 9.288991928100586, + "delta_nll": 3.3039681911468506, + "max_kl": 12.560635566711426, + "mean_kl": 3.287487745285034, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.30994153022766113 + }, + { + "candidate_nll": 3.7021515369415283, + "code_tokens": 40, + "cvar95_kl": 11.154193878173828, + "delta_nll": 2.4972872734069824, + "max_kl": 13.563769340515137, + "mean_kl": 2.7789580821990967, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 4.537960052490234, + "code_tokens": 48, + "cvar95_kl": 8.98200511932373, + "delta_nll": 3.3019461631774902, + "max_kl": 9.758700370788574, + "mean_kl": 3.449458360671997, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 4.938316822052002, + "code_tokens": 24, + "cvar95_kl": 9.726177215576172, + "delta_nll": 3.3333964347839355, + "max_kl": 10.461243629455566, + "mean_kl": 3.986863374710083, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.5111372470855713, + "code_tokens": 48, + "cvar95_kl": 7.08132791519165, + "delta_nll": 2.5240330696105957, + "max_kl": 7.29359769821167, + "mean_kl": 2.4576427936553955, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 2.9162650108337402, + "code_tokens": 25, + "cvar95_kl": 6.637014865875244, + "delta_nll": 1.5293818712234497, + "max_kl": 9.088689804077148, + "mean_kl": 1.5155768394470215, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 4.368386745452881, + "code_tokens": 30, + "cvar95_kl": 10.062666893005371, + "delta_nll": 2.8696417808532715, + "max_kl": 12.786702156066895, + "mean_kl": 2.8101131916046143, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.215651750564575, + "code_tokens": 19, + "cvar95_kl": 6.672066688537598, + "delta_nll": 1.7888338565826416, + "max_kl": 6.672066688537598, + "mean_kl": 2.098344087600708, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.5263158082962036 + }, + { + "candidate_nll": 2.4004318714141846, + "code_tokens": 12, + "cvar95_kl": 5.028165817260742, + "delta_nll": 1.321359395980835, + "max_kl": 5.028165817260742, + "mean_kl": 1.4174336194992065, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 3.797213077545166, + "code_tokens": 37, + "cvar95_kl": 10.156929016113281, + "delta_nll": 2.4569034576416016, + "max_kl": 11.10788631439209, + "mean_kl": 2.7914321422576904, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 2.879987955093384, + "code_tokens": 33, + "cvar95_kl": 5.248059272766113, + "delta_nll": 1.674558162689209, + "max_kl": 5.521737575531006, + "mean_kl": 1.6735128164291382, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.4848484992980957 + }, + { + "candidate_nll": 4.686479091644287, + "code_tokens": 94, + "cvar95_kl": 9.294256210327148, + "delta_nll": 3.83436918258667, + "max_kl": 10.215092658996582, + "mean_kl": 3.951448440551758, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.27659574151039124 + }, + { + "candidate_nll": 3.43428373336792, + "code_tokens": 90, + "cvar95_kl": 8.659860610961914, + "delta_nll": 2.809518814086914, + "max_kl": 11.810229301452637, + "mean_kl": 2.950932025909424, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.3777777850627899 + }, + { + "candidate_nll": 3.0646233558654785, + "code_tokens": 47, + "cvar95_kl": 5.7657294273376465, + "delta_nll": 1.6168594360351562, + "max_kl": 6.488391399383545, + "mean_kl": 1.9555484056472778, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.4680851101875305 + }, + { + "candidate_nll": 3.7934505939483643, + "code_tokens": 61, + "cvar95_kl": 7.113189697265625, + "delta_nll": 2.8696165084838867, + "max_kl": 7.809606075286865, + "mean_kl": 3.043883800506592, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.2950819730758667 + }, + { + "candidate_nll": 3.8114964962005615, + "code_tokens": 39, + "cvar95_kl": 6.832348346710205, + "delta_nll": 2.587981700897217, + "max_kl": 7.422207832336426, + "mean_kl": 2.792099952697754, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.0396728515625, + "code_tokens": 152, + "cvar95_kl": 8.508940696716309, + "delta_nll": 3.107511043548584, + "max_kl": 10.38015079498291, + "mean_kl": 3.120532512664795, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.3618420958518982 + }, + { + "candidate_nll": 3.983668804168701, + "code_tokens": 100, + "cvar95_kl": 8.203938484191895, + "delta_nll": 3.014686346054077, + "max_kl": 9.761510848999023, + "mean_kl": 2.973780870437622, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.23000000417232513 + }, + { + "candidate_nll": 3.124004602432251, + "code_tokens": 89, + "cvar95_kl": 8.008688926696777, + "delta_nll": 2.5030767917633057, + "max_kl": 9.2323579788208, + "mean_kl": 2.345400094985962, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.47191011905670166 + }, + { + "candidate_nll": 4.489890098571777, + "code_tokens": 40, + "cvar95_kl": 9.131536483764648, + "delta_nll": 3.1788887977600098, + "max_kl": 9.710447311401367, + "mean_kl": 3.317194700241089, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.7536749839782715, + "code_tokens": 76, + "cvar95_kl": 8.92607593536377, + "delta_nll": 3.0521080493927, + "max_kl": 9.971076011657715, + "mean_kl": 3.021836042404175, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 3.2812774181365967, + "code_tokens": 56, + "cvar95_kl": 7.167760372161865, + "delta_nll": 2.032289981842041, + "max_kl": 9.53904914855957, + "mean_kl": 1.98512864112854, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.4107142984867096 + }, + { + "candidate_nll": 4.959966659545898, + "code_tokens": 34, + "cvar95_kl": 7.586277008056641, + "delta_nll": 3.470536231994629, + "max_kl": 8.325507164001465, + "mean_kl": 3.447880744934082, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 5.030614376068115, + "code_tokens": 96, + "cvar95_kl": 10.715006828308105, + "delta_nll": 4.352502822875977, + "max_kl": 12.081476211547852, + "mean_kl": 4.223163604736328, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.2604166567325592 + }, + { + "candidate_nll": 3.076525926589966, + "code_tokens": 451, + "cvar95_kl": 8.454312324523926, + "delta_nll": 2.377830743789673, + "max_kl": 16.071327209472656, + "mean_kl": 2.328768491744995, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.4301552176475525 + }, + { + "candidate_nll": 4.721820831298828, + "code_tokens": 37, + "cvar95_kl": 9.891840934753418, + "delta_nll": 3.109511137008667, + "max_kl": 11.614641189575195, + "mean_kl": 3.3913917541503906, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 3.481374740600586, + "code_tokens": 42, + "cvar95_kl": 8.094329833984375, + "delta_nll": 2.239776849746704, + "max_kl": 9.535771369934082, + "mean_kl": 2.5196900367736816, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.852731943130493, + "code_tokens": 33, + "cvar95_kl": 6.8843255043029785, + "delta_nll": 2.8359761238098145, + "max_kl": 7.681029319763184, + "mean_kl": 2.7874367237091064, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 3.9748339653015137, + "code_tokens": 43, + "cvar95_kl": 7.885704517364502, + "delta_nll": 3.0487823486328125, + "max_kl": 8.28994369506836, + "mean_kl": 3.082103729248047, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.23255814611911774 + }, + { + "candidate_nll": 3.957385778427124, + "code_tokens": 73, + "cvar95_kl": 8.122105598449707, + "delta_nll": 3.220949411392212, + "max_kl": 9.577972412109375, + "mean_kl": 3.2259461879730225, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.27397260069847107 + }, + { + "candidate_nll": 2.9203953742980957, + "code_tokens": 15, + "cvar95_kl": 6.953248500823975, + "delta_nll": 2.0079193115234375, + "max_kl": 6.953248500823975, + "mean_kl": 1.9512102603912354, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 4.352602481842041, + "code_tokens": 34, + "cvar95_kl": 8.979070663452148, + "delta_nll": 3.093522548675537, + "max_kl": 9.079025268554688, + "mean_kl": 3.6733510494232178, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 3.9414119720458984, + "code_tokens": 136, + "cvar95_kl": 6.921658039093018, + "delta_nll": 2.8329286575317383, + "max_kl": 9.15943717956543, + "mean_kl": 2.777693033218384, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.27941176295280457 + }, + { + "candidate_nll": 2.745262622833252, + "code_tokens": 17, + "cvar95_kl": 3.9664249420166016, + "delta_nll": 1.499106526374817, + "max_kl": 3.9664249420166016, + "mean_kl": 1.7413907051086426, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.782144069671631, + "code_tokens": 33, + "cvar95_kl": 11.672353744506836, + "delta_nll": 3.9890894889831543, + "max_kl": 12.667572975158691, + "mean_kl": 3.8131673336029053, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.413050174713135, + "code_tokens": 38, + "cvar95_kl": 7.932477951049805, + "delta_nll": 3.570603370666504, + "max_kl": 8.85235595703125, + "mean_kl": 3.4632370471954346, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 3.7270121574401855, + "code_tokens": 61, + "cvar95_kl": 8.030855178833008, + "delta_nll": 3.3294875621795654, + "max_kl": 8.521870613098145, + "mean_kl": 3.3028805255889893, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.2786885201931 + }, + { + "candidate_nll": 3.495051145553589, + "code_tokens": 143, + "cvar95_kl": 6.926228046417236, + "delta_nll": 2.759331703186035, + "max_kl": 8.085725784301758, + "mean_kl": 2.874425172805786, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.32867133617401123 + }, + { + "candidate_nll": 3.268603801727295, + "code_tokens": 41, + "cvar95_kl": 8.498809814453125, + "delta_nll": 2.179537773132324, + "max_kl": 9.19472599029541, + "mean_kl": 2.593693256378174, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.4390243887901306 + }, + { + "candidate_nll": 2.8142483234405518, + "code_tokens": 67, + "cvar95_kl": 9.52180004119873, + "delta_nll": 2.449566602706909, + "max_kl": 10.061285018920898, + "mean_kl": 2.6396186351776123, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.43283581733703613 + }, + { + "candidate_nll": 4.251237392425537, + "code_tokens": 76, + "cvar95_kl": 8.063348770141602, + "delta_nll": 2.8609774112701416, + "max_kl": 10.707235336303711, + "mean_kl": 2.739410400390625, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 3.3152518272399902, + "code_tokens": 137, + "cvar95_kl": 7.228533744812012, + "delta_nll": 2.679399251937866, + "max_kl": 8.430209159851074, + "mean_kl": 2.609990119934082, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.40145984292030334 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 4 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 4.8394551277160645, + "median": 2.7445207834243774, + "minimum": 0.7190301418304443, + "q25": 2.2812315225601196, + "q75": 3.2263612151145935 + }, + "task_macro": { + "candidate_nll": 3.7542882390022276, + "delta_nll": 2.7402416603565216, + "mean_kl": 2.823219413399696, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.3605463787317276 + }, + "token_weighted": { + "candidate_nll": 3.7894246578216553, + "cvar95_kl": 8.718489646911621, + "delta_nll": 2.872587203979492, + "max_kl": 17.669700622558594, + "mean_kl": 2.9325098991394043, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.34982144832611084 + } + }, + "read_risk_l0_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 0.5702063694000244, + "macro_mean_kl": 0.5921501630544662, + "macro_top1_agreement": 0.7581791734695434, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 0.7392329616546631, + "macro_mean_kl": 0.7701967484951019, + "macro_top1_agreement": 0.6997111868858338, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 0.8596235785484314, + "macro_mean_kl": 0.869078378200531, + "macro_top1_agreement": 0.6703913822174072, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 1.0457885901927948, + "macro_mean_kl": 1.030025339603424, + "macro_top1_agreement": 0.6364823293685913, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 1.437837839126587, + "code_tokens": 98, + "cvar95_kl": 4.351431846618652, + "delta_nll": 0.7109821438789368, + "max_kl": 6.220755577087402, + "mean_kl": 0.8511053323745728, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.7448979616165161 + }, + { + "candidate_nll": 1.5275182723999023, + "code_tokens": 19, + "cvar95_kl": 2.3682641983032227, + "delta_nll": 0.3681051731109619, + "max_kl": 2.3682641983032227, + "mean_kl": 0.4774366319179535, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.7894737124443054 + }, + { + "candidate_nll": 1.8732476234436035, + "code_tokens": 39, + "cvar95_kl": 5.275177955627441, + "delta_nll": 0.8578983545303345, + "max_kl": 6.5553202629089355, + "mean_kl": 0.91459721326828, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.731768012046814, + "code_tokens": 24, + "cvar95_kl": 2.365508556365967, + "delta_nll": 0.2854783535003662, + "max_kl": 2.484398126602173, + "mean_kl": 0.74074387550354, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.661340355873108, + "code_tokens": 25, + "cvar95_kl": 1.4808666706085205, + "delta_nll": 0.2535820007324219, + "max_kl": 1.9732763767242432, + "mean_kl": 0.33196744322776794, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.5109857320785522, + "code_tokens": 55, + "cvar95_kl": 3.3475801944732666, + "delta_nll": 0.6153741478919983, + "max_kl": 4.112247467041016, + "mean_kl": 0.6234012842178345, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 1.6018259525299072, + "code_tokens": 16, + "cvar95_kl": 3.564999580383301, + "delta_nll": 0.7723853588104248, + "max_kl": 3.564999580383301, + "mean_kl": 0.6987718343734741, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.515711545944214, + "code_tokens": 199, + "cvar95_kl": 6.729245662689209, + "delta_nll": 1.2938181161880493, + "max_kl": 11.558697700500488, + "mean_kl": 1.3418058156967163, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.5929648280143738 + }, + { + "candidate_nll": 2.3652708530426025, + "code_tokens": 28, + "cvar95_kl": 2.6924314498901367, + "delta_nll": 0.7991927862167358, + "max_kl": 3.071814775466919, + "mean_kl": 0.6068517565727234, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5663589239120483, + "code_tokens": 87, + "cvar95_kl": 3.9407448768615723, + "delta_nll": 0.6933763027191162, + "max_kl": 5.059126377105713, + "mean_kl": 0.7755260467529297, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.7011494040489197 + }, + { + "candidate_nll": 1.7174787521362305, + "code_tokens": 35, + "cvar95_kl": 3.3629608154296875, + "delta_nll": 0.48451459407806396, + "max_kl": 3.828634738922119, + "mean_kl": 0.7272599339485168, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.6285714507102966 + }, + { + "candidate_nll": 1.5806151628494263, + "code_tokens": 62, + "cvar95_kl": 3.7029905319213867, + "delta_nll": 0.9413235783576965, + "max_kl": 4.377253532409668, + "mean_kl": 0.8842069506645203, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 2.112473249435425, + "code_tokens": 58, + "cvar95_kl": 5.068362712860107, + "delta_nll": 1.007415533065796, + "max_kl": 6.522302150726318, + "mean_kl": 0.976349949836731, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.6896551847457886 + }, + { + "candidate_nll": 2.333246946334839, + "code_tokens": 79, + "cvar95_kl": 6.9539594650268555, + "delta_nll": 1.1741987466812134, + "max_kl": 8.396726608276367, + "mean_kl": 1.4176898002624512, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.5316455960273743 + }, + { + "candidate_nll": 1.8327233791351318, + "code_tokens": 58, + "cvar95_kl": 4.431029796600342, + "delta_nll": 0.7396035194396973, + "max_kl": 5.23704719543457, + "mean_kl": 0.8934630155563354, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 2.3810391426086426, + "code_tokens": 48, + "cvar95_kl": 5.999946594238281, + "delta_nll": 1.218841314315796, + "max_kl": 6.224644660949707, + "mean_kl": 1.4035091400146484, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.5208333134651184 + }, + { + "candidate_nll": 1.9071476459503174, + "code_tokens": 37, + "cvar95_kl": 3.333885669708252, + "delta_nll": 0.7817151546478271, + "max_kl": 4.774967193603516, + "mean_kl": 0.6543330550193787, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.6756756901741028 + }, + { + "candidate_nll": 1.3782262802124023, + "code_tokens": 61, + "cvar95_kl": 3.308793544769287, + "delta_nll": 0.670819103717804, + "max_kl": 4.517922878265381, + "mean_kl": 0.679639995098114, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.6721311211585999 + }, + { + "candidate_nll": 1.6747642755508423, + "code_tokens": 77, + "cvar95_kl": 3.6212124824523926, + "delta_nll": 0.9050447344779968, + "max_kl": 4.1913862228393555, + "mean_kl": 0.8814529776573181, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 2.3024284839630127, + "code_tokens": 98, + "cvar95_kl": 6.912336826324463, + "delta_nll": 1.280725359916687, + "max_kl": 9.342297554016113, + "mean_kl": 1.227766513824463, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.6224489808082581 + }, + { + "candidate_nll": 1.940861701965332, + "code_tokens": 162, + "cvar95_kl": 5.761036396026611, + "delta_nll": 1.355710506439209, + "max_kl": 7.307144641876221, + "mean_kl": 1.4084795713424683, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.5370370149612427 + }, + { + "candidate_nll": 1.9787956476211548, + "code_tokens": 103, + "cvar95_kl": 4.169778347015381, + "delta_nll": 1.2343297004699707, + "max_kl": 4.880768775939941, + "mean_kl": 1.2213464975357056, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.6310679316520691 + }, + { + "candidate_nll": 2.2950422763824463, + "code_tokens": 73, + "cvar95_kl": 3.780155658721924, + "delta_nll": 0.976071834564209, + "max_kl": 5.071652412414551, + "mean_kl": 0.7210307717323303, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.7397260069847107 + }, + { + "candidate_nll": 2.143333673477173, + "code_tokens": 119, + "cvar95_kl": 4.810328960418701, + "delta_nll": 1.1603600978851318, + "max_kl": 6.720498561859131, + "mean_kl": 1.1632674932479858, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.5630252361297607 + }, + { + "candidate_nll": 1.3111584186553955, + "code_tokens": 17, + "cvar95_kl": 4.171998977661133, + "delta_nll": 0.1216350793838501, + "max_kl": 4.171998977661133, + "mean_kl": 0.7182435393333435, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 2.308159589767456, + "code_tokens": 56, + "cvar95_kl": 4.133174419403076, + "delta_nll": 0.8824578523635864, + "max_kl": 5.817321300506592, + "mean_kl": 0.8873200416564941, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.6607142686843872 + }, + { + "candidate_nll": 1.2595347166061401, + "code_tokens": 57, + "cvar95_kl": 5.3571457862854, + "delta_nll": 0.792174220085144, + "max_kl": 7.418487548828125, + "mean_kl": 0.7693098187446594, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.8070175647735596 + }, + { + "candidate_nll": 1.3133705854415894, + "code_tokens": 63, + "cvar95_kl": 3.237618923187256, + "delta_nll": 0.6806253790855408, + "max_kl": 3.6589512825012207, + "mean_kl": 0.6985809206962585, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 2.0654091835021973, + "code_tokens": 200, + "cvar95_kl": 5.61568546295166, + "delta_nll": 1.2274736166000366, + "max_kl": 8.750672340393066, + "mean_kl": 1.217779517173767, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.5799999833106995 + }, + { + "candidate_nll": 1.554159164428711, + "code_tokens": 30, + "cvar95_kl": 3.463693380355835, + "delta_nll": 0.46995651721954346, + "max_kl": 4.301401615142822, + "mean_kl": 0.6157476902008057, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.8970035314559937, + "code_tokens": 28, + "cvar95_kl": 5.645899295806885, + "delta_nll": 1.2806525230407715, + "max_kl": 5.692856788635254, + "mean_kl": 1.0834671258926392, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.7571412324905396, + "code_tokens": 26, + "cvar95_kl": 5.2432074546813965, + "delta_nll": 0.9861845970153809, + "max_kl": 6.532666206359863, + "mean_kl": 0.9541648626327515, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.5769230723381042 + }, + { + "candidate_nll": 1.5984139442443848, + "code_tokens": 54, + "cvar95_kl": 3.7895619869232178, + "delta_nll": 0.7654725909233093, + "max_kl": 4.494767189025879, + "mean_kl": 0.7355435490608215, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.6818748712539673, + "code_tokens": 47, + "cvar95_kl": 4.013880252838135, + "delta_nll": 0.7251387238502502, + "max_kl": 6.493446350097656, + "mean_kl": 0.7611171007156372, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.7021276354789734 + }, + { + "candidate_nll": 1.8551881313323975, + "code_tokens": 93, + "cvar95_kl": 5.879759788513184, + "delta_nll": 1.1206586360931396, + "max_kl": 9.46138858795166, + "mean_kl": 1.075543999671936, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.2261435985565186, + "code_tokens": 30, + "cvar95_kl": 2.757434129714966, + "delta_nll": 0.366769015789032, + "max_kl": 3.489267110824585, + "mean_kl": 0.4687761664390564, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.7666666507720947 + }, + { + "candidate_nll": 2.0647683143615723, + "code_tokens": 89, + "cvar95_kl": 5.481202125549316, + "delta_nll": 0.9962650537490845, + "max_kl": 7.952178955078125, + "mean_kl": 1.0418976545333862, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.6629213690757751 + }, + { + "candidate_nll": 1.628982424736023, + "code_tokens": 70, + "cvar95_kl": 3.3860714435577393, + "delta_nll": 0.6647331118583679, + "max_kl": 4.6697587966918945, + "mean_kl": 0.8595913648605347, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 2.0597481727600098, + "code_tokens": 26, + "cvar95_kl": 2.989271879196167, + "delta_nll": 0.4465210437774658, + "max_kl": 3.2768750190734863, + "mean_kl": 0.6326386332511902, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.7307692170143127 + }, + { + "candidate_nll": 2.046163558959961, + "code_tokens": 48, + "cvar95_kl": 3.5074586868286133, + "delta_nll": 1.088456630706787, + "max_kl": 4.420538425445557, + "mean_kl": 0.8607252240180969, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.6458333134651184 + }, + { + "candidate_nll": 1.3034740686416626, + "code_tokens": 29, + "cvar95_kl": 2.1741554737091064, + "delta_nll": 0.43432605266571045, + "max_kl": 2.288686513900757, + "mean_kl": 0.4486943185329437, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.8275862336158752 + }, + { + "candidate_nll": 1.6809512376785278, + "code_tokens": 17, + "cvar95_kl": 2.059248685836792, + "delta_nll": 0.5677881240844727, + "max_kl": 2.059248685836792, + "mean_kl": 0.5237215757369995, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.4593381881713867, + "code_tokens": 37, + "cvar95_kl": 2.116163730621338, + "delta_nll": 0.5391220450401306, + "max_kl": 2.314948081970215, + "mean_kl": 0.49860671162605286, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 1.5211553573608398, + "code_tokens": 121, + "cvar95_kl": 5.004239559173584, + "delta_nll": 1.009753704071045, + "max_kl": 6.1679911613464355, + "mean_kl": 0.9638730883598328, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.6611570119857788 + }, + { + "candidate_nll": 2.239880084991455, + "code_tokens": 30, + "cvar95_kl": 3.435619592666626, + "delta_nll": 0.841036319732666, + "max_kl": 4.433403491973877, + "mean_kl": 0.9730808734893799, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.5666666626930237 + }, + { + "candidate_nll": 2.0859503746032715, + "code_tokens": 74, + "cvar95_kl": 5.128108978271484, + "delta_nll": 1.1874786615371704, + "max_kl": 6.625497817993164, + "mean_kl": 1.0472651720046997, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 2.1283061504364014, + "code_tokens": 53, + "cvar95_kl": 4.611486911773682, + "delta_nll": 1.2938823699951172, + "max_kl": 5.384513854980469, + "mean_kl": 1.2723076343536377, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.6415094137191772 + }, + { + "candidate_nll": 2.0187103748321533, + "code_tokens": 21, + "cvar95_kl": 0.8156514167785645, + "delta_nll": 0.15931463241577148, + "max_kl": 0.8530668616294861, + "mean_kl": 0.2385498285293579, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.9047619104385376 + }, + { + "candidate_nll": 1.6533855199813843, + "code_tokens": 20, + "cvar95_kl": 1.4527795314788818, + "delta_nll": 0.29097187519073486, + "max_kl": 1.4527795314788818, + "mean_kl": 0.354911744594574, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.9091107845306396, + "code_tokens": 122, + "cvar95_kl": 4.473572731018066, + "delta_nll": 1.0901124477386475, + "max_kl": 5.267557144165039, + "mean_kl": 1.1033142805099487, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.6147540807723999 + }, + { + "candidate_nll": 1.7527860403060913, + "code_tokens": 94, + "cvar95_kl": 4.880742073059082, + "delta_nll": 0.9126462340354919, + "max_kl": 6.542252063751221, + "mean_kl": 0.8952356576919556, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.6808510422706604 + }, + { + "candidate_nll": 1.2744691371917725, + "code_tokens": 12, + "cvar95_kl": 0.8177033066749573, + "delta_nll": 0.20916759967803955, + "max_kl": 0.8177033066749573, + "mean_kl": 0.18918752670288086, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.141298532485962, + "code_tokens": 38, + "cvar95_kl": 4.041401386260986, + "delta_nll": 0.63489830493927, + "max_kl": 4.969882965087891, + "mean_kl": 0.7539941668510437, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 1.2633423805236816, + "code_tokens": 58, + "cvar95_kl": 1.7498207092285156, + "delta_nll": 0.35855525732040405, + "max_kl": 2.4351818561553955, + "mean_kl": 0.32653743028640747, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.8275862336158752 + }, + { + "candidate_nll": 1.620820164680481, + "code_tokens": 54, + "cvar95_kl": 5.895735263824463, + "delta_nll": 1.0245563983917236, + "max_kl": 8.724684715270996, + "mean_kl": 0.9615625739097595, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.5960285663604736, + "code_tokens": 40, + "cvar95_kl": 3.86961030960083, + "delta_nll": 0.664488673210144, + "max_kl": 4.1336798667907715, + "mean_kl": 0.739889919757843, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.7749999761581421 + }, + { + "candidate_nll": 1.4100948572158813, + "code_tokens": 115, + "cvar95_kl": 3.976778268814087, + "delta_nll": 0.8975489139556885, + "max_kl": 4.926577568054199, + "mean_kl": 0.849139392375946, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.7043478488922119 + }, + { + "candidate_nll": 1.330396056175232, + "code_tokens": 60, + "cvar95_kl": 5.183268070220947, + "delta_nll": 0.8057823777198792, + "max_kl": 5.726524353027344, + "mean_kl": 0.870845377445221, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.7446472644805908, + "code_tokens": 124, + "cvar95_kl": 4.777351379394531, + "delta_nll": 0.9746601581573486, + "max_kl": 7.386229515075684, + "mean_kl": 0.916567862033844, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.6048387289047241 + }, + { + "candidate_nll": 1.522768497467041, + "code_tokens": 91, + "cvar95_kl": 3.4123008251190186, + "delta_nll": 0.7520198225975037, + "max_kl": 4.66524076461792, + "mean_kl": 0.7261348366737366, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.7362637519836426 + }, + { + "candidate_nll": 2.352339029312134, + "code_tokens": 125, + "cvar95_kl": 6.275059700012207, + "delta_nll": 1.532421588897705, + "max_kl": 7.9018120765686035, + "mean_kl": 1.4746301174163818, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.527999997138977 + }, + { + "candidate_nll": 1.540420413017273, + "code_tokens": 26, + "cvar95_kl": 1.7736355066299438, + "delta_nll": 0.4045438766479492, + "max_kl": 1.8003995418548584, + "mean_kl": 0.4136735796928406, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 2.790148973464966, + "code_tokens": 24, + "cvar95_kl": 2.1047091484069824, + "delta_nll": 0.5497238636016846, + "max_kl": 2.5175137519836426, + "mean_kl": 0.5278720855712891, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.6732341051101685, + "code_tokens": 180, + "cvar95_kl": 3.640596628189087, + "delta_nll": 0.8190522789955139, + "max_kl": 6.573902606964111, + "mean_kl": 0.8366333842277527, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 2.4713895320892334, + "code_tokens": 40, + "cvar95_kl": 6.343910217285156, + "delta_nll": 1.096580982208252, + "max_kl": 7.595676898956299, + "mean_kl": 1.192630648612976, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.732940435409546, + "code_tokens": 72, + "cvar95_kl": 4.269186496734619, + "delta_nll": 0.7579435706138611, + "max_kl": 5.194479465484619, + "mean_kl": 0.8687319755554199, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 1.483136773109436, + "code_tokens": 20, + "cvar95_kl": 1.9894614219665527, + "delta_nll": -0.09443438053131104, + "max_kl": 1.9894614219665527, + "mean_kl": 0.39476698637008667, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5741811990737915, + "code_tokens": 76, + "cvar95_kl": 2.7298948764801025, + "delta_nll": 0.5835368633270264, + "max_kl": 4.470555782318115, + "mean_kl": 0.5563408136367798, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.8639322519302368, + "code_tokens": 42, + "cvar95_kl": 1.9470844268798828, + "delta_nll": 0.4921908378601074, + "max_kl": 3.8022232055664062, + "mean_kl": 0.4195159375667572, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 0.9814545512199402, + "code_tokens": 28, + "cvar95_kl": 2.935736656188965, + "delta_nll": 0.3982313871383667, + "max_kl": 4.827160358428955, + "mean_kl": 0.3858254551887512, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.871192216873169, + "code_tokens": 58, + "cvar95_kl": 4.539680004119873, + "delta_nll": 0.8377153873443604, + "max_kl": 4.981967449188232, + "mean_kl": 0.7963540554046631, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.3158167600631714, + "code_tokens": 25, + "cvar95_kl": 2.733332633972168, + "delta_nll": 0.4471673369407654, + "max_kl": 4.223751544952393, + "mean_kl": 0.43643128871917725, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.7155133485794067, + "code_tokens": 88, + "cvar95_kl": 3.522193193435669, + "delta_nll": 0.7505055069923401, + "max_kl": 3.939423084259033, + "mean_kl": 0.7995510101318359, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.6955205202102661, + "code_tokens": 42, + "cvar95_kl": 1.7402547597885132, + "delta_nll": 0.5680032968521118, + "max_kl": 2.30548095703125, + "mean_kl": 0.4710255563259125, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 1.5900367498397827, + "code_tokens": 24, + "cvar95_kl": 3.362295627593994, + "delta_nll": 0.7704859972000122, + "max_kl": 5.007962226867676, + "mean_kl": 0.5839657187461853, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 1.0337450504302979, + "code_tokens": 24, + "cvar95_kl": 1.7097326517105103, + "delta_nll": 0.34887367486953735, + "max_kl": 1.820784091949463, + "mean_kl": 0.42712411284446716, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.9986655712127686, + "code_tokens": 43, + "cvar95_kl": 3.490222215652466, + "delta_nll": 0.9314864873886108, + "max_kl": 4.165478706359863, + "mean_kl": 0.6279502511024475, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.7906976938247681 + }, + { + "candidate_nll": 1.4780582189559937, + "code_tokens": 24, + "cvar95_kl": 2.7348403930664062, + "delta_nll": 0.591838538646698, + "max_kl": 2.764106512069702, + "mean_kl": 0.62176513671875, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.7916666865348816 + }, + { + "candidate_nll": 1.3939669132232666, + "code_tokens": 14, + "cvar95_kl": 2.5942883491516113, + "delta_nll": 0.08134126663208008, + "max_kl": 2.5942883491516113, + "mean_kl": 0.34973660111427307, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.7820640802383423, + "code_tokens": 41, + "cvar95_kl": 4.869851589202881, + "delta_nll": 1.2910208702087402, + "max_kl": 5.4240617752075195, + "mean_kl": 1.1749684810638428, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.5853658318519592 + }, + { + "candidate_nll": 1.5149013996124268, + "code_tokens": 32, + "cvar95_kl": 2.0438811779022217, + "delta_nll": 0.5265614986419678, + "max_kl": 2.6868960857391357, + "mean_kl": 0.5388344526290894, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.728800892829895, + "code_tokens": 53, + "cvar95_kl": 5.352388858795166, + "delta_nll": 0.9671059846878052, + "max_kl": 6.601135730743408, + "mean_kl": 0.8195364475250244, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.7358490824699402 + }, + { + "candidate_nll": 1.0845825672149658, + "code_tokens": 51, + "cvar95_kl": 3.7059097290039062, + "delta_nll": 0.7062914371490479, + "max_kl": 4.448524475097656, + "mean_kl": 0.6864140629768372, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.7254902124404907 + }, + { + "candidate_nll": 2.463548183441162, + "code_tokens": 36, + "cvar95_kl": 3.073725700378418, + "delta_nll": 0.771392822265625, + "max_kl": 3.097757339477539, + "mean_kl": 0.7697619795799255, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 2.062177896499634, + "code_tokens": 24, + "cvar95_kl": 1.7952526807785034, + "delta_nll": 0.5014256238937378, + "max_kl": 2.6227877140045166, + "mean_kl": 0.3531194031238556, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.159846067428589, + "code_tokens": 39, + "cvar95_kl": 4.348177909851074, + "delta_nll": 0.8455116748809814, + "max_kl": 4.41887092590332, + "mean_kl": 1.273642659187317, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.5897436141967773 + }, + { + "candidate_nll": 1.7310853004455566, + "code_tokens": 84, + "cvar95_kl": 4.761188507080078, + "delta_nll": 1.032672643661499, + "max_kl": 6.709907531738281, + "mean_kl": 1.0695093870162964, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.6309523582458496 + }, + { + "candidate_nll": 1.6105939149856567, + "code_tokens": 32, + "cvar95_kl": 3.1254615783691406, + "delta_nll": 0.6878256797790527, + "max_kl": 3.208247661590576, + "mean_kl": 0.6040952801704407, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.71875 + }, + { + "candidate_nll": 1.84221613407135, + "code_tokens": 19, + "cvar95_kl": 3.2909109592437744, + "delta_nll": 0.5320525169372559, + "max_kl": 3.2909109592437744, + "mean_kl": 0.6661286354064941, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.5683282613754272, + "code_tokens": 50, + "cvar95_kl": 4.090367794036865, + "delta_nll": 0.7503514885902405, + "max_kl": 6.3829026222229, + "mean_kl": 0.8958125114440918, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.6200000047683716 + }, + { + "candidate_nll": 1.689907431602478, + "code_tokens": 71, + "cvar95_kl": 5.390604019165039, + "delta_nll": 1.190093755722046, + "max_kl": 5.7891316413879395, + "mean_kl": 1.200129747390747, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.6478873491287231 + }, + { + "candidate_nll": 2.016000270843506, + "code_tokens": 28, + "cvar95_kl": 3.39534592628479, + "delta_nll": 0.628031849861145, + "max_kl": 3.703434944152832, + "mean_kl": 0.6244677901268005, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.5485821962356567, + "code_tokens": 76, + "cvar95_kl": 3.133467674255371, + "delta_nll": 0.6793208122253418, + "max_kl": 4.241065502166748, + "mean_kl": 0.678875744342804, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 1.839333176612854, + "code_tokens": 34, + "cvar95_kl": 3.196134567260742, + "delta_nll": 0.8767479658126831, + "max_kl": 3.306941032409668, + "mean_kl": 0.624768078327179, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.469204902648926, + "code_tokens": 11, + "cvar95_kl": 1.7996630668640137, + "delta_nll": 0.32181310653686523, + "max_kl": 1.7996630668640137, + "mean_kl": 0.46649789810180664, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 2.114027500152588, + "code_tokens": 31, + "cvar95_kl": 2.774495840072632, + "delta_nll": 0.6977599859237671, + "max_kl": 2.913696765899658, + "mean_kl": 0.7990099191665649, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.568469524383545, + "code_tokens": 118, + "cvar95_kl": 3.1507060527801514, + "delta_nll": 0.7216932773590088, + "max_kl": 4.130584716796875, + "mean_kl": 0.7494707107543945, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.6610169410705566 + }, + { + "candidate_nll": 1.1507320404052734, + "code_tokens": 60, + "cvar95_kl": 3.0679829120635986, + "delta_nll": 0.375036358833313, + "max_kl": 3.3976972103118896, + "mean_kl": 0.5510272979736328, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.7833333611488342 + }, + { + "candidate_nll": 1.5297048091888428, + "code_tokens": 47, + "cvar95_kl": 3.4964656829833984, + "delta_nll": 0.6342548727989197, + "max_kl": 4.989348888397217, + "mean_kl": 0.8003712296485901, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.7234042286872864 + }, + { + "candidate_nll": 2.136024236679077, + "code_tokens": 84, + "cvar95_kl": 4.908811569213867, + "delta_nll": 0.984009861946106, + "max_kl": 8.32959270477295, + "mean_kl": 0.9612395167350769, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.6547619104385376 + }, + { + "candidate_nll": 1.7778691053390503, + "code_tokens": 27, + "cvar95_kl": 2.3641579151153564, + "delta_nll": 0.7838430404663086, + "max_kl": 2.7429707050323486, + "mean_kl": 0.5442574620246887, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 1.654711127281189, + "code_tokens": 19, + "cvar95_kl": 1.6343698501586914, + "delta_nll": 0.04280400276184082, + "max_kl": 1.6343698501586914, + "mean_kl": 0.44417592883110046, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.611034870147705, + "code_tokens": 114, + "cvar95_kl": 3.9537465572357178, + "delta_nll": 0.8380223512649536, + "max_kl": 4.154403209686279, + "mean_kl": 0.9484782218933105, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.466111183166504, + "code_tokens": 40, + "cvar95_kl": 4.551571369171143, + "delta_nll": 0.7382179498672485, + "max_kl": 5.785944938659668, + "mean_kl": 0.7726505398750305, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 2.170844078063965, + "code_tokens": 25, + "cvar95_kl": 1.830533742904663, + "delta_nll": 0.6851658821105957, + "max_kl": 1.9657474756240845, + "mean_kl": 0.47394439578056335, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.4679319858551025, + "code_tokens": 23, + "cvar95_kl": 2.166440725326538, + "delta_nll": 0.6972429156303406, + "max_kl": 2.2522311210632324, + "mean_kl": 0.5364603400230408, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.2561111450195312, + "code_tokens": 77, + "cvar95_kl": 5.846391677856445, + "delta_nll": 1.2397615909576416, + "max_kl": 6.894842147827148, + "mean_kl": 1.2531832456588745, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.8633546829223633, + "code_tokens": 19, + "cvar95_kl": 1.6350184679031372, + "delta_nll": 0.755689263343811, + "max_kl": 1.6350184679031372, + "mean_kl": 0.3480040431022644, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.3318679332733154, + "code_tokens": 37, + "cvar95_kl": 6.5147705078125, + "delta_nll": 1.3486242294311523, + "max_kl": 6.986086845397949, + "mean_kl": 1.5934216976165771, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.5135135054588318 + }, + { + "candidate_nll": 1.8219059705734253, + "code_tokens": 33, + "cvar95_kl": 2.950320243835449, + "delta_nll": 0.4387410879135132, + "max_kl": 3.499119281768799, + "mean_kl": 0.5838028788566589, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 1.571190595626831, + "code_tokens": 95, + "cvar95_kl": 5.023710250854492, + "delta_nll": 0.9113491773605347, + "max_kl": 7.053951263427734, + "mean_kl": 0.9139501452445984, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.7157894968986511 + }, + { + "candidate_nll": 2.039400815963745, + "code_tokens": 125, + "cvar95_kl": 5.159369468688965, + "delta_nll": 1.0262404680252075, + "max_kl": 7.109761714935303, + "mean_kl": 1.0650709867477417, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.6399999856948853 + }, + { + "candidate_nll": 1.925919532775879, + "code_tokens": 168, + "cvar95_kl": 5.99751091003418, + "delta_nll": 1.2128710746765137, + "max_kl": 7.38164758682251, + "mean_kl": 1.317311406135559, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.5357142686843872 + }, + { + "candidate_nll": 1.532333254814148, + "code_tokens": 31, + "cvar95_kl": 2.7582340240478516, + "delta_nll": 0.21693110466003418, + "max_kl": 3.589409351348877, + "mean_kl": 0.6136119961738586, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.8064516186714172 + }, + { + "candidate_nll": 1.9273406267166138, + "code_tokens": 83, + "cvar95_kl": 5.184304237365723, + "delta_nll": 1.1656339168548584, + "max_kl": 6.698734760284424, + "mean_kl": 1.0433247089385986, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.6746987700462341 + }, + { + "candidate_nll": 1.5729767084121704, + "code_tokens": 50, + "cvar95_kl": 4.1302056312561035, + "delta_nll": 0.8730342388153076, + "max_kl": 4.913782119750977, + "mean_kl": 0.8164222836494446, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.6399999856948853 + }, + { + "candidate_nll": 1.329505205154419, + "code_tokens": 69, + "cvar95_kl": 4.265841007232666, + "delta_nll": 0.9623232483863831, + "max_kl": 4.862369060516357, + "mean_kl": 0.9329202771186829, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.6811594367027283 + }, + { + "candidate_nll": 2.0954368114471436, + "code_tokens": 50, + "cvar95_kl": 5.004846572875977, + "delta_nll": 1.0105406045913696, + "max_kl": 6.332308292388916, + "mean_kl": 1.0917145013809204, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.081373453140259, + "code_tokens": 161, + "cvar95_kl": 5.520568370819092, + "delta_nll": 1.0775917768478394, + "max_kl": 7.0954766273498535, + "mean_kl": 1.0200989246368408, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.7018633484840393 + }, + { + "candidate_nll": 1.5112152099609375, + "code_tokens": 53, + "cvar95_kl": 4.294021129608154, + "delta_nll": 0.8554390668869019, + "max_kl": 5.576375961303711, + "mean_kl": 0.7886008620262146, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.6603773832321167 + }, + { + "candidate_nll": 1.3950626850128174, + "code_tokens": 97, + "cvar95_kl": 4.250563621520996, + "delta_nll": 0.9495346546173096, + "max_kl": 4.96079683303833, + "mean_kl": 0.8034611940383911, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.7525773048400879 + }, + { + "candidate_nll": 1.9310895204544067, + "code_tokens": 22, + "cvar95_kl": 2.032834053039551, + "delta_nll": 0.6172765493392944, + "max_kl": 2.534329891204834, + "mean_kl": 0.5720844864845276, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.5270134210586548, + "code_tokens": 34, + "cvar95_kl": 4.532155513763428, + "delta_nll": 0.6581093072891235, + "max_kl": 6.442066669464111, + "mean_kl": 0.7767821550369263, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.394924283027649, + "code_tokens": 81, + "cvar95_kl": 3.5088982582092285, + "delta_nll": 0.5362810492515564, + "max_kl": 5.037527561187744, + "mean_kl": 0.7119351625442505, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.654321014881134 + }, + { + "candidate_nll": 1.3312137126922607, + "code_tokens": 17, + "cvar95_kl": 1.5652077198028564, + "delta_nll": 0.06911039352416992, + "max_kl": 1.5652077198028564, + "mean_kl": 0.22962011396884918, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.8823529481887817 + }, + { + "candidate_nll": 1.594070315361023, + "code_tokens": 181, + "cvar95_kl": 4.279249668121338, + "delta_nll": 0.768812894821167, + "max_kl": 8.66480827331543, + "mean_kl": 0.8002496361732483, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.6850828528404236 + }, + { + "candidate_nll": 1.6760903596878052, + "code_tokens": 63, + "cvar95_kl": 3.077331066131592, + "delta_nll": 0.6956818699836731, + "max_kl": 4.401571273803711, + "mean_kl": 0.6958545446395874, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.7301587462425232 + }, + { + "candidate_nll": 0.9973301291465759, + "code_tokens": 35, + "cvar95_kl": 2.331942081451416, + "delta_nll": 0.30380702018737793, + "max_kl": 2.804950714111328, + "mean_kl": 0.41396552324295044, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.8285714387893677 + }, + { + "candidate_nll": 1.4302093982696533, + "code_tokens": 25, + "cvar95_kl": 2.184866189956665, + "delta_nll": 0.625421941280365, + "max_kl": 3.1138641834259033, + "mean_kl": 0.4299301505088806, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.9984405040740967, + "code_tokens": 60, + "cvar95_kl": 6.066128253936768, + "delta_nll": 0.9100750684738159, + "max_kl": 8.73543930053711, + "mean_kl": 0.956393837928772, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.013244867324829, + "code_tokens": 86, + "cvar95_kl": 4.19992733001709, + "delta_nll": 1.1015980243682861, + "max_kl": 6.164646148681641, + "mean_kl": 0.934320867061615, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.6976743936538696 + }, + { + "candidate_nll": 1.571995496749878, + "code_tokens": 40, + "cvar95_kl": 3.1403117179870605, + "delta_nll": 0.6458297371864319, + "max_kl": 3.3989503383636475, + "mean_kl": 0.6274427175521851, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.7250000238418579 + }, + { + "candidate_nll": 2.090998649597168, + "code_tokens": 27, + "cvar95_kl": 1.9174832105636597, + "delta_nll": 0.5274384021759033, + "max_kl": 1.9958009719848633, + "mean_kl": 0.47828900814056396, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.9251688718795776, + "code_tokens": 52, + "cvar95_kl": 5.209041595458984, + "delta_nll": 0.8355859518051147, + "max_kl": 5.464498519897461, + "mean_kl": 1.0447616577148438, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.6346153616905212 + }, + { + "candidate_nll": 1.6442748308181763, + "code_tokens": 71, + "cvar95_kl": 5.6382036209106445, + "delta_nll": 0.9024007320404053, + "max_kl": 7.8199968338012695, + "mean_kl": 0.9220471382141113, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.6901408433914185 + }, + { + "candidate_nll": 1.6607983112335205, + "code_tokens": 25, + "cvar95_kl": 4.604377269744873, + "delta_nll": 0.5589466094970703, + "max_kl": 6.188091278076172, + "mean_kl": 0.793627917766571, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.4450721740722656, + "code_tokens": 103, + "cvar95_kl": 3.9904944896698, + "delta_nll": 0.8505765199661255, + "max_kl": 4.764760494232178, + "mean_kl": 0.8784255385398865, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.6699029207229614 + }, + { + "candidate_nll": 1.8436771631240845, + "code_tokens": 116, + "cvar95_kl": 3.7996137142181396, + "delta_nll": 0.7861391305923462, + "max_kl": 6.018502712249756, + "mean_kl": 0.8208531737327576, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.6637930870056152 + }, + { + "candidate_nll": 1.7308849096298218, + "code_tokens": 107, + "cvar95_kl": 5.088382244110107, + "delta_nll": 1.0825386047363281, + "max_kl": 6.895065784454346, + "mean_kl": 1.054499864578247, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.5794392228126526 + }, + { + "candidate_nll": 1.5361276865005493, + "code_tokens": 54, + "cvar95_kl": 2.5602316856384277, + "delta_nll": 0.5930899381637573, + "max_kl": 3.058537006378174, + "mean_kl": 0.5787672996520996, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.7856906652450562, + "code_tokens": 46, + "cvar95_kl": 5.896358489990234, + "delta_nll": 1.1522982120513916, + "max_kl": 8.174508094787598, + "mean_kl": 1.1224322319030762, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.3115392923355103, + "code_tokens": 170, + "cvar95_kl": 5.511594772338867, + "delta_nll": 0.9823925495147705, + "max_kl": 8.774435043334961, + "mean_kl": 1.0058202743530273, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 2.3939850330352783, + "code_tokens": 45, + "cvar95_kl": 3.928727388381958, + "delta_nll": 0.9672927856445312, + "max_kl": 4.192455291748047, + "mean_kl": 1.0962470769882202, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 1.8299256563186646, + "code_tokens": 24, + "cvar95_kl": 2.0445573329925537, + "delta_nll": 0.42162251472473145, + "max_kl": 2.141908645629883, + "mean_kl": 0.40619850158691406, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.0432076454162598, + "code_tokens": 78, + "cvar95_kl": 3.181962013244629, + "delta_nll": 0.8812477588653564, + "max_kl": 3.667109251022339, + "mean_kl": 0.8838945031166077, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.5897436141967773 + }, + { + "candidate_nll": 1.4589956998825073, + "code_tokens": 33, + "cvar95_kl": 5.343605995178223, + "delta_nll": 0.8284386396408081, + "max_kl": 7.80642032623291, + "mean_kl": 0.7979444861412048, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 2.252589225769043, + "code_tokens": 32, + "cvar95_kl": 3.1679248809814453, + "delta_nll": 1.136919617652893, + "max_kl": 3.732572555541992, + "mean_kl": 1.0272454023361206, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 2.004063606262207, + "code_tokens": 77, + "cvar95_kl": 3.696166753768921, + "delta_nll": 1.0153138637542725, + "max_kl": 5.7238311767578125, + "mean_kl": 0.8602582216262817, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.649350643157959 + }, + { + "candidate_nll": 0.8638460040092468, + "code_tokens": 222, + "cvar95_kl": 2.752610445022583, + "delta_nll": 0.4415954053401947, + "max_kl": 4.615993976593018, + "mean_kl": 0.4772477149963379, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.815315306186676 + }, + { + "candidate_nll": 2.3223893642425537, + "code_tokens": 86, + "cvar95_kl": 4.934406757354736, + "delta_nll": 1.2737945318222046, + "max_kl": 6.604762077331543, + "mean_kl": 1.308000922203064, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.6395348906517029 + }, + { + "candidate_nll": 0.8021089434623718, + "code_tokens": 31, + "cvar95_kl": 2.7074379920959473, + "delta_nll": 0.4680594503879547, + "max_kl": 3.5851571559906006, + "mean_kl": 0.47544974088668823, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.7419354915618896 + }, + { + "candidate_nll": 1.103749394416809, + "code_tokens": 34, + "cvar95_kl": 3.2629623413085938, + "delta_nll": 0.5921493172645569, + "max_kl": 4.271722316741943, + "mean_kl": 0.5829216241836548, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.1370041370391846, + "code_tokens": 38, + "cvar95_kl": 3.715728998184204, + "delta_nll": 0.7495437860488892, + "max_kl": 4.9643402099609375, + "mean_kl": 0.724778950214386, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 2.0550122261047363, + "code_tokens": 93, + "cvar95_kl": 3.579421281814575, + "delta_nll": 1.0919675827026367, + "max_kl": 4.614491939544678, + "mean_kl": 0.9615810513496399, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.838106393814087, + "code_tokens": 70, + "cvar95_kl": 4.960568428039551, + "delta_nll": 1.0595006942749023, + "max_kl": 5.538738250732422, + "mean_kl": 1.0343478918075562, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.691083550453186, + "code_tokens": 67, + "cvar95_kl": 3.2460126876831055, + "delta_nll": 0.8805915117263794, + "max_kl": 3.755563735961914, + "mean_kl": 0.7991440296173096, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.6865671873092651 + }, + { + "candidate_nll": 2.3151888847351074, + "code_tokens": 66, + "cvar95_kl": 4.5163254737854, + "delta_nll": 1.0290615558624268, + "max_kl": 6.432679176330566, + "mean_kl": 0.9873060584068298, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.5757575631141663 + }, + { + "candidate_nll": 1.7566393613815308, + "code_tokens": 34, + "cvar95_kl": 3.181601047515869, + "delta_nll": 0.7443451881408691, + "max_kl": 3.2493536472320557, + "mean_kl": 0.6402506232261658, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.6176470518112183 + }, + { + "candidate_nll": 1.8556638956069946, + "code_tokens": 64, + "cvar95_kl": 3.943228244781494, + "delta_nll": 0.9456385374069214, + "max_kl": 4.87177848815918, + "mean_kl": 0.9277783036231995, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.609375 + }, + { + "candidate_nll": 2.1626455783843994, + "code_tokens": 56, + "cvar95_kl": 6.1824822425842285, + "delta_nll": 0.8522900342941284, + "max_kl": 7.676549911499023, + "mean_kl": 0.9068112373352051, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.6861999034881592, + "code_tokens": 21, + "cvar95_kl": 1.332353115081787, + "delta_nll": 0.5611175298690796, + "max_kl": 1.4406205415725708, + "mean_kl": 0.448677122592926, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.5580421686172485, + "code_tokens": 60, + "cvar95_kl": 4.17443323135376, + "delta_nll": 0.8927018046379089, + "max_kl": 5.071746826171875, + "mean_kl": 0.9052857160568237, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.0340769290924072, + "code_tokens": 29, + "cvar95_kl": 3.3199303150177, + "delta_nll": 0.6803009510040283, + "max_kl": 3.7341411113739014, + "mean_kl": 0.6923015117645264, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.3338102102279663, + "code_tokens": 34, + "cvar95_kl": 3.5734474658966064, + "delta_nll": 0.6117653250694275, + "max_kl": 4.77994441986084, + "mean_kl": 0.6395746469497681, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 2.3149831295013428, + "code_tokens": 88, + "cvar95_kl": 5.636975288391113, + "delta_nll": 1.3958756923675537, + "max_kl": 10.449828147888184, + "mean_kl": 1.3487963676452637, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.5568181872367859 + }, + { + "candidate_nll": 1.212145447731018, + "code_tokens": 17, + "cvar95_kl": 2.9401886463165283, + "delta_nll": 0.6689218282699585, + "max_kl": 2.9401886463165283, + "mean_kl": 0.5465613007545471, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 2.4588000774383545, + "code_tokens": 35, + "cvar95_kl": 1.3194866180419922, + "delta_nll": 0.5682529211044312, + "max_kl": 1.352709412574768, + "mean_kl": 0.5214791893959045, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.357059359550476, + "code_tokens": 48, + "cvar95_kl": 1.8637690544128418, + "delta_nll": 0.46728283166885376, + "max_kl": 1.9415417909622192, + "mean_kl": 0.45307672023773193, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.7291666865348816 + }, + { + "candidate_nll": 1.9790363311767578, + "code_tokens": 130, + "cvar95_kl": 4.627747535705566, + "delta_nll": 0.9887639284133911, + "max_kl": 6.140337944030762, + "mean_kl": 0.9725895524024963, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.607692301273346 + }, + { + "candidate_nll": 3.0495872497558594, + "code_tokens": 65, + "cvar95_kl": 7.1827921867370605, + "delta_nll": 1.9546477794647217, + "max_kl": 8.93212890625, + "mean_kl": 1.8015974760055542, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.4153846204280853 + }, + { + "candidate_nll": 2.235452175140381, + "code_tokens": 145, + "cvar95_kl": 5.031227111816406, + "delta_nll": 1.3604371547698975, + "max_kl": 6.25171422958374, + "mean_kl": 1.239142656326294, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.5586206912994385 + }, + { + "candidate_nll": 1.0810580253601074, + "code_tokens": 80, + "cvar95_kl": 4.0204315185546875, + "delta_nll": 0.36535871028900146, + "max_kl": 5.251855850219727, + "mean_kl": 0.5212334990501404, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.8374999761581421 + }, + { + "candidate_nll": 1.7141131162643433, + "code_tokens": 70, + "cvar95_kl": 5.562346935272217, + "delta_nll": 1.1319763660430908, + "max_kl": 7.40985107421875, + "mean_kl": 1.2132337093353271, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.9817008972167969, + "code_tokens": 25, + "cvar95_kl": 2.557723045349121, + "delta_nll": 0.4934118986129761, + "max_kl": 2.932297945022583, + "mean_kl": 0.5099629163742065, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 2.5396459102630615, + "code_tokens": 50, + "cvar95_kl": 6.171702861785889, + "delta_nll": 0.8852624893188477, + "max_kl": 7.636078357696533, + "mean_kl": 1.166253924369812, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.46000000834465027 + }, + { + "candidate_nll": 1.2731434106826782, + "code_tokens": 48, + "cvar95_kl": 2.3191959857940674, + "delta_nll": 0.47510045766830444, + "max_kl": 2.891127586364746, + "mean_kl": 0.4860738515853882, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.7708333134651184 + }, + { + "candidate_nll": 1.4226531982421875, + "code_tokens": 104, + "cvar95_kl": 6.255232334136963, + "delta_nll": 1.0785808563232422, + "max_kl": 8.623175621032715, + "mean_kl": 1.032798171043396, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.7211538553237915 + }, + { + "candidate_nll": 1.9666929244995117, + "code_tokens": 68, + "cvar95_kl": 3.412477493286133, + "delta_nll": 0.976698637008667, + "max_kl": 4.074804306030273, + "mean_kl": 0.8548634648323059, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 1.494521975517273, + "code_tokens": 145, + "cvar95_kl": 3.5522561073303223, + "delta_nll": 0.7396098375320435, + "max_kl": 4.686226844787598, + "mean_kl": 0.7456536889076233, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.7172414064407349 + }, + { + "candidate_nll": 1.237764596939087, + "code_tokens": 41, + "cvar95_kl": 2.923518180847168, + "delta_nll": 0.3928768038749695, + "max_kl": 4.6337385177612305, + "mean_kl": 0.5110464692115784, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.7804877758026123 + }, + { + "candidate_nll": 1.5545239448547363, + "code_tokens": 53, + "cvar95_kl": 3.6254470348358154, + "delta_nll": 0.7934016585350037, + "max_kl": 3.9424192905426025, + "mean_kl": 0.7189407348632812, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.7169811129570007 + }, + { + "candidate_nll": 1.6420966386795044, + "code_tokens": 58, + "cvar95_kl": 3.273249387741089, + "delta_nll": 0.8498668670654297, + "max_kl": 3.664853096008301, + "mean_kl": 0.7556498050689697, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.6896551847457886 + }, + { + "candidate_nll": 2.3292598724365234, + "code_tokens": 24, + "cvar95_kl": 4.473161220550537, + "delta_nll": 0.5971997976303101, + "max_kl": 4.638923168182373, + "mean_kl": 0.7550384998321533, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.207904100418091, + "code_tokens": 89, + "cvar95_kl": 5.26861572265625, + "delta_nll": 1.216050148010254, + "max_kl": 7.20202112197876, + "mean_kl": 1.2106013298034668, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.6067415475845337 + }, + { + "candidate_nll": 1.6184386014938354, + "code_tokens": 101, + "cvar95_kl": 5.860593795776367, + "delta_nll": 1.1333624124526978, + "max_kl": 8.293519973754883, + "mean_kl": 1.047418475151062, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.6732673048973083 + }, + { + "candidate_nll": 2.233825206756592, + "code_tokens": 33, + "cvar95_kl": 5.080671310424805, + "delta_nll": 0.776267409324646, + "max_kl": 5.697253704071045, + "mean_kl": 0.8108587265014648, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.7304071187973022, + "code_tokens": 50, + "cvar95_kl": 4.312148571014404, + "delta_nll": 0.683741569519043, + "max_kl": 5.763941287994385, + "mean_kl": 0.7180960178375244, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.2879302501678467, + "code_tokens": 58, + "cvar95_kl": 2.9492263793945312, + "delta_nll": 0.879177451133728, + "max_kl": 3.6061527729034424, + "mean_kl": 0.677055299282074, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.6034482717514038 + }, + { + "candidate_nll": 1.9441567659378052, + "code_tokens": 62, + "cvar95_kl": 4.397705554962158, + "delta_nll": 0.9649978280067444, + "max_kl": 5.614462375640869, + "mean_kl": 0.840583086013794, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.2633821964263916, + "code_tokens": 42, + "cvar95_kl": 2.704160690307617, + "delta_nll": 0.5716922283172607, + "max_kl": 3.2994444370269775, + "mean_kl": 0.6693840622901917, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8394999504089355, + "code_tokens": 15, + "cvar95_kl": 3.4482526779174805, + "delta_nll": 0.9147589802742004, + "max_kl": 3.4482526779174805, + "mean_kl": 0.7181757092475891, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.791417121887207, + "code_tokens": 57, + "cvar95_kl": 5.377686977386475, + "delta_nll": 1.0526349544525146, + "max_kl": 6.553092956542969, + "mean_kl": 0.9371963739395142, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.6315789222717285 + }, + { + "candidate_nll": 2.2950026988983154, + "code_tokens": 58, + "cvar95_kl": 4.248537063598633, + "delta_nll": 1.2514667510986328, + "max_kl": 4.306375980377197, + "mean_kl": 1.1180880069732666, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.5862069129943848 + }, + { + "candidate_nll": 1.9169104099273682, + "code_tokens": 45, + "cvar95_kl": 5.500059604644775, + "delta_nll": 1.0449392795562744, + "max_kl": 7.531994819641113, + "mean_kl": 1.091720700263977, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.644444465637207 + }, + { + "candidate_nll": 2.7144839763641357, + "code_tokens": 32, + "cvar95_kl": 3.588292360305786, + "delta_nll": 0.6852536201477051, + "max_kl": 5.384130001068115, + "mean_kl": 0.9199109077453613, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 2.0151829719543457, + "code_tokens": 39, + "cvar95_kl": 3.4913129806518555, + "delta_nll": 0.6945165395736694, + "max_kl": 3.6355254650115967, + "mean_kl": 0.7591186761856079, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.5903136730194092, + "code_tokens": 136, + "cvar95_kl": 4.787999629974365, + "delta_nll": 1.1795523166656494, + "max_kl": 5.622316837310791, + "mean_kl": 1.1059013605117798, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.595588207244873 + }, + { + "candidate_nll": 1.8570283651351929, + "code_tokens": 50, + "cvar95_kl": 2.906562089920044, + "delta_nll": 0.6362717151641846, + "max_kl": 3.2378296852111816, + "mean_kl": 0.6789836883544922, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.714455008506775, + "code_tokens": 34, + "cvar95_kl": 3.5784621238708496, + "delta_nll": 0.8485392928123474, + "max_kl": 3.6134321689605713, + "mean_kl": 0.8952341079711914, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.7352941036224365 + }, + { + "candidate_nll": 1.6534298658370972, + "code_tokens": 44, + "cvar95_kl": 4.005340576171875, + "delta_nll": 0.4746396541595459, + "max_kl": 8.045111656188965, + "mean_kl": 0.631790280342102, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.866131067276001, + "code_tokens": 36, + "cvar95_kl": 1.3476462364196777, + "delta_nll": 0.4327092170715332, + "max_kl": 1.6028351783752441, + "mean_kl": 0.44165897369384766, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 1.4961055517196655, + "code_tokens": 46, + "cvar95_kl": 3.403928756713867, + "delta_nll": 0.6929483413696289, + "max_kl": 4.585171699523926, + "mean_kl": 0.7065553069114685, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.760869562625885 + }, + { + "candidate_nll": 2.2846481800079346, + "code_tokens": 48, + "cvar95_kl": 4.934628009796143, + "delta_nll": 1.1043082475662231, + "max_kl": 6.056484222412109, + "mean_kl": 0.9168172478675842, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.6041666865348816 + }, + { + "candidate_nll": 1.6638339757919312, + "code_tokens": 25, + "cvar95_kl": 3.2705488204956055, + "delta_nll": 0.7684229016304016, + "max_kl": 4.139795780181885, + "mean_kl": 0.5923088192939758, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.394541025161743, + "code_tokens": 69, + "cvar95_kl": 6.7038679122924805, + "delta_nll": 0.8902620077133179, + "max_kl": 10.661364555358887, + "mean_kl": 1.2651560306549072, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.49275362491607666 + }, + { + "candidate_nll": 1.3562698364257812, + "code_tokens": 60, + "cvar95_kl": 2.6273107528686523, + "delta_nll": 0.3908121585845947, + "max_kl": 3.030283212661743, + "mean_kl": 0.5341432094573975, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.8166666626930237 + }, + { + "candidate_nll": 1.6419490575790405, + "code_tokens": 49, + "cvar95_kl": 4.407739162445068, + "delta_nll": 0.7274715900421143, + "max_kl": 6.119176387786865, + "mean_kl": 0.7175212502479553, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.6938775777816772 + }, + { + "candidate_nll": 1.965149998664856, + "code_tokens": 44, + "cvar95_kl": 1.6601191759109497, + "delta_nll": 0.2646217346191406, + "max_kl": 1.9823342561721802, + "mean_kl": 0.4484603703022003, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 2.2091293334960938, + "code_tokens": 81, + "cvar95_kl": 4.895567893981934, + "delta_nll": 1.0637151002883911, + "max_kl": 6.820925712585449, + "mean_kl": 1.0465245246887207, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.7598247528076172, + "code_tokens": 26, + "cvar95_kl": 2.982357978820801, + "delta_nll": 0.32308948040008545, + "max_kl": 3.1492247581481934, + "mean_kl": 0.6875041723251343, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.3608142137527466, + "code_tokens": 32, + "cvar95_kl": 3.4163453578948975, + "delta_nll": 0.4808931350708008, + "max_kl": 4.928884506225586, + "mean_kl": 0.6390304565429688, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.78125 + }, + { + "candidate_nll": 1.8975330591201782, + "code_tokens": 48, + "cvar95_kl": 3.5240275859832764, + "delta_nll": 0.9497911334037781, + "max_kl": 4.060540676116943, + "mean_kl": 0.7865393757820129, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.978298306465149, + "code_tokens": 180, + "cvar95_kl": 5.0296783447265625, + "delta_nll": 1.2789771556854248, + "max_kl": 7.069170951843262, + "mean_kl": 1.2029547691345215, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.5777778029441833 + }, + { + "candidate_nll": 1.5074663162231445, + "code_tokens": 38, + "cvar95_kl": 3.4338808059692383, + "delta_nll": 0.7857047319412231, + "max_kl": 3.6890652179718018, + "mean_kl": 0.7474092841148376, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.5789473652839661 + }, + { + "candidate_nll": 1.934070348739624, + "code_tokens": 90, + "cvar95_kl": 6.209603309631348, + "delta_nll": 1.2390086650848389, + "max_kl": 8.92114543914795, + "mean_kl": 1.2147440910339355, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.6333333253860474 + }, + { + "candidate_nll": 1.279183268547058, + "code_tokens": 44, + "cvar95_kl": 4.104951858520508, + "delta_nll": 0.7045531868934631, + "max_kl": 4.532039642333984, + "mean_kl": 0.8774381875991821, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.6818181872367859 + }, + { + "candidate_nll": 1.5062382221221924, + "code_tokens": 60, + "cvar95_kl": 4.598969459533691, + "delta_nll": 0.7623380422592163, + "max_kl": 7.09505558013916, + "mean_kl": 0.7661399841308594, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.893372654914856, + "code_tokens": 69, + "cvar95_kl": 4.813685894012451, + "delta_nll": 0.89057457447052, + "max_kl": 7.786147594451904, + "mean_kl": 0.776037335395813, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.687326192855835, + "code_tokens": 71, + "cvar95_kl": 5.39392614364624, + "delta_nll": 1.1697643995285034, + "max_kl": 6.970792293548584, + "mean_kl": 1.147890567779541, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.6197183132171631 + }, + { + "candidate_nll": 1.697794795036316, + "code_tokens": 22, + "cvar95_kl": 2.194110870361328, + "delta_nll": 0.45512163639068604, + "max_kl": 2.9233522415161133, + "mean_kl": 0.43486425280570984, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.3446471691131592, + "code_tokens": 215, + "cvar95_kl": 3.96205997467041, + "delta_nll": 0.7908676862716675, + "max_kl": 6.131277561187744, + "mean_kl": 0.8570724129676819, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.6558139324188232 + }, + { + "candidate_nll": 1.5275791883468628, + "code_tokens": 26, + "cvar95_kl": 1.6029101610183716, + "delta_nll": 0.42478954792022705, + "max_kl": 1.9135972261428833, + "mean_kl": 0.39831897616386414, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.8846153616905212 + }, + { + "candidate_nll": 1.753525972366333, + "code_tokens": 33, + "cvar95_kl": 4.487386226654053, + "delta_nll": 0.8852792978286743, + "max_kl": 5.0195112228393555, + "mean_kl": 0.8944844603538513, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.0879502296447754, + "code_tokens": 18, + "cvar95_kl": 0.6777887940406799, + "delta_nll": 0.15180212259292603, + "max_kl": 0.6777887940406799, + "mean_kl": 0.24614855647087097, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 2.2590761184692383, + "code_tokens": 71, + "cvar95_kl": 5.509476661682129, + "delta_nll": 0.9790143966674805, + "max_kl": 6.910741329193115, + "mean_kl": 1.1907845735549927, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.6056337952613831 + }, + { + "candidate_nll": 1.4496458768844604, + "code_tokens": 107, + "cvar95_kl": 3.9019651412963867, + "delta_nll": 0.6682444214820862, + "max_kl": 4.687696933746338, + "mean_kl": 0.6866754293441772, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.7196261882781982 + }, + { + "candidate_nll": 2.8429195880889893, + "code_tokens": 40, + "cvar95_kl": 5.153518199920654, + "delta_nll": 0.7878949642181396, + "max_kl": 5.5966010093688965, + "mean_kl": 0.9146377444267273, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 2.001621723175049, + "code_tokens": 32, + "cvar95_kl": 3.4253008365631104, + "delta_nll": 0.6373248100280762, + "max_kl": 3.5417118072509766, + "mean_kl": 0.805347740650177, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.71875 + }, + { + "candidate_nll": 1.629675030708313, + "code_tokens": 130, + "cvar95_kl": 4.378066062927246, + "delta_nll": 0.9304212927818298, + "max_kl": 7.064347267150879, + "mean_kl": 0.9283219575881958, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.6461538672447205 + }, + { + "candidate_nll": 2.3776659965515137, + "code_tokens": 27, + "cvar95_kl": 4.762880802154541, + "delta_nll": 0.9062442779541016, + "max_kl": 6.56791353225708, + "mean_kl": 0.8794907331466675, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.56169855594635, + "code_tokens": 42, + "cvar95_kl": 3.5952131748199463, + "delta_nll": 0.5887582302093506, + "max_kl": 5.95373010635376, + "mean_kl": 0.6388250589370728, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 1.854259729385376, + "code_tokens": 30, + "cvar95_kl": 4.0978007316589355, + "delta_nll": 1.1087465286254883, + "max_kl": 5.7968244552612305, + "mean_kl": 0.8492130041122437, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 2.81780743598938, + "code_tokens": 58, + "cvar95_kl": 5.652854919433594, + "delta_nll": 1.3985745906829834, + "max_kl": 6.680981636047363, + "mean_kl": 1.2162731885910034, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.517241358757019 + }, + { + "candidate_nll": 1.8249861001968384, + "code_tokens": 34, + "cvar95_kl": 3.721623182296753, + "delta_nll": 0.5947588682174683, + "max_kl": 3.7677230834960938, + "mean_kl": 0.6241449117660522, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 1.677505373954773, + "code_tokens": 208, + "cvar95_kl": 4.425317287445068, + "delta_nll": 0.9760428667068481, + "max_kl": 9.65798282623291, + "mean_kl": 0.9900054335594177, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.6394230723381042 + }, + { + "candidate_nll": 1.5022534132003784, + "code_tokens": 83, + "cvar95_kl": 2.515969753265381, + "delta_nll": 0.603347659111023, + "max_kl": 3.6074225902557373, + "mean_kl": 0.7021126747131348, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.6265060305595398 + }, + { + "candidate_nll": 1.918843388557434, + "code_tokens": 170, + "cvar95_kl": 6.531036376953125, + "delta_nll": 1.3962876796722412, + "max_kl": 10.504840850830078, + "mean_kl": 1.402720332145691, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.5588235259056091 + }, + { + "candidate_nll": 1.5509626865386963, + "code_tokens": 40, + "cvar95_kl": 6.642059803009033, + "delta_nll": 0.7695421576499939, + "max_kl": 7.107378959655762, + "mean_kl": 0.8668548464775085, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.995086669921875, + "code_tokens": 31, + "cvar95_kl": 4.016458511352539, + "delta_nll": 0.7483770847320557, + "max_kl": 5.105479717254639, + "mean_kl": 0.8075739741325378, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.523214340209961, + "code_tokens": 42, + "cvar95_kl": 3.702531099319458, + "delta_nll": 0.7284443378448486, + "max_kl": 6.634225845336914, + "mean_kl": 0.7620762586593628, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.6904761791229248 + }, + { + "candidate_nll": 2.3896312713623047, + "code_tokens": 27, + "cvar95_kl": 3.555924415588379, + "delta_nll": 0.6205652952194214, + "max_kl": 4.269290924072266, + "mean_kl": 0.6885077357292175, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 2.3434765338897705, + "code_tokens": 25, + "cvar95_kl": 2.072944164276123, + "delta_nll": 0.14366602897644043, + "max_kl": 2.18044376373291, + "mean_kl": 0.6416712403297424, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.2202625274658203, + "code_tokens": 40, + "cvar95_kl": 4.748190879821777, + "delta_nll": 0.6518760919570923, + "max_kl": 4.939370155334473, + "mean_kl": 0.638741672039032, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.204289197921753, + "code_tokens": 24, + "cvar95_kl": 2.964901924133301, + "delta_nll": 0.819934606552124, + "max_kl": 3.553415298461914, + "mean_kl": 0.6125619411468506, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.3745003938674927, + "code_tokens": 28, + "cvar95_kl": 3.0922820568084717, + "delta_nll": 0.4917554259300232, + "max_kl": 3.1954879760742188, + "mean_kl": 0.5234538316726685, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.25815749168396, + "code_tokens": 71, + "cvar95_kl": 4.260236740112305, + "delta_nll": 0.7792929410934448, + "max_kl": 5.760156154632568, + "mean_kl": 0.7824870347976685, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.7323943376541138 + }, + { + "candidate_nll": 1.0163840055465698, + "code_tokens": 24, + "cvar95_kl": 1.5815180540084839, + "delta_nll": 0.4870232343673706, + "max_kl": 2.1410579681396484, + "mean_kl": 0.335100382566452, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 2.3539557456970215, + "code_tokens": 34, + "cvar95_kl": 7.563270568847656, + "delta_nll": 1.0823746919631958, + "max_kl": 8.60350513458252, + "mean_kl": 1.3974666595458984, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 2.053457736968994, + "code_tokens": 64, + "cvar95_kl": 7.214577674865723, + "delta_nll": 0.9341638088226318, + "max_kl": 9.157510757446289, + "mean_kl": 1.2422655820846558, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.671875 + }, + { + "candidate_nll": 1.3067593574523926, + "code_tokens": 45, + "cvar95_kl": 3.69111704826355, + "delta_nll": 0.6001680493354797, + "max_kl": 4.184877872467041, + "mean_kl": 0.6168801188468933, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.6888889074325562 + }, + { + "candidate_nll": 2.2819278240203857, + "code_tokens": 48, + "cvar95_kl": 5.935068607330322, + "delta_nll": 1.0832237005233765, + "max_kl": 6.314632892608643, + "mean_kl": 1.0914994478225708, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.0842819213867188, + "code_tokens": 21, + "cvar95_kl": 1.64229416847229, + "delta_nll": 0.38408076763153076, + "max_kl": 2.218334674835205, + "mean_kl": 0.37952980399131775, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.8095238208770752 + }, + { + "candidate_nll": 1.6648787260055542, + "code_tokens": 24, + "cvar95_kl": 5.616354942321777, + "delta_nll": 0.9104645848274231, + "max_kl": 6.097921371459961, + "mean_kl": 1.0816696882247925, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.5383594036102295, + "code_tokens": 63, + "cvar95_kl": 5.217547416687012, + "delta_nll": 0.9664320945739746, + "max_kl": 8.618024826049805, + "mean_kl": 1.0535266399383545, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.4116727113723755, + "code_tokens": 22, + "cvar95_kl": 2.8223984241485596, + "delta_nll": 0.13291358947753906, + "max_kl": 4.028323650360107, + "mean_kl": 0.5219434499740601, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.5755139589309692, + "code_tokens": 25, + "cvar95_kl": 2.665889263153076, + "delta_nll": 0.44685161113739014, + "max_kl": 3.6610512733459473, + "mean_kl": 0.7221518754959106, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.4314682483673096, + "code_tokens": 21, + "cvar95_kl": 1.3364009857177734, + "delta_nll": 0.4711425304412842, + "max_kl": 1.5617436170578003, + "mean_kl": 0.4020133316516876, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.9894105195999146, + "code_tokens": 24, + "cvar95_kl": 2.6003944873809814, + "delta_nll": 0.523078441619873, + "max_kl": 3.1083710193634033, + "mean_kl": 0.5830727219581604, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.7916666865348816 + }, + { + "candidate_nll": 2.9030909538269043, + "code_tokens": 15, + "cvar95_kl": 6.340234756469727, + "delta_nll": 1.1096031665802002, + "max_kl": 6.340234756469727, + "mean_kl": 0.8005049228668213, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.6358495950698853, + "code_tokens": 59, + "cvar95_kl": 3.1014702320098877, + "delta_nll": 0.8549931645393372, + "max_kl": 3.5077762603759766, + "mean_kl": 0.7149861454963684, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.6610169410705566 + }, + { + "candidate_nll": 1.6072207689285278, + "code_tokens": 54, + "cvar95_kl": 2.798616647720337, + "delta_nll": 0.6731025576591492, + "max_kl": 2.838393449783325, + "mean_kl": 0.8256797194480896, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.5925925970077515 + }, + { + "candidate_nll": 2.4017252922058105, + "code_tokens": 28, + "cvar95_kl": 2.305636405944824, + "delta_nll": 0.4498661756515503, + "max_kl": 2.44950008392334, + "mean_kl": 0.5075047016143799, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.8928571343421936 + }, + { + "candidate_nll": 1.6199557781219482, + "code_tokens": 46, + "cvar95_kl": 3.8294360637664795, + "delta_nll": 0.5352298021316528, + "max_kl": 4.08341121673584, + "mean_kl": 0.826393723487854, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 1.8967223167419434, + "code_tokens": 27, + "cvar95_kl": 3.379631996154785, + "delta_nll": 0.38189995288848877, + "max_kl": 4.300095081329346, + "mean_kl": 0.6284482479095459, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.7777777910232544 + }, + { + "candidate_nll": 2.391467809677124, + "code_tokens": 97, + "cvar95_kl": 4.3888115882873535, + "delta_nll": 1.1340371370315552, + "max_kl": 5.161090850830078, + "mean_kl": 1.057094931602478, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.6082473993301392 + }, + { + "candidate_nll": 1.8346141576766968, + "code_tokens": 24, + "cvar95_kl": 2.866438388824463, + "delta_nll": 1.0323817729949951, + "max_kl": 3.2612571716308594, + "mean_kl": 0.6640434861183167, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.491055727005005, + "code_tokens": 30, + "cvar95_kl": 5.694753646850586, + "delta_nll": 1.2259070873260498, + "max_kl": 6.15928840637207, + "mean_kl": 1.0961772203445435, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.8979215621948242, + "code_tokens": 38, + "cvar95_kl": 3.3727941513061523, + "delta_nll": 0.4648517370223999, + "max_kl": 4.0166754722595215, + "mean_kl": 0.6539572477340698, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.7631579041481018 + }, + { + "candidate_nll": 1.8850573301315308, + "code_tokens": 21, + "cvar95_kl": 1.8053181171417236, + "delta_nll": 0.3972783088684082, + "max_kl": 1.8481935262680054, + "mean_kl": 0.5291749835014343, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.369319200515747, + "code_tokens": 65, + "cvar95_kl": 4.046999931335449, + "delta_nll": 0.8323574066162109, + "max_kl": 6.070259094238281, + "mean_kl": 0.7907069325447083, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.6307692527770996 + }, + { + "candidate_nll": 0.9778472781181335, + "code_tokens": 27, + "cvar95_kl": 1.5884100198745728, + "delta_nll": 0.19332098960876465, + "max_kl": 1.6270277500152588, + "mean_kl": 0.32111626863479614, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.8148148059844971 + }, + { + "candidate_nll": 1.4316010475158691, + "code_tokens": 35, + "cvar95_kl": 5.328105449676514, + "delta_nll": 0.7077849507331848, + "max_kl": 7.468367576599121, + "mean_kl": 0.8774805068969727, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.735164761543274, + "code_tokens": 75, + "cvar95_kl": 5.0317583084106445, + "delta_nll": 0.8106406331062317, + "max_kl": 6.880044460296631, + "mean_kl": 1.0299311876296997, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.6266666650772095 + }, + { + "candidate_nll": 2.3429009914398193, + "code_tokens": 25, + "cvar95_kl": 3.8272459506988525, + "delta_nll": 0.7011654376983643, + "max_kl": 4.6284260749816895, + "mean_kl": 1.0043768882751465, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.5777878761291504, + "code_tokens": 52, + "cvar95_kl": 3.069300889968872, + "delta_nll": 0.6537989377975464, + "max_kl": 4.233497142791748, + "mean_kl": 0.6401729583740234, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.9131393432617188, + "code_tokens": 110, + "cvar95_kl": 5.498794078826904, + "delta_nll": 1.3024792671203613, + "max_kl": 6.290473937988281, + "mean_kl": 1.3385440111160278, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.9648257493972778, + "code_tokens": 25, + "cvar95_kl": 3.739701509475708, + "delta_nll": 0.7320038080215454, + "max_kl": 3.987820625305176, + "mean_kl": 0.8204809427261353, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.6489132642745972, + "code_tokens": 56, + "cvar95_kl": 2.87494158744812, + "delta_nll": 0.6319272518157959, + "max_kl": 3.102954149246216, + "mean_kl": 0.7540364861488342, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.6785714030265808 + }, + { + "candidate_nll": 2.811880350112915, + "code_tokens": 108, + "cvar95_kl": 4.006493091583252, + "delta_nll": 1.2580320835113525, + "max_kl": 4.893801689147949, + "mean_kl": 1.044417381286621, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 1.940338134765625, + "code_tokens": 47, + "cvar95_kl": 4.2197370529174805, + "delta_nll": 0.5685982704162598, + "max_kl": 4.915261745452881, + "mean_kl": 0.8544566631317139, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.6595744490623474 + }, + { + "candidate_nll": 2.126844644546509, + "code_tokens": 102, + "cvar95_kl": 5.176737308502197, + "delta_nll": 1.1284360885620117, + "max_kl": 6.814855575561523, + "mean_kl": 1.206226110458374, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.5588235259056091 + }, + { + "candidate_nll": 2.018544912338257, + "code_tokens": 18, + "cvar95_kl": 1.7832688093185425, + "delta_nll": 0.39291930198669434, + "max_kl": 1.7832688093185425, + "mean_kl": 0.5559535026550293, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.039269208908081, + "code_tokens": 28, + "cvar95_kl": 4.464201927185059, + "delta_nll": 0.9396049976348877, + "max_kl": 6.379303455352783, + "mean_kl": 0.7926393151283264, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 2.211474895477295, + "code_tokens": 30, + "cvar95_kl": 2.1380836963653564, + "delta_nll": 0.6680740118026733, + "max_kl": 2.141587734222412, + "mean_kl": 0.5180180668830872, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.7666666507720947 + }, + { + "candidate_nll": 1.4775985479354858, + "code_tokens": 48, + "cvar95_kl": 4.732845783233643, + "delta_nll": 1.0024815797805786, + "max_kl": 6.091846466064453, + "mean_kl": 1.0739169120788574, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 1.3423503637313843, + "code_tokens": 59, + "cvar95_kl": 5.409399509429932, + "delta_nll": 0.8053353428840637, + "max_kl": 5.998922348022461, + "mean_kl": 0.8272143006324768, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.7796609997749329 + }, + { + "candidate_nll": 1.9327728748321533, + "code_tokens": 102, + "cvar95_kl": 3.5502431392669678, + "delta_nll": 0.8905789852142334, + "max_kl": 4.954209804534912, + "mean_kl": 0.8284209966659546, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.6764705777168274 + }, + { + "candidate_nll": 1.5817111730575562, + "code_tokens": 36, + "cvar95_kl": 2.9861900806427, + "delta_nll": 0.8413455486297607, + "max_kl": 3.0498547554016113, + "mean_kl": 0.7214430570602417, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.8784481287002563, + "code_tokens": 51, + "cvar95_kl": 4.3032145500183105, + "delta_nll": 0.9895166158676147, + "max_kl": 5.679299831390381, + "mean_kl": 0.912172794342041, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 2.485616445541382, + "code_tokens": 62, + "cvar95_kl": 5.717792987823486, + "delta_nll": 1.2399377822875977, + "max_kl": 6.415706157684326, + "mean_kl": 1.3577053546905518, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.6129032373428345 + }, + { + "candidate_nll": 1.8079123497009277, + "code_tokens": 39, + "cvar95_kl": 6.829513072967529, + "delta_nll": 1.2316303253173828, + "max_kl": 6.982604503631592, + "mean_kl": 1.3465116024017334, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.6153846383094788 + }, + { + "candidate_nll": 1.502866506576538, + "code_tokens": 76, + "cvar95_kl": 3.8723974227905273, + "delta_nll": 0.7631821632385254, + "max_kl": 4.565370082855225, + "mean_kl": 0.6854333877563477, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.6973684430122375 + }, + { + "candidate_nll": 2.170872926712036, + "code_tokens": 58, + "cvar95_kl": 3.5489377975463867, + "delta_nll": 0.6662218570709229, + "max_kl": 3.6588220596313477, + "mean_kl": 0.6807057857513428, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 2.3633079528808594, + "code_tokens": 93, + "cvar95_kl": 4.347151756286621, + "delta_nll": 1.2359126806259155, + "max_kl": 5.3135762214660645, + "mean_kl": 1.0022687911987305, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.5806451439857483 + }, + { + "candidate_nll": 2.041283369064331, + "code_tokens": 45, + "cvar95_kl": 4.600948333740234, + "delta_nll": 0.6860378980636597, + "max_kl": 8.34201431274414, + "mean_kl": 0.718752384185791, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.481890082359314, + "code_tokens": 209, + "cvar95_kl": 4.503603458404541, + "delta_nll": 0.9249146580696106, + "max_kl": 5.918145656585693, + "mean_kl": 1.007394552230835, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.6172248721122742 + }, + { + "candidate_nll": 1.7768257856369019, + "code_tokens": 46, + "cvar95_kl": 3.2651360034942627, + "delta_nll": 0.8083581328392029, + "max_kl": 4.387097358703613, + "mean_kl": 0.6620118021965027, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.804347813129425 + }, + { + "candidate_nll": 1.3745371103286743, + "code_tokens": 39, + "cvar95_kl": 4.754983425140381, + "delta_nll": 0.6537501215934753, + "max_kl": 6.58702278137207, + "mean_kl": 0.6235072016716003, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.3445266485214233, + "code_tokens": 25, + "cvar95_kl": 2.5755693912506104, + "delta_nll": 0.4028312563896179, + "max_kl": 4.240860939025879, + "mean_kl": 0.3520977795124054, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.8399999737739563 + }, + { + "candidate_nll": 1.7583348751068115, + "code_tokens": 28, + "cvar95_kl": 1.6424822807312012, + "delta_nll": 0.5831385850906372, + "max_kl": 1.7156944274902344, + "mean_kl": 0.5225332379341125, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 2.192363977432251, + "code_tokens": 84, + "cvar95_kl": 4.257025718688965, + "delta_nll": 0.9384413957595825, + "max_kl": 5.538780689239502, + "mean_kl": 0.8417872190475464, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.6309523582458496 + }, + { + "candidate_nll": 1.3636388778686523, + "code_tokens": 34, + "cvar95_kl": 2.069392442703247, + "delta_nll": 0.5513585805892944, + "max_kl": 2.3290133476257324, + "mean_kl": 0.4581373929977417, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.8235294222831726 + }, + { + "candidate_nll": 1.8156776428222656, + "code_tokens": 28, + "cvar95_kl": 2.298698663711548, + "delta_nll": 0.5606305599212646, + "max_kl": 2.455296277999878, + "mean_kl": 0.5367228388786316, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.195101261138916, + "code_tokens": 92, + "cvar95_kl": 5.323950290679932, + "delta_nll": 1.0962557792663574, + "max_kl": 7.809332370758057, + "mean_kl": 1.1034612655639648, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.5760869383811951 + }, + { + "candidate_nll": 1.924422264099121, + "code_tokens": 185, + "cvar95_kl": 4.962944030761719, + "delta_nll": 1.194307565689087, + "max_kl": 7.171605587005615, + "mean_kl": 1.1585057973861694, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 1.5151300430297852, + "code_tokens": 106, + "cvar95_kl": 4.892622470855713, + "delta_nll": 1.1621354818344116, + "max_kl": 7.02252197265625, + "mean_kl": 1.1589603424072266, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.6320754885673523 + }, + { + "candidate_nll": 2.93650221824646, + "code_tokens": 64, + "cvar95_kl": 5.444129943847656, + "delta_nll": 1.393947958946228, + "max_kl": 8.062878608703613, + "mean_kl": 1.1890745162963867, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.53125 + }, + { + "candidate_nll": 2.288447141647339, + "code_tokens": 74, + "cvar95_kl": 4.688990592956543, + "delta_nll": 1.1776630878448486, + "max_kl": 5.277153968811035, + "mean_kl": 1.247626781463623, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.5810810923576355 + }, + { + "candidate_nll": 1.7771239280700684, + "code_tokens": 27, + "cvar95_kl": 3.6366748809814453, + "delta_nll": 0.7897200584411621, + "max_kl": 4.716245651245117, + "mean_kl": 0.7655318975448608, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.2180447578430176, + "code_tokens": 72, + "cvar95_kl": 3.770374298095703, + "delta_nll": 0.5942068696022034, + "max_kl": 4.390496730804443, + "mean_kl": 0.7001088857650757, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.6805555820465088 + }, + { + "candidate_nll": 1.6118693351745605, + "code_tokens": 62, + "cvar95_kl": 3.7237703800201416, + "delta_nll": 0.6785669922828674, + "max_kl": 5.993428707122803, + "mean_kl": 0.7262714505195618, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.4764209985733032, + "code_tokens": 42, + "cvar95_kl": 3.5520236492156982, + "delta_nll": 0.7782546281814575, + "max_kl": 4.016918182373047, + "mean_kl": 0.8874324560165405, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 1.8903356790542603, + "code_tokens": 238, + "cvar95_kl": 4.626539707183838, + "delta_nll": 1.0030028820037842, + "max_kl": 6.333737373352051, + "mean_kl": 1.007502555847168, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.651260495185852 + }, + { + "candidate_nll": 1.5304481983184814, + "code_tokens": 68, + "cvar95_kl": 4.022569179534912, + "delta_nll": 0.6159364581108093, + "max_kl": 6.678524494171143, + "mean_kl": 0.651828944683075, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.779411792755127 + }, + { + "candidate_nll": 2.240602970123291, + "code_tokens": 83, + "cvar95_kl": 5.882298469543457, + "delta_nll": 1.256453037261963, + "max_kl": 6.749444961547852, + "mean_kl": 1.2417844533920288, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.5542168617248535 + }, + { + "candidate_nll": 1.977936029434204, + "code_tokens": 64, + "cvar95_kl": 4.066760540008545, + "delta_nll": 0.7141685485839844, + "max_kl": 4.803587436676025, + "mean_kl": 0.7168624997138977, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.1726034879684448, + "code_tokens": 37, + "cvar95_kl": 2.4078779220581055, + "delta_nll": 0.5158035755157471, + "max_kl": 2.8215465545654297, + "mean_kl": 0.4947506785392761, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 2.191648006439209, + "code_tokens": 27, + "cvar95_kl": 4.200430870056152, + "delta_nll": 0.6817208528518677, + "max_kl": 5.165392875671387, + "mean_kl": 0.920294463634491, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.5191147327423096, + "code_tokens": 40, + "cvar95_kl": 4.229637622833252, + "delta_nll": 0.6273983716964722, + "max_kl": 4.811721324920654, + "mean_kl": 0.6423074007034302, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.824999988079071 + }, + { + "candidate_nll": 1.6087592840194702, + "code_tokens": 28, + "cvar95_kl": 3.656810760498047, + "delta_nll": 0.610986590385437, + "max_kl": 4.832565784454346, + "mean_kl": 0.7691978812217712, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.6071428656578064 + }, + { + "candidate_nll": 2.077192544937134, + "code_tokens": 59, + "cvar95_kl": 4.8621978759765625, + "delta_nll": 1.2225629091262817, + "max_kl": 4.9956207275390625, + "mean_kl": 1.238348126411438, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.5254237055778503 + }, + { + "candidate_nll": 1.6513302326202393, + "code_tokens": 57, + "cvar95_kl": 4.111303806304932, + "delta_nll": 1.0813300609588623, + "max_kl": 4.830085277557373, + "mean_kl": 0.9468443989753723, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.323896646499634, + "code_tokens": 25, + "cvar95_kl": 3.709001064300537, + "delta_nll": 0.939956545829773, + "max_kl": 3.73199462890625, + "mean_kl": 0.8215607404708862, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.5705699920654297, + "code_tokens": 48, + "cvar95_kl": 3.790027379989624, + "delta_nll": 0.8234176635742188, + "max_kl": 4.488565921783447, + "mean_kl": 0.7136160731315613, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 2.2560110092163086, + "code_tokens": 35, + "cvar95_kl": 6.557774543762207, + "delta_nll": 1.3942351341247559, + "max_kl": 8.099321365356445, + "mean_kl": 1.0183093547821045, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.6285714507102966 + }, + { + "candidate_nll": 1.0428683757781982, + "code_tokens": 30, + "cvar95_kl": 2.4282617568969727, + "delta_nll": 0.3981611132621765, + "max_kl": 2.4878106117248535, + "mean_kl": 0.4819530248641968, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.6268430948257446, + "code_tokens": 49, + "cvar95_kl": 2.703545331954956, + "delta_nll": 0.5037655830383301, + "max_kl": 3.32241153717041, + "mean_kl": 0.533726155757904, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.8163265585899353 + }, + { + "candidate_nll": 2.1179404258728027, + "code_tokens": 98, + "cvar95_kl": 5.151853084564209, + "delta_nll": 1.101723551750183, + "max_kl": 6.265498161315918, + "mean_kl": 1.0643218755722046, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.5415500402450562, + "code_tokens": 25, + "cvar95_kl": 1.9616461992263794, + "delta_nll": 0.16074371337890625, + "max_kl": 1.9914664030075073, + "mean_kl": 0.5274837017059326, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.6792467832565308, + "code_tokens": 27, + "cvar95_kl": 6.6540727615356445, + "delta_nll": 0.9015235900878906, + "max_kl": 8.010356903076172, + "mean_kl": 0.9426111578941345, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.7407407164573669 + }, + { + "candidate_nll": 1.4321391582489014, + "code_tokens": 20, + "cvar95_kl": 2.757904291152954, + "delta_nll": 0.387054443359375, + "max_kl": 2.757904291152954, + "mean_kl": 0.540091872215271, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.9783357381820679, + "code_tokens": 253, + "cvar95_kl": 5.750490188598633, + "delta_nll": 1.28566575050354, + "max_kl": 8.462579727172852, + "mean_kl": 1.3112311363220215, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.573122501373291 + }, + { + "candidate_nll": 1.5513908863067627, + "code_tokens": 55, + "cvar95_kl": 5.037249565124512, + "delta_nll": 0.7222706079483032, + "max_kl": 6.185207366943359, + "mean_kl": 0.9286230802536011, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.6909090876579285 + }, + { + "candidate_nll": 1.7075077295303345, + "code_tokens": 31, + "cvar95_kl": 2.9380156993865967, + "delta_nll": 0.5010093450546265, + "max_kl": 3.006361484527588, + "mean_kl": 0.6303184032440186, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 1.4210706949234009, + "code_tokens": 35, + "cvar95_kl": 3.557878017425537, + "delta_nll": 0.7168245315551758, + "max_kl": 4.532816410064697, + "mean_kl": 0.6324841976165771, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 1.3909335136413574, + "code_tokens": 61, + "cvar95_kl": 4.305072784423828, + "delta_nll": 0.5880650281906128, + "max_kl": 5.565624237060547, + "mean_kl": 0.7529662847518921, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.7049180269241333 + }, + { + "candidate_nll": 2.104436159133911, + "code_tokens": 55, + "cvar95_kl": 6.045368194580078, + "delta_nll": 1.1497185230255127, + "max_kl": 7.894254207611084, + "mean_kl": 1.1388628482818604, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.5272727012634277 + }, + { + "candidate_nll": 2.455811023712158, + "code_tokens": 108, + "cvar95_kl": 4.015523433685303, + "delta_nll": 0.9488438367843628, + "max_kl": 4.621423244476318, + "mean_kl": 0.853942334651947, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.8685200214385986, + "code_tokens": 58, + "cvar95_kl": 3.192451238632202, + "delta_nll": 0.5812139511108398, + "max_kl": 4.082377910614014, + "mean_kl": 0.5818920135498047, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.7068965435028076 + }, + { + "candidate_nll": 2.2756502628326416, + "code_tokens": 91, + "cvar95_kl": 6.469282627105713, + "delta_nll": 1.448194980621338, + "max_kl": 10.094683647155762, + "mean_kl": 1.4185662269592285, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.5274725556373596 + }, + { + "candidate_nll": 1.6420491933822632, + "code_tokens": 105, + "cvar95_kl": 3.7321863174438477, + "delta_nll": 0.8783097267150879, + "max_kl": 4.274991512298584, + "mean_kl": 0.8876848220825195, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.6952381134033203 + }, + { + "candidate_nll": 1.6528997421264648, + "code_tokens": 49, + "cvar95_kl": 3.457401990890503, + "delta_nll": 1.0056471824645996, + "max_kl": 3.9171180725097656, + "mean_kl": 0.8698972463607788, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.6938775777816772 + }, + { + "candidate_nll": 2.1378047466278076, + "code_tokens": 27, + "cvar95_kl": 3.873096466064453, + "delta_nll": 0.6756417751312256, + "max_kl": 4.432293891906738, + "mean_kl": 0.8573782444000244, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 1.5541309118270874, + "code_tokens": 25, + "cvar95_kl": 2.7307331562042236, + "delta_nll": 0.5137542486190796, + "max_kl": 2.9405405521392822, + "mean_kl": 0.6988351941108704, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 2.2432618141174316, + "code_tokens": 98, + "cvar95_kl": 4.589789867401123, + "delta_nll": 0.9482073783874512, + "max_kl": 5.73980188369751, + "mean_kl": 1.151818871498108, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.6020408272743225 + }, + { + "candidate_nll": 1.3970036506652832, + "code_tokens": 23, + "cvar95_kl": 2.6582131385803223, + "delta_nll": 0.7472219467163086, + "max_kl": 3.1739344596862793, + "mean_kl": 0.6522051692008972, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.782608687877655 + }, + { + "candidate_nll": 3.051603317260742, + "code_tokens": 31, + "cvar95_kl": 5.87539529800415, + "delta_nll": 1.1862295866012573, + "max_kl": 7.6394362449646, + "mean_kl": 1.0920718908309937, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 1.3704065084457397, + "code_tokens": 36, + "cvar95_kl": 3.57853364944458, + "delta_nll": 0.622313380241394, + "max_kl": 4.717039108276367, + "mean_kl": 0.705575704574585, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.6944444179534912 + }, + { + "candidate_nll": 1.860507607460022, + "code_tokens": 38, + "cvar95_kl": 3.6113638877868652, + "delta_nll": 0.4268300533294678, + "max_kl": 3.7077741622924805, + "mean_kl": 0.7494192123413086, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.6578947305679321 + }, + { + "candidate_nll": 2.3780930042266846, + "code_tokens": 25, + "cvar95_kl": 1.8934836387634277, + "delta_nll": 0.772003173828125, + "max_kl": 2.6258270740509033, + "mean_kl": 0.47493094205856323, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 1.4012622833251953, + "code_tokens": 26, + "cvar95_kl": 3.1061973571777344, + "delta_nll": 0.521813154220581, + "max_kl": 3.6092917919158936, + "mean_kl": 0.6802133321762085, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.8871623277664185, + "code_tokens": 62, + "cvar95_kl": 5.499540328979492, + "delta_nll": 1.1447203159332275, + "max_kl": 6.343327522277832, + "mean_kl": 1.0124648809432983, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.725806474685669 + }, + { + "candidate_nll": 2.680555820465088, + "code_tokens": 36, + "cvar95_kl": 3.5386977195739746, + "delta_nll": 0.9420367479324341, + "max_kl": 3.6486566066741943, + "mean_kl": 0.900456964969635, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 2.139406204223633, + "code_tokens": 119, + "cvar95_kl": 4.925670146942139, + "delta_nll": 0.8088172674179077, + "max_kl": 6.419483661651611, + "mean_kl": 1.0452830791473389, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.6134454011917114 + }, + { + "candidate_nll": 0.7195303440093994, + "code_tokens": 37, + "cvar95_kl": 1.5333824157714844, + "delta_nll": 0.4068227708339691, + "max_kl": 1.5682716369628906, + "mean_kl": 0.3124232292175293, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.8918918967247009 + }, + { + "candidate_nll": 1.512635588645935, + "code_tokens": 28, + "cvar95_kl": 2.7634453773498535, + "delta_nll": 0.7299020886421204, + "max_kl": 3.0156073570251465, + "mean_kl": 0.7275063395500183, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.7345579862594604, + "code_tokens": 143, + "cvar95_kl": 6.432378768920898, + "delta_nll": 1.1682112216949463, + "max_kl": 8.112530708312988, + "mean_kl": 1.150236964225769, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 2.5057480335235596, + "code_tokens": 29, + "cvar95_kl": 5.213614463806152, + "delta_nll": 0.7234417200088501, + "max_kl": 6.646160125732422, + "mean_kl": 1.1417155265808105, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.6206896305084229 + }, + { + "candidate_nll": 1.9839590787887573, + "code_tokens": 28, + "cvar95_kl": 3.8970746994018555, + "delta_nll": 0.9114706516265869, + "max_kl": 4.315847873687744, + "mean_kl": 0.8506141901016235, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 2.324286699295044, + "code_tokens": 30, + "cvar95_kl": 6.662139415740967, + "delta_nll": 1.0602489709854126, + "max_kl": 7.575652599334717, + "mean_kl": 1.3029311895370483, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.5333333611488342 + }, + { + "candidate_nll": 1.5511318445205688, + "code_tokens": 85, + "cvar95_kl": 5.213353157043457, + "delta_nll": 1.129321813583374, + "max_kl": 6.161235809326172, + "mean_kl": 0.9853991270065308, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.6705882549285889 + }, + { + "candidate_nll": 1.9771220684051514, + "code_tokens": 44, + "cvar95_kl": 3.613473653793335, + "delta_nll": 0.999878466129303, + "max_kl": 3.7269208431243896, + "mean_kl": 0.9557847380638123, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.7045454382896423 + }, + { + "candidate_nll": 1.700090765953064, + "code_tokens": 19, + "cvar95_kl": 2.5391578674316406, + "delta_nll": 0.8959885239601135, + "max_kl": 2.5391578674316406, + "mean_kl": 0.41961634159088135, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.8421052694320679 + }, + { + "candidate_nll": 1.7778817415237427, + "code_tokens": 113, + "cvar95_kl": 5.010654926300049, + "delta_nll": 1.0404589176177979, + "max_kl": 5.521114826202393, + "mean_kl": 1.0111110210418701, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.6814159154891968 + }, + { + "candidate_nll": 2.1917521953582764, + "code_tokens": 40, + "cvar95_kl": 2.5426506996154785, + "delta_nll": 0.7165577411651611, + "max_kl": 3.0086240768432617, + "mean_kl": 0.7825903296470642, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 2.6108341217041016, + "code_tokens": 51, + "cvar95_kl": 4.485322952270508, + "delta_nll": 1.0041741132736206, + "max_kl": 5.762330055236816, + "mean_kl": 0.9821942448616028, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.6274510025978088 + }, + { + "candidate_nll": 2.5914008617401123, + "code_tokens": 55, + "cvar95_kl": 3.9831457138061523, + "delta_nll": 1.0103683471679688, + "max_kl": 4.471619606018066, + "mean_kl": 1.1583616733551025, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.5636363625526428 + }, + { + "candidate_nll": 1.587894320487976, + "code_tokens": 27, + "cvar95_kl": 2.653705596923828, + "delta_nll": 0.45782601833343506, + "max_kl": 2.8706603050231934, + "mean_kl": 0.6045097708702087, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.7037037014961243 + }, + { + "candidate_nll": 1.3655283451080322, + "code_tokens": 32, + "cvar95_kl": 3.6716599464416504, + "delta_nll": 0.5257221460342407, + "max_kl": 4.817238807678223, + "mean_kl": 0.5330272316932678, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.8125 + }, + { + "candidate_nll": 1.4917309284210205, + "code_tokens": 64, + "cvar95_kl": 3.8507461547851562, + "delta_nll": 0.8391928672790527, + "max_kl": 4.18488883972168, + "mean_kl": 0.9025732278823853, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.640625 + }, + { + "candidate_nll": 1.8944809436798096, + "code_tokens": 35, + "cvar95_kl": 4.026094913482666, + "delta_nll": 0.8448266983032227, + "max_kl": 5.210236549377441, + "mean_kl": 0.6827400326728821, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.7714285850524902 + }, + { + "candidate_nll": 1.8742053508758545, + "code_tokens": 152, + "cvar95_kl": 5.616546630859375, + "delta_nll": 1.4223577976226807, + "max_kl": 7.033771514892578, + "mean_kl": 1.3192098140716553, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.5855262875556946 + }, + { + "candidate_nll": 2.18206787109375, + "code_tokens": 77, + "cvar95_kl": 3.9789440631866455, + "delta_nll": 0.9207335710525513, + "max_kl": 7.354038715362549, + "mean_kl": 0.9330724477767944, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.6233766078948975 + }, + { + "candidate_nll": 1.498720645904541, + "code_tokens": 77, + "cvar95_kl": 4.80938720703125, + "delta_nll": 0.9304395914077759, + "max_kl": 5.985877513885498, + "mean_kl": 0.9007807970046997, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.701298713684082 + }, + { + "candidate_nll": 0.9036263227462769, + "code_tokens": 71, + "cvar95_kl": 2.8585245609283447, + "delta_nll": 0.3415375351905823, + "max_kl": 3.7261784076690674, + "mean_kl": 0.40242427587509155, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.8732394576072693 + }, + { + "candidate_nll": 2.155066728591919, + "code_tokens": 112, + "cvar95_kl": 5.187180995941162, + "delta_nll": 1.2766873836517334, + "max_kl": 7.192046165466309, + "mean_kl": 1.1026033163070679, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.6517857313156128 + }, + { + "candidate_nll": 1.625439167022705, + "code_tokens": 111, + "cvar95_kl": 3.4046356678009033, + "delta_nll": 0.9055354595184326, + "max_kl": 4.674134254455566, + "mean_kl": 0.7453005909919739, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.6936936974525452 + }, + { + "candidate_nll": 1.6928848028182983, + "code_tokens": 62, + "cvar95_kl": 4.2129225730896, + "delta_nll": 0.9049885869026184, + "max_kl": 5.252715587615967, + "mean_kl": 0.8332744836807251, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.725806474685669 + }, + { + "candidate_nll": 1.3746886253356934, + "code_tokens": 52, + "cvar95_kl": 3.6807117462158203, + "delta_nll": 0.7641085982322693, + "max_kl": 4.107074737548828, + "mean_kl": 0.6197985410690308, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 2.527170181274414, + "code_tokens": 24, + "cvar95_kl": 3.3880810737609863, + "delta_nll": 0.6319435834884644, + "max_kl": 3.6072747707366943, + "mean_kl": 0.6365644335746765, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5597962141036987, + "code_tokens": 48, + "cvar95_kl": 3.8562328815460205, + "delta_nll": 0.6680262684822083, + "max_kl": 4.089693546295166, + "mean_kl": 0.7915723919868469, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.239861249923706, + "code_tokens": 98, + "cvar95_kl": 5.3904643058776855, + "delta_nll": 1.189568281173706, + "max_kl": 8.03225326538086, + "mean_kl": 1.131460428237915, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.5918367505073547 + }, + { + "candidate_nll": 2.045419692993164, + "code_tokens": 48, + "cvar95_kl": 4.044170379638672, + "delta_nll": 1.052734136581421, + "max_kl": 4.517620086669922, + "mean_kl": 0.9947978854179382, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.6041666865348816 + }, + { + "candidate_nll": 1.9753788709640503, + "code_tokens": 56, + "cvar95_kl": 4.580817699432373, + "delta_nll": 1.0293834209442139, + "max_kl": 5.7667365074157715, + "mean_kl": 0.9393439292907715, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.5892857313156128 + }, + { + "candidate_nll": 1.6974215507507324, + "code_tokens": 88, + "cvar95_kl": 5.133063316345215, + "delta_nll": 1.096144676208496, + "max_kl": 7.938470840454102, + "mean_kl": 0.9468786120414734, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 2.522062063217163, + "code_tokens": 50, + "cvar95_kl": 4.262121677398682, + "delta_nll": 0.8885318040847778, + "max_kl": 4.9585676193237305, + "mean_kl": 0.7952315807342529, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.0627713203430176, + "code_tokens": 85, + "cvar95_kl": 3.428539991378784, + "delta_nll": 0.625784158706665, + "max_kl": 6.193647861480713, + "mean_kl": 0.623600423336029, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.7176470756530762 + }, + { + "candidate_nll": 2.032162666320801, + "code_tokens": 27, + "cvar95_kl": 4.615974426269531, + "delta_nll": 0.9579495191574097, + "max_kl": 4.630626678466797, + "mean_kl": 0.7586186528205872, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.6635199785232544, + "code_tokens": 48, + "cvar95_kl": 4.345244884490967, + "delta_nll": 0.5459015369415283, + "max_kl": 4.923090934753418, + "mean_kl": 0.6494598388671875, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.7708333134651184 + }, + { + "candidate_nll": 2.4960618019104004, + "code_tokens": 29, + "cvar95_kl": 3.2959439754486084, + "delta_nll": 0.8482145071029663, + "max_kl": 3.5303761959075928, + "mean_kl": 0.8986372947692871, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.7969094514846802, + "code_tokens": 63, + "cvar95_kl": 6.120684623718262, + "delta_nll": 0.8586940765380859, + "max_kl": 8.019082069396973, + "mean_kl": 1.1101628541946411, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.682539701461792 + }, + { + "candidate_nll": 1.994989275932312, + "code_tokens": 89, + "cvar95_kl": 6.566096305847168, + "delta_nll": 1.3199305534362793, + "max_kl": 9.557247161865234, + "mean_kl": 1.2475693225860596, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.6292135119438171 + }, + { + "candidate_nll": 2.7919669151306152, + "code_tokens": 133, + "cvar95_kl": 6.792613506317139, + "delta_nll": 1.4476885795593262, + "max_kl": 9.373529434204102, + "mean_kl": 1.3771607875823975, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.5789473652839661 + }, + { + "candidate_nll": 1.4148375988006592, + "code_tokens": 25, + "cvar95_kl": 2.424076557159424, + "delta_nll": 0.4663008451461792, + "max_kl": 4.026508808135986, + "mean_kl": 0.367410272359848, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.4166722297668457, + "code_tokens": 33, + "cvar95_kl": 1.6157784461975098, + "delta_nll": 0.37937188148498535, + "max_kl": 2.097414970397949, + "mean_kl": 0.4384549856185913, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 1.1931188106536865, + "code_tokens": 93, + "cvar95_kl": 3.671412706375122, + "delta_nll": 0.43634384870529175, + "max_kl": 5.28640604019165, + "mean_kl": 0.575340211391449, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.7419354915618896 + }, + { + "candidate_nll": 2.238922119140625, + "code_tokens": 101, + "cvar95_kl": 4.508175373077393, + "delta_nll": 1.2705225944519043, + "max_kl": 5.371719837188721, + "mean_kl": 1.192131519317627, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.5742574334144592 + }, + { + "candidate_nll": 1.8429956436157227, + "code_tokens": 152, + "cvar95_kl": 5.774280548095703, + "delta_nll": 1.2782323360443115, + "max_kl": 7.950103282928467, + "mean_kl": 1.2130051851272583, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.7994669675827026, + "code_tokens": 33, + "cvar95_kl": 2.515392780303955, + "delta_nll": 0.3985954523086548, + "max_kl": 2.5488643646240234, + "mean_kl": 0.5508022308349609, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8661563396453857, + "code_tokens": 29, + "cvar95_kl": 2.251554012298584, + "delta_nll": 0.5796934366226196, + "max_kl": 2.447996139526367, + "mean_kl": 0.5567067265510559, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.7931034564971924 + }, + { + "candidate_nll": 2.279224157333374, + "code_tokens": 27, + "cvar95_kl": 2.5819664001464844, + "delta_nll": 0.8211543560028076, + "max_kl": 3.250417470932007, + "mean_kl": 0.5394114255905151, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.8518518805503845 + }, + { + "candidate_nll": 0.9362607598304749, + "code_tokens": 33, + "cvar95_kl": 3.2657816410064697, + "delta_nll": 0.6262206435203552, + "max_kl": 3.4824490547180176, + "mean_kl": 0.6449345350265503, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 1.2061724662780762, + "code_tokens": 27, + "cvar95_kl": 1.4551807641983032, + "delta_nll": 0.40977418422698975, + "max_kl": 1.8822896480560303, + "mean_kl": 0.38445231318473816, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.8148148059844971 + }, + { + "candidate_nll": 1.6315175294876099, + "code_tokens": 84, + "cvar95_kl": 3.5841541290283203, + "delta_nll": 0.9276240468025208, + "max_kl": 4.168164253234863, + "mean_kl": 0.8206908702850342, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.7023809552192688 + }, + { + "candidate_nll": 1.9190973043441772, + "code_tokens": 103, + "cvar95_kl": 3.412926435470581, + "delta_nll": 0.8736274242401123, + "max_kl": 4.673380374908447, + "mean_kl": 0.8163552284240723, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.6504854559898376 + }, + { + "candidate_nll": 1.6255496740341187, + "code_tokens": 88, + "cvar95_kl": 3.3105404376983643, + "delta_nll": 0.7092023491859436, + "max_kl": 4.296755790710449, + "mean_kl": 0.7984740734100342, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.6818181872367859 + }, + { + "candidate_nll": 2.5903899669647217, + "code_tokens": 62, + "cvar95_kl": 6.622922420501709, + "delta_nll": 1.2814592123031616, + "max_kl": 7.930595874786377, + "mean_kl": 1.161439299583435, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.6451612710952759 + }, + { + "candidate_nll": 2.054344415664673, + "code_tokens": 24, + "cvar95_kl": 1.7230397462844849, + "delta_nll": 0.5176355838775635, + "max_kl": 1.7735570669174194, + "mean_kl": 0.475236177444458, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.329566478729248, + "code_tokens": 34, + "cvar95_kl": 3.791318416595459, + "delta_nll": 0.9911143779754639, + "max_kl": 4.538718223571777, + "mean_kl": 0.7433950901031494, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 1.2177144289016724, + "code_tokens": 43, + "cvar95_kl": 2.812276601791382, + "delta_nll": 0.49796903133392334, + "max_kl": 3.5550522804260254, + "mean_kl": 0.5208929777145386, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.7674418687820435 + }, + { + "candidate_nll": 2.354874849319458, + "code_tokens": 54, + "cvar95_kl": 2.7027556896209717, + "delta_nll": 0.8655695915222168, + "max_kl": 3.183993339538574, + "mean_kl": 0.7881979942321777, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.3539825677871704, + "code_tokens": 50, + "cvar95_kl": 2.2427492141723633, + "delta_nll": 0.6240971088409424, + "max_kl": 2.5996031761169434, + "mean_kl": 0.6260632276535034, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.7200000286102295 + }, + { + "candidate_nll": 1.9906904697418213, + "code_tokens": 235, + "cvar95_kl": 5.352676868438721, + "delta_nll": 1.233180046081543, + "max_kl": 7.757999897003174, + "mean_kl": 1.265448808670044, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.5957446694374084 + }, + { + "candidate_nll": 2.7449393272399902, + "code_tokens": 33, + "cvar95_kl": 3.9554662704467773, + "delta_nll": 0.7162842750549316, + "max_kl": 4.604513168334961, + "mean_kl": 0.8217800855636597, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 1.6091920137405396, + "code_tokens": 45, + "cvar95_kl": 2.164774179458618, + "delta_nll": 0.6226896047592163, + "max_kl": 2.7322998046875, + "mean_kl": 0.40764668583869934, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.8222222328186035 + }, + { + "candidate_nll": 1.4304077625274658, + "code_tokens": 28, + "cvar95_kl": 4.170969486236572, + "delta_nll": 0.8074478507041931, + "max_kl": 4.405911445617676, + "mean_kl": 0.7255359888076782, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.3164142370224, + "code_tokens": 49, + "cvar95_kl": 3.2707908153533936, + "delta_nll": 0.6381498575210571, + "max_kl": 3.873025417327881, + "mean_kl": 0.6754935383796692, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.7755101919174194 + }, + { + "candidate_nll": 1.9847557544708252, + "code_tokens": 107, + "cvar95_kl": 6.388322353363037, + "delta_nll": 1.7353401184082031, + "max_kl": 7.6242170333862305, + "mean_kl": 1.5805730819702148, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.5514018535614014 + }, + { + "candidate_nll": 2.0806782245635986, + "code_tokens": 48, + "cvar95_kl": 5.808570384979248, + "delta_nll": 0.8597681522369385, + "max_kl": 8.68789291381836, + "mean_kl": 0.9656276702880859, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.6458333134651184 + }, + { + "candidate_nll": 2.3674824237823486, + "code_tokens": 40, + "cvar95_kl": 3.4216370582580566, + "delta_nll": 0.5279415845870972, + "max_kl": 3.4371256828308105, + "mean_kl": 0.9865511655807495, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.550000011920929 + }, + { + "candidate_nll": 1.1363399028778076, + "code_tokens": 46, + "cvar95_kl": 4.449655055999756, + "delta_nll": 0.6668562293052673, + "max_kl": 6.841485977172852, + "mean_kl": 0.7823375463485718, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.695652186870575 + }, + { + "candidate_nll": 1.7480570077896118, + "code_tokens": 31, + "cvar95_kl": 4.747533798217773, + "delta_nll": 0.7056072950363159, + "max_kl": 5.806318283081055, + "mean_kl": 0.8250923156738281, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.6774193644523621 + }, + { + "candidate_nll": 0.8370722532272339, + "code_tokens": 58, + "cvar95_kl": 2.677840232849121, + "delta_nll": 0.42058777809143066, + "max_kl": 3.656283140182495, + "mean_kl": 0.40103650093078613, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.8620689511299133 + }, + { + "candidate_nll": 1.8198202848434448, + "code_tokens": 49, + "cvar95_kl": 3.2929112911224365, + "delta_nll": 0.7020504474639893, + "max_kl": 3.759334087371826, + "mean_kl": 0.7468516230583191, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.4711377620697021, + "code_tokens": 18, + "cvar95_kl": 1.3338173627853394, + "delta_nll": 0.5245441794395447, + "max_kl": 1.3338173627853394, + "mean_kl": 0.4559480845928192, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.5236079692840576, + "code_tokens": 28, + "cvar95_kl": 1.7586771249771118, + "delta_nll": 0.36419641971588135, + "max_kl": 1.9965360164642334, + "mean_kl": 0.3914887607097626, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.7857142686843872 + }, + { + "candidate_nll": 1.6075725555419922, + "code_tokens": 57, + "cvar95_kl": 3.919938087463379, + "delta_nll": 0.8655058145523071, + "max_kl": 5.161026477813721, + "mean_kl": 0.8400904536247253, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.6491228342056274 + }, + { + "candidate_nll": 1.8485015630722046, + "code_tokens": 88, + "cvar95_kl": 4.441246032714844, + "delta_nll": 1.1483917236328125, + "max_kl": 4.955263137817383, + "mean_kl": 1.162744402885437, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.5795454382896423 + }, + { + "candidate_nll": 1.7367023229599, + "code_tokens": 23, + "cvar95_kl": 2.4674456119537354, + "delta_nll": 0.5379154682159424, + "max_kl": 2.9459657669067383, + "mean_kl": 0.5605763792991638, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.0812032222747803, + "code_tokens": 40, + "cvar95_kl": 5.429068565368652, + "delta_nll": 0.8217458724975586, + "max_kl": 5.740419387817383, + "mean_kl": 1.0132805109024048, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.9877849817276, + "code_tokens": 20, + "cvar95_kl": 2.583487033843994, + "delta_nll": 0.8265699148178101, + "max_kl": 2.583487033843994, + "mean_kl": 0.8181670904159546, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.5018057823181152, + "code_tokens": 63, + "cvar95_kl": 2.653573751449585, + "delta_nll": 0.48602020740509033, + "max_kl": 3.6663224697113037, + "mean_kl": 0.5618097186088562, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.761904776096344 + }, + { + "candidate_nll": 1.8989145755767822, + "code_tokens": 39, + "cvar95_kl": 4.510194301605225, + "delta_nll": 0.9462332725524902, + "max_kl": 5.930915832519531, + "mean_kl": 0.8037245273590088, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 2.5289695262908936, + "code_tokens": 50, + "cvar95_kl": 4.680840969085693, + "delta_nll": 1.1171048879623413, + "max_kl": 5.418771743774414, + "mean_kl": 0.9846261739730835, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 1.694691777229309, + "code_tokens": 63, + "cvar95_kl": 4.905213832855225, + "delta_nll": 0.727214515209198, + "max_kl": 6.441895484924316, + "mean_kl": 0.9299089908599854, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.6984127163887024 + }, + { + "candidate_nll": 1.6146516799926758, + "code_tokens": 46, + "cvar95_kl": 2.89825701713562, + "delta_nll": 0.6420896649360657, + "max_kl": 3.263378858566284, + "mean_kl": 0.6554650068283081, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.6304348111152649 + }, + { + "candidate_nll": 1.2120332717895508, + "code_tokens": 25, + "cvar95_kl": 2.483729839324951, + "delta_nll": 0.5331387519836426, + "max_kl": 3.11545467376709, + "mean_kl": 0.4870784282684326, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.8399999737739563 + }, + { + "candidate_nll": 1.628605604171753, + "code_tokens": 111, + "cvar95_kl": 3.602576971054077, + "delta_nll": 0.8143396973609924, + "max_kl": 5.108577251434326, + "mean_kl": 0.8086228966712952, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.7207207083702087 + }, + { + "candidate_nll": 1.5814217329025269, + "code_tokens": 71, + "cvar95_kl": 4.152688503265381, + "delta_nll": 0.8528639078140259, + "max_kl": 5.25747537612915, + "mean_kl": 0.7992809414863586, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.6056337952613831 + }, + { + "candidate_nll": 1.794948935508728, + "code_tokens": 28, + "cvar95_kl": 2.7291083335876465, + "delta_nll": 0.4242253303527832, + "max_kl": 2.8965158462524414, + "mean_kl": 0.6024642586708069, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.8214285969734192 + }, + { + "candidate_nll": 1.8578033447265625, + "code_tokens": 25, + "cvar95_kl": 3.907350540161133, + "delta_nll": 0.6573474407196045, + "max_kl": 4.988344669342041, + "mean_kl": 0.754241406917572, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 2.242889881134033, + "code_tokens": 40, + "cvar95_kl": 3.62453031539917, + "delta_nll": 0.9401180744171143, + "max_kl": 3.6451587677001953, + "mean_kl": 0.6931511759757996, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.8117915391921997, + "code_tokens": 177, + "cvar95_kl": 4.348735332489014, + "delta_nll": 0.7673473358154297, + "max_kl": 4.992046356201172, + "mean_kl": 1.03815758228302, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.5649717450141907 + }, + { + "candidate_nll": 1.7746394872665405, + "code_tokens": 51, + "cvar95_kl": 2.8033905029296875, + "delta_nll": 0.6834609508514404, + "max_kl": 3.6767616271972656, + "mean_kl": 0.6188623309135437, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.7058823704719543 + }, + { + "candidate_nll": 1.3560545444488525, + "code_tokens": 69, + "cvar95_kl": 3.8130247592926025, + "delta_nll": 0.9015941619873047, + "max_kl": 4.985284328460693, + "mean_kl": 0.7402786612510681, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.7246376872062683 + }, + { + "candidate_nll": 1.1247410774230957, + "code_tokens": 26, + "cvar95_kl": 2.673274040222168, + "delta_nll": 0.3764811158180237, + "max_kl": 2.915193557739258, + "mean_kl": 0.4312869906425476, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.8846153616905212 + }, + { + "candidate_nll": 2.18928599357605, + "code_tokens": 24, + "cvar95_kl": 2.4850246906280518, + "delta_nll": 0.6709467172622681, + "max_kl": 2.857862949371338, + "mean_kl": 0.5408361554145813, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.5670676231384277, + "code_tokens": 15, + "cvar95_kl": 1.8579357862472534, + "delta_nll": 0.4165917634963989, + "max_kl": 1.8579357862472534, + "mean_kl": 0.46434858441352844, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.8753670454025269, + "code_tokens": 43, + "cvar95_kl": 4.580312252044678, + "delta_nll": 0.7260180711746216, + "max_kl": 6.0887322425842285, + "mean_kl": 0.939144492149353, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.6511628031730652 + }, + { + "candidate_nll": 1.964885950088501, + "code_tokens": 18, + "cvar95_kl": 3.1426448822021484, + "delta_nll": 0.8725602626800537, + "max_kl": 3.1426448822021484, + "mean_kl": 0.5725545883178711, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.6302486658096313, + "code_tokens": 49, + "cvar95_kl": 4.702908515930176, + "delta_nll": 0.902255117893219, + "max_kl": 6.562788963317871, + "mean_kl": 0.8895532488822937, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.6734693646430969 + }, + { + "candidate_nll": 1.4496278762817383, + "code_tokens": 59, + "cvar95_kl": 4.4395012855529785, + "delta_nll": 0.5818718075752258, + "max_kl": 5.216719150543213, + "mean_kl": 0.7620542645454407, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.7118644118309021 + }, + { + "candidate_nll": 1.4793739318847656, + "code_tokens": 221, + "cvar95_kl": 4.145755767822266, + "delta_nll": 0.8941077589988708, + "max_kl": 5.459139823913574, + "mean_kl": 0.949744462966919, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.6380090713500977 + }, + { + "candidate_nll": 1.7008758783340454, + "code_tokens": 25, + "cvar95_kl": 2.5234665870666504, + "delta_nll": 0.4990028142929077, + "max_kl": 2.979872465133667, + "mean_kl": 0.6594104170799255, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.6800000071525574 + }, + { + "candidate_nll": 1.4832713603973389, + "code_tokens": 36, + "cvar95_kl": 3.4864540100097656, + "delta_nll": 0.7593507766723633, + "max_kl": 3.8500118255615234, + "mean_kl": 0.9004418849945068, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.7222222089767456 + }, + { + "candidate_nll": 1.6386096477508545, + "code_tokens": 158, + "cvar95_kl": 4.4020795822143555, + "delta_nll": 0.9944648146629333, + "max_kl": 5.722357749938965, + "mean_kl": 1.0565747022628784, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.6392405033111572 + }, + { + "candidate_nll": 2.3318722248077393, + "code_tokens": 92, + "cvar95_kl": 4.1030707359313965, + "delta_nll": 1.0163371562957764, + "max_kl": 4.4947357177734375, + "mean_kl": 0.8861488699913025, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.6739130616188049 + }, + { + "candidate_nll": 2.086777448654175, + "code_tokens": 102, + "cvar95_kl": 3.5176117420196533, + "delta_nll": 1.1727347373962402, + "max_kl": 3.944974660873413, + "mean_kl": 1.0406348705291748, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 2.0272433757781982, + "code_tokens": 171, + "cvar95_kl": 6.224020004272461, + "delta_nll": 1.4172732830047607, + "max_kl": 9.511417388916016, + "mean_kl": 1.298951506614685, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.6023392081260681 + }, + { + "candidate_nll": 2.1939501762390137, + "code_tokens": 40, + "cvar95_kl": 4.299635410308838, + "delta_nll": 0.9890860319137573, + "max_kl": 5.411892890930176, + "mean_kl": 0.8343197703361511, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 2.5446465015411377, + "code_tokens": 48, + "cvar95_kl": 5.993013381958008, + "delta_nll": 1.3086326122283936, + "max_kl": 7.740626811981201, + "mean_kl": 1.3074978590011597, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 2.487170934677124, + "code_tokens": 24, + "cvar95_kl": 4.310807228088379, + "delta_nll": 0.8822505474090576, + "max_kl": 5.371591091156006, + "mean_kl": 1.1495672464370728, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.8458298444747925, + "code_tokens": 48, + "cvar95_kl": 3.349505662918091, + "delta_nll": 0.8587256669998169, + "max_kl": 4.369076728820801, + "mean_kl": 0.7298898696899414, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 2.3076891899108887, + "code_tokens": 25, + "cvar95_kl": 3.0880017280578613, + "delta_nll": 0.9208060503005981, + "max_kl": 3.8127524852752686, + "mean_kl": 0.6821739077568054, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 2.7114932537078857, + "code_tokens": 30, + "cvar95_kl": 3.0199058055877686, + "delta_nll": 1.212748408317566, + "max_kl": 3.604248046875, + "mean_kl": 0.7702673673629761, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 1.8144229650497437, + "code_tokens": 19, + "cvar95_kl": 1.3330941200256348, + "delta_nll": 0.38760507106781006, + "max_kl": 1.3330941200256348, + "mean_kl": 0.31136083602905273, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 1.192251443862915, + "code_tokens": 12, + "cvar95_kl": 0.9215580821037292, + "delta_nll": 0.11317896842956543, + "max_kl": 0.9215580821037292, + "mean_kl": 0.2525990307331085, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.8333333134651184 + }, + { + "candidate_nll": 2.240018844604492, + "code_tokens": 37, + "cvar95_kl": 3.7033417224884033, + "delta_nll": 0.8997091054916382, + "max_kl": 4.424580097198486, + "mean_kl": 0.8891890048980713, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.6216216087341309 + }, + { + "candidate_nll": 1.990130066871643, + "code_tokens": 33, + "cvar95_kl": 3.969839096069336, + "delta_nll": 0.7847002744674683, + "max_kl": 5.106308460235596, + "mean_kl": 0.7631688117980957, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 2.250197649002075, + "code_tokens": 94, + "cvar95_kl": 6.027419090270996, + "delta_nll": 1.3980878591537476, + "max_kl": 7.83931303024292, + "mean_kl": 1.253301978111267, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.5957446694374084 + }, + { + "candidate_nll": 1.4279842376708984, + "code_tokens": 90, + "cvar95_kl": 6.60873556137085, + "delta_nll": 0.8032193183898926, + "max_kl": 12.522397994995117, + "mean_kl": 0.9029083847999573, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.7555555701255798 + }, + { + "candidate_nll": 1.8147941827774048, + "code_tokens": 47, + "cvar95_kl": 2.71942400932312, + "delta_nll": 0.3670302629470825, + "max_kl": 3.8847901821136475, + "mean_kl": 0.4795018434524536, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.8297872543334961 + }, + { + "candidate_nll": 1.5986686944961548, + "code_tokens": 61, + "cvar95_kl": 3.6304187774658203, + "delta_nll": 0.6748345494270325, + "max_kl": 4.022110939025879, + "mean_kl": 0.8659714460372925, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.7049180269241333 + }, + { + "candidate_nll": 1.6789050102233887, + "code_tokens": 39, + "cvar95_kl": 2.813908576965332, + "delta_nll": 0.45539021492004395, + "max_kl": 3.5261008739471436, + "mean_kl": 0.47447508573532104, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.8461538553237915 + }, + { + "candidate_nll": 2.109078884124756, + "code_tokens": 152, + "cvar95_kl": 6.218183517456055, + "delta_nll": 1.1769170761108398, + "max_kl": 8.306821823120117, + "mean_kl": 1.2882473468780518, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.6052631735801697 + }, + { + "candidate_nll": 2.3201797008514404, + "code_tokens": 100, + "cvar95_kl": 6.639056205749512, + "delta_nll": 1.3511972427368164, + "max_kl": 8.059879302978516, + "mean_kl": 1.3853851556777954, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.5400000214576721 + }, + { + "candidate_nll": 2.2011401653289795, + "code_tokens": 89, + "cvar95_kl": 6.332533836364746, + "delta_nll": 1.5802123546600342, + "max_kl": 6.869868278503418, + "mean_kl": 1.3072713613510132, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.6516854166984558 + }, + { + "candidate_nll": 2.657830238342285, + "code_tokens": 40, + "cvar95_kl": 8.30843734741211, + "delta_nll": 1.3468290567398071, + "max_kl": 8.587394714355469, + "mean_kl": 1.1955164670944214, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.6499999761581421 + }, + { + "candidate_nll": 1.7099981307983398, + "code_tokens": 76, + "cvar95_kl": 4.167205810546875, + "delta_nll": 1.0084311962127686, + "max_kl": 6.013448715209961, + "mean_kl": 0.9633511900901794, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.9731255769729614, + "code_tokens": 56, + "cvar95_kl": 4.398210048675537, + "delta_nll": 0.7241382598876953, + "max_kl": 6.134716987609863, + "mean_kl": 0.8193928599357605, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 2.729743480682373, + "code_tokens": 34, + "cvar95_kl": 5.040526866912842, + "delta_nll": 1.240313172340393, + "max_kl": 6.230953693389893, + "mean_kl": 1.0351667404174805, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.5882353186607361 + }, + { + "candidate_nll": 2.0100228786468506, + "code_tokens": 96, + "cvar95_kl": 4.323399543762207, + "delta_nll": 1.331911325454712, + "max_kl": 4.757334232330322, + "mean_kl": 1.1791504621505737, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.5625 + }, + { + "candidate_nll": 1.7526545524597168, + "code_tokens": 451, + "cvar95_kl": 4.104779243469238, + "delta_nll": 1.0539593696594238, + "max_kl": 6.036643028259277, + "mean_kl": 0.9818211197853088, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.6430155038833618 + }, + { + "candidate_nll": 2.444943428039551, + "code_tokens": 37, + "cvar95_kl": 3.694509744644165, + "delta_nll": 0.8326337337493896, + "max_kl": 3.999302864074707, + "mean_kl": 0.9852608442306519, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.5945945978164673 + }, + { + "candidate_nll": 1.9620448350906372, + "code_tokens": 42, + "cvar95_kl": 2.923443078994751, + "delta_nll": 0.7204469442367554, + "max_kl": 3.1802635192871094, + "mean_kl": 0.82347571849823, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 1.8664860725402832, + "code_tokens": 33, + "cvar95_kl": 3.0055153369903564, + "delta_nll": 0.8497301340103149, + "max_kl": 3.621992826461792, + "mean_kl": 0.885533332824707, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.5757575631141663 + }, + { + "candidate_nll": 1.7591516971588135, + "code_tokens": 43, + "cvar95_kl": 2.899519681930542, + "delta_nll": 0.8331000208854675, + "max_kl": 4.016707420349121, + "mean_kl": 0.7433263063430786, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.7906976938247681 + }, + { + "candidate_nll": 1.6405245065689087, + "code_tokens": 73, + "cvar95_kl": 5.414511203765869, + "delta_nll": 0.9040880799293518, + "max_kl": 6.315585613250732, + "mean_kl": 0.9300881028175354, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.6438356041908264 + }, + { + "candidate_nll": 1.6506397724151611, + "code_tokens": 15, + "cvar95_kl": 2.4385921955108643, + "delta_nll": 0.7381635904312134, + "max_kl": 2.4385921955108643, + "mean_kl": 0.5809726715087891, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.7333333492279053 + }, + { + "candidate_nll": 2.0242769718170166, + "code_tokens": 34, + "cvar95_kl": 5.796079635620117, + "delta_nll": 0.7651970386505127, + "max_kl": 6.729874610900879, + "mean_kl": 0.956612765789032, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 2.102324962615967, + "code_tokens": 136, + "cvar95_kl": 3.926332473754883, + "delta_nll": 0.9938417673110962, + "max_kl": 7.819728374481201, + "mean_kl": 0.9722220301628113, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.6029411554336548 + }, + { + "candidate_nll": 1.7426315546035767, + "code_tokens": 17, + "cvar95_kl": 1.8987520933151245, + "delta_nll": 0.4964754581451416, + "max_kl": 1.8987520933151245, + "mean_kl": 0.6199160218238831, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 1.6832304000854492, + "code_tokens": 33, + "cvar95_kl": 4.113759517669678, + "delta_nll": 0.8901757001876831, + "max_kl": 5.589293956756592, + "mean_kl": 0.7941402196884155, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.6935676336288452, + "code_tokens": 38, + "cvar95_kl": 3.698211431503296, + "delta_nll": 0.8511208295822144, + "max_kl": 4.401806354522705, + "mean_kl": 0.7705394625663757, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.7368420958518982 + }, + { + "candidate_nll": 1.3498765230178833, + "code_tokens": 61, + "cvar95_kl": 3.5474791526794434, + "delta_nll": 0.9523519277572632, + "max_kl": 5.123800277709961, + "mean_kl": 0.8148788213729858, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.6721311211585999 + }, + { + "candidate_nll": 1.8594954013824463, + "code_tokens": 143, + "cvar95_kl": 5.1855549812316895, + "delta_nll": 1.1237759590148926, + "max_kl": 6.7812347412109375, + "mean_kl": 1.1749000549316406, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.5804196000099182 + }, + { + "candidate_nll": 1.6676586866378784, + "code_tokens": 41, + "cvar95_kl": 3.284538507461548, + "delta_nll": 0.5785927772521973, + "max_kl": 3.730771780014038, + "mean_kl": 0.7019644379615784, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.7560975551605225 + }, + { + "candidate_nll": 0.9157043099403381, + "code_tokens": 67, + "cvar95_kl": 4.738999366760254, + "delta_nll": 0.5510225296020508, + "max_kl": 6.981268405914307, + "mean_kl": 0.7067411541938782, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.7761194109916687 + }, + { + "candidate_nll": 2.416775703430176, + "code_tokens": 76, + "cvar95_kl": 3.672361373901367, + "delta_nll": 1.0265157222747803, + "max_kl": 4.410040378570557, + "mean_kl": 0.9892578125, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.5657894611358643 + }, + { + "candidate_nll": 1.694367527961731, + "code_tokens": 137, + "cvar95_kl": 4.1326823234558105, + "delta_nll": 1.0585148334503174, + "max_kl": 4.647824764251709, + "mean_kl": 0.9976497888565063, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.6715328693389893 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 1.9546477794647217, + "median": 0.7915209531784058, + "minimum": -0.09443438053131104, + "q25": 0.5939276367425919, + "q75": 1.0005292445421219 + }, + "task_macro": { + "candidate_nll": 1.8177594603300096, + "delta_nll": 0.8037128749489785, + "mean_kl": 0.8153626573383808, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.6911910179853439 + }, + "token_weighted": { + "candidate_nll": 1.827401041984558, + "cvar95_kl": 4.839138984680176, + "delta_nll": 0.9105634689331055, + "max_kl": 12.522397994995117, + "mean_kl": 0.9145802855491638, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.665189802646637 + } + }, + "read_risk_l0_stochastic_seed_2339": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.3937035074234005, + "macro_mean_kl": 4.546505675315857, + "macro_top1_agreement": 0.2693974330425262, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 5.541626895904541, + "macro_mean_kl": 5.72391662979126, + "macro_top1_agreement": 0.19541577589511872, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 6.114364700317383, + "macro_mean_kl": 6.234801998138428, + "macro_top1_agreement": 0.15419903153181075, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 7.259360958099365, + "macro_mean_kl": 7.3192968139648436, + "macro_top1_agreement": 0.11592043879628182, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 7.999780654907227, + "code_tokens": 98, + "cvar95_kl": 17.835737228393555, + "delta_nll": 7.272924900054932, + "max_kl": 20.443260192871094, + "mean_kl": 7.655306339263916, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.11224489659070969 + }, + { + "candidate_nll": 5.410630702972412, + "code_tokens": 19, + "cvar95_kl": 13.70484733581543, + "delta_nll": 4.251217842102051, + "max_kl": 13.70484733581543, + "mean_kl": 4.3441667556762695, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 5.927031517028809, + "code_tokens": 39, + "cvar95_kl": 11.051970481872559, + "delta_nll": 4.91168212890625, + "max_kl": 12.318289756774902, + "mean_kl": 5.031731128692627, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 6.359016418457031, + "code_tokens": 24, + "cvar95_kl": 11.61469841003418, + "delta_nll": 4.912726879119873, + "max_kl": 12.7107572555542, + "mean_kl": 4.8465704917907715, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.807863712310791, + "code_tokens": 25, + "cvar95_kl": 13.326201438903809, + "delta_nll": 4.4001054763793945, + "max_kl": 15.358230590820312, + "mean_kl": 4.786117076873779, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 7.175792694091797, + "code_tokens": 55, + "cvar95_kl": 16.698928833007812, + "delta_nll": 6.280180931091309, + "max_kl": 18.842792510986328, + "mean_kl": 6.380944728851318, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.002734661102295, + "code_tokens": 16, + "cvar95_kl": 13.917835235595703, + "delta_nll": 4.1732940673828125, + "max_kl": 13.917835235595703, + "mean_kl": 4.217881679534912, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 10.98082160949707, + "code_tokens": 199, + "cvar95_kl": 19.5179386138916, + "delta_nll": 9.758928298950195, + "max_kl": 22.78620719909668, + "mean_kl": 9.91106128692627, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.04522613063454628 + }, + { + "candidate_nll": 7.134607791900635, + "code_tokens": 28, + "cvar95_kl": 13.178363800048828, + "delta_nll": 5.5685296058654785, + "max_kl": 13.36964225769043, + "mean_kl": 5.9141950607299805, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 6.978120803833008, + "code_tokens": 87, + "cvar95_kl": 13.745725631713867, + "delta_nll": 6.105138301849365, + "max_kl": 15.591426849365234, + "mean_kl": 6.142224311828613, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.16091954708099365 + }, + { + "candidate_nll": 7.834475994110107, + "code_tokens": 35, + "cvar95_kl": 16.21403694152832, + "delta_nll": 6.6015119552612305, + "max_kl": 17.28212547302246, + "mean_kl": 7.046483516693115, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.969610691070557, + "code_tokens": 62, + "cvar95_kl": 15.299945831298828, + "delta_nll": 7.330318927764893, + "max_kl": 18.41578483581543, + "mean_kl": 7.240665435791016, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 7.064022541046143, + "code_tokens": 58, + "cvar95_kl": 14.33913803100586, + "delta_nll": 5.958964824676514, + "max_kl": 16.367172241210938, + "mean_kl": 5.526932716369629, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.063475131988525, + "code_tokens": 79, + "cvar95_kl": 15.050533294677734, + "delta_nll": 5.9044270515441895, + "max_kl": 16.4996280670166, + "mean_kl": 5.88665246963501, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.16455696523189545 + }, + { + "candidate_nll": 6.834928512573242, + "code_tokens": 58, + "cvar95_kl": 16.077943801879883, + "delta_nll": 5.741808891296387, + "max_kl": 18.818443298339844, + "mean_kl": 6.251506328582764, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 6.195492267608643, + "code_tokens": 48, + "cvar95_kl": 10.848221778869629, + "delta_nll": 5.033294677734375, + "max_kl": 12.135080337524414, + "mean_kl": 5.465519428253174, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.379671573638916, + "code_tokens": 37, + "cvar95_kl": 16.634450912475586, + "delta_nll": 6.254239082336426, + "max_kl": 17.17158317565918, + "mean_kl": 5.987957000732422, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 7.314334392547607, + "code_tokens": 61, + "cvar95_kl": 16.20990562438965, + "delta_nll": 6.606927394866943, + "max_kl": 20.315143585205078, + "mean_kl": 6.399477958679199, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.19672131538391113 + }, + { + "candidate_nll": 7.277837753295898, + "code_tokens": 77, + "cvar95_kl": 17.401187896728516, + "delta_nll": 6.508118152618408, + "max_kl": 18.876033782958984, + "mean_kl": 6.253069877624512, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.19480518996715546 + }, + { + "candidate_nll": 8.118515014648438, + "code_tokens": 98, + "cvar95_kl": 16.483016967773438, + "delta_nll": 7.096811771392822, + "max_kl": 18.586788177490234, + "mean_kl": 7.355885982513428, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 8.919602394104004, + "code_tokens": 162, + "cvar95_kl": 18.89950180053711, + "delta_nll": 8.334451675415039, + "max_kl": 25.34468650817871, + "mean_kl": 8.483884811401367, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.12345679104328156 + }, + { + "candidate_nll": 7.328088760375977, + "code_tokens": 103, + "cvar95_kl": 16.102243423461914, + "delta_nll": 6.583622932434082, + "max_kl": 18.94607162475586, + "mean_kl": 6.618315696716309, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.13592232763767242 + }, + { + "candidate_nll": 7.880439758300781, + "code_tokens": 73, + "cvar95_kl": 14.892207145690918, + "delta_nll": 6.561469078063965, + "max_kl": 15.685386657714844, + "mean_kl": 6.6553497314453125, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.12328767031431198 + }, + { + "candidate_nll": 7.525879859924316, + "code_tokens": 119, + "cvar95_kl": 17.10489845275879, + "delta_nll": 6.542906284332275, + "max_kl": 18.52760887145996, + "mean_kl": 6.923072338104248, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.15126051008701324 + }, + { + "candidate_nll": 4.095948219299316, + "code_tokens": 17, + "cvar95_kl": 9.128386497497559, + "delta_nll": 2.9064249992370605, + "max_kl": 9.128386497497559, + "mean_kl": 3.5439260005950928, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 5.889840126037598, + "code_tokens": 56, + "cvar95_kl": 11.35069751739502, + "delta_nll": 4.464138507843018, + "max_kl": 14.028042793273926, + "mean_kl": 4.364567279815674, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.2321428507566452 + }, + { + "candidate_nll": 7.285951614379883, + "code_tokens": 57, + "cvar95_kl": 16.310150146484375, + "delta_nll": 6.818591117858887, + "max_kl": 19.17317008972168, + "mean_kl": 6.897828102111816, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.17543859779834747 + }, + { + "candidate_nll": 7.858747959136963, + "code_tokens": 63, + "cvar95_kl": 16.203109741210938, + "delta_nll": 7.2260026931762695, + "max_kl": 17.532007217407227, + "mean_kl": 7.310332298278809, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 9.234641075134277, + "code_tokens": 200, + "cvar95_kl": 18.324146270751953, + "delta_nll": 8.396705627441406, + "max_kl": 21.34641456604004, + "mean_kl": 8.460341453552246, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.10999999940395355 + }, + { + "candidate_nll": 7.687392711639404, + "code_tokens": 30, + "cvar95_kl": 11.809520721435547, + "delta_nll": 6.603189945220947, + "max_kl": 12.413446426391602, + "mean_kl": 6.16714334487915, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 5.898873329162598, + "code_tokens": 28, + "cvar95_kl": 16.15195655822754, + "delta_nll": 5.282522201538086, + "max_kl": 17.96308708190918, + "mean_kl": 5.346791744232178, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.190456867218018, + "code_tokens": 26, + "cvar95_kl": 11.873533248901367, + "delta_nll": 5.419500350952148, + "max_kl": 12.088775634765625, + "mean_kl": 5.137136459350586, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 6.514983654022217, + "code_tokens": 54, + "cvar95_kl": 14.189186096191406, + "delta_nll": 5.682042121887207, + "max_kl": 15.245013236999512, + "mean_kl": 5.852825164794922, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 8.100337982177734, + "code_tokens": 47, + "cvar95_kl": 16.183820724487305, + "delta_nll": 7.143601894378662, + "max_kl": 19.577634811401367, + "mean_kl": 7.229353427886963, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.08510638028383255 + }, + { + "candidate_nll": 7.543529033660889, + "code_tokens": 93, + "cvar95_kl": 14.690633773803711, + "delta_nll": 6.808999538421631, + "max_kl": 17.687963485717773, + "mean_kl": 6.707052707672119, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.09677419066429138 + }, + { + "candidate_nll": 4.910989761352539, + "code_tokens": 30, + "cvar95_kl": 9.849830627441406, + "delta_nll": 4.051615238189697, + "max_kl": 10.715736389160156, + "mean_kl": 4.1570563316345215, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 6.705653667449951, + "code_tokens": 89, + "cvar95_kl": 16.021703720092773, + "delta_nll": 5.637150287628174, + "max_kl": 19.535776138305664, + "mean_kl": 5.639558792114258, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.21348313987255096 + }, + { + "candidate_nll": 7.464160919189453, + "code_tokens": 70, + "cvar95_kl": 15.45363712310791, + "delta_nll": 6.499911785125732, + "max_kl": 19.174592971801758, + "mean_kl": 6.435127258300781, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.768429279327393, + "code_tokens": 26, + "cvar95_kl": 8.997989654541016, + "delta_nll": 4.1552019119262695, + "max_kl": 9.108329772949219, + "mean_kl": 4.511185646057129, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 7.349405288696289, + "code_tokens": 48, + "cvar95_kl": 14.337994575500488, + "delta_nll": 6.391698360443115, + "max_kl": 15.561829566955566, + "mean_kl": 6.492677211761475, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 5.81911039352417, + "code_tokens": 29, + "cvar95_kl": 14.303764343261719, + "delta_nll": 4.949962615966797, + "max_kl": 15.31222152709961, + "mean_kl": 4.727010726928711, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 5.0879998207092285, + "code_tokens": 17, + "cvar95_kl": 12.713203430175781, + "delta_nll": 3.974836826324463, + "max_kl": 12.713203430175781, + "mean_kl": 3.709146022796631, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 6.5510478019714355, + "code_tokens": 37, + "cvar95_kl": 15.59457778930664, + "delta_nll": 5.630831718444824, + "max_kl": 16.143136978149414, + "mean_kl": 5.85267972946167, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.13513512909412384 + }, + { + "candidate_nll": 8.396812438964844, + "code_tokens": 121, + "cvar95_kl": 16.85604476928711, + "delta_nll": 7.885410785675049, + "max_kl": 18.717876434326172, + "mean_kl": 7.923160552978516, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.11570248007774353 + }, + { + "candidate_nll": 5.169050216674805, + "code_tokens": 30, + "cvar95_kl": 10.730598449707031, + "delta_nll": 3.7702064514160156, + "max_kl": 10.896947860717773, + "mean_kl": 3.7976582050323486, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.784791469573975, + "code_tokens": 74, + "cvar95_kl": 16.167247772216797, + "delta_nll": 5.886319637298584, + "max_kl": 21.780643463134766, + "mean_kl": 5.811023712158203, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.12162162363529205 + }, + { + "candidate_nll": 5.929165363311768, + "code_tokens": 53, + "cvar95_kl": 11.83897876739502, + "delta_nll": 5.094741344451904, + "max_kl": 13.019099235534668, + "mean_kl": 5.0515313148498535, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.11320754885673523 + }, + { + "candidate_nll": 4.146872520446777, + "code_tokens": 21, + "cvar95_kl": 7.475359916687012, + "delta_nll": 2.2874767780303955, + "max_kl": 7.550463676452637, + "mean_kl": 3.0428836345672607, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.1834747791290283, + "code_tokens": 20, + "cvar95_kl": 4.60949182510376, + "delta_nll": 1.821061134338379, + "max_kl": 4.60949182510376, + "mean_kl": 1.764696478843689, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 8.459267616271973, + "code_tokens": 122, + "cvar95_kl": 16.285602569580078, + "delta_nll": 7.6402692794799805, + "max_kl": 17.687301635742188, + "mean_kl": 7.975423812866211, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.10655737668275833 + }, + { + "candidate_nll": 7.896559715270996, + "code_tokens": 94, + "cvar95_kl": 15.976964950561523, + "delta_nll": 7.056419849395752, + "max_kl": 17.65463638305664, + "mean_kl": 7.063248634338379, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.13829787075519562 + }, + { + "candidate_nll": 3.6385104656219482, + "code_tokens": 12, + "cvar95_kl": 7.728235244750977, + "delta_nll": 2.573208808898926, + "max_kl": 7.728235244750977, + "mean_kl": 2.5576374530792236, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.748922348022461, + "code_tokens": 38, + "cvar95_kl": 15.374537467956543, + "delta_nll": 6.242522239685059, + "max_kl": 17.24933433532715, + "mean_kl": 6.682720184326172, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.10526315867900848 + }, + { + "candidate_nll": 7.5184102058410645, + "code_tokens": 58, + "cvar95_kl": 18.292781829833984, + "delta_nll": 6.613623142242432, + "max_kl": 20.36016082763672, + "mean_kl": 6.862143516540527, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.12068965286016464 + }, + { + "candidate_nll": 7.3111138343811035, + "code_tokens": 54, + "cvar95_kl": 16.340187072753906, + "delta_nll": 6.714849948883057, + "max_kl": 17.960744857788086, + "mean_kl": 6.9126811027526855, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.365097522735596, + "code_tokens": 40, + "cvar95_kl": 17.65353012084961, + "delta_nll": 5.433557510375977, + "max_kl": 18.037790298461914, + "mean_kl": 5.942745685577393, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 6.8056721687316895, + "code_tokens": 115, + "cvar95_kl": 14.935090065002441, + "delta_nll": 6.293126106262207, + "max_kl": 18.167125701904297, + "mean_kl": 6.2461347579956055, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 6.15835428237915, + "code_tokens": 60, + "cvar95_kl": 12.79296588897705, + "delta_nll": 5.633740425109863, + "max_kl": 14.974174499511719, + "mean_kl": 5.838118076324463, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 8.925254821777344, + "code_tokens": 124, + "cvar95_kl": 16.606475830078125, + "delta_nll": 8.155267715454102, + "max_kl": 18.45932388305664, + "mean_kl": 8.069876670837402, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.10483870655298233 + }, + { + "candidate_nll": 8.556020736694336, + "code_tokens": 91, + "cvar95_kl": 17.589984893798828, + "delta_nll": 7.785272121429443, + "max_kl": 20.700302124023438, + "mean_kl": 8.014094352722168, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.09890110045671463 + }, + { + "candidate_nll": 8.718636512756348, + "code_tokens": 125, + "cvar95_kl": 16.742673873901367, + "delta_nll": 7.898719310760498, + "max_kl": 19.636489868164062, + "mean_kl": 7.9224348068237305, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.09600000083446503 + }, + { + "candidate_nll": 5.552863597869873, + "code_tokens": 26, + "cvar95_kl": 12.080047607421875, + "delta_nll": 4.41698694229126, + "max_kl": 12.233746528625488, + "mean_kl": 4.176104545593262, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 6.0065598487854, + "code_tokens": 24, + "cvar95_kl": 10.097207069396973, + "delta_nll": 3.766134738922119, + "max_kl": 11.603795051574707, + "mean_kl": 4.101385593414307, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 9.724821090698242, + "code_tokens": 180, + "cvar95_kl": 18.873973846435547, + "delta_nll": 8.870638847351074, + "max_kl": 22.480091094970703, + "mean_kl": 8.83507251739502, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 7.968076229095459, + "code_tokens": 40, + "cvar95_kl": 14.966304779052734, + "delta_nll": 6.593267440795898, + "max_kl": 15.867354393005371, + "mean_kl": 7.181577205657959, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 6.157110691070557, + "code_tokens": 72, + "cvar95_kl": 13.98104190826416, + "delta_nll": 5.1821136474609375, + "max_kl": 15.463722229003906, + "mean_kl": 5.366453170776367, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 4.498697280883789, + "code_tokens": 20, + "cvar95_kl": 8.97920036315918, + "delta_nll": 2.921126127243042, + "max_kl": 8.97920036315918, + "mean_kl": 3.1653010845184326, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 6.3789963722229, + "code_tokens": 76, + "cvar95_kl": 14.974018096923828, + "delta_nll": 5.388351917266846, + "max_kl": 16.1617431640625, + "mean_kl": 5.69752836227417, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.464747905731201, + "code_tokens": 42, + "cvar95_kl": 12.346320152282715, + "delta_nll": 6.093006610870361, + "max_kl": 13.44980525970459, + "mean_kl": 5.617464065551758, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 4.781329154968262, + "code_tokens": 28, + "cvar95_kl": 11.482465744018555, + "delta_nll": 4.198105812072754, + "max_kl": 12.382795333862305, + "mean_kl": 3.917635679244995, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 7.817653656005859, + "code_tokens": 58, + "cvar95_kl": 16.316394805908203, + "delta_nll": 6.784176826477051, + "max_kl": 18.959075927734375, + "mean_kl": 6.9427571296691895, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.1034482792019844 + }, + { + "candidate_nll": 5.9860920906066895, + "code_tokens": 25, + "cvar95_kl": 15.904703140258789, + "delta_nll": 5.117442607879639, + "max_kl": 19.46404266357422, + "mean_kl": 5.347589492797852, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.8786468505859375, + "code_tokens": 88, + "cvar95_kl": 15.861793518066406, + "delta_nll": 6.913639068603516, + "max_kl": 17.098365783691406, + "mean_kl": 7.079182147979736, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.181339263916016, + "code_tokens": 42, + "cvar95_kl": 12.635223388671875, + "delta_nll": 5.053822040557861, + "max_kl": 14.679112434387207, + "mean_kl": 5.2740702629089355, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 6.193907260894775, + "code_tokens": 24, + "cvar95_kl": 11.730216026306152, + "delta_nll": 5.374356269836426, + "max_kl": 12.223974227905273, + "mean_kl": 5.0162034034729, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.033883094787598, + "code_tokens": 24, + "cvar95_kl": 8.21326732635498, + "delta_nll": 3.3490116596221924, + "max_kl": 9.243989944458008, + "mean_kl": 3.359031915664673, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.7058820724487305, + "code_tokens": 43, + "cvar95_kl": 11.818339347839355, + "delta_nll": 5.638702869415283, + "max_kl": 12.387765884399414, + "mean_kl": 5.535354137420654, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.20930232107639313 + }, + { + "candidate_nll": 5.798099517822266, + "code_tokens": 24, + "cvar95_kl": 12.871503829956055, + "delta_nll": 4.911880016326904, + "max_kl": 15.134382247924805, + "mean_kl": 4.871102333068848, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.8984992504119873, + "code_tokens": 14, + "cvar95_kl": 4.833341598510742, + "delta_nll": 1.5858736038208008, + "max_kl": 4.833341598510742, + "mean_kl": 2.131908655166626, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 6.702897071838379, + "code_tokens": 41, + "cvar95_kl": 14.034405708312988, + "delta_nll": 6.211853981018066, + "max_kl": 14.95837688446045, + "mean_kl": 5.937404155731201, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.1463414579629898 + }, + { + "candidate_nll": 5.606973171234131, + "code_tokens": 32, + "cvar95_kl": 11.618907928466797, + "delta_nll": 4.618633270263672, + "max_kl": 11.742294311523438, + "mean_kl": 5.284224987030029, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 6.846023082733154, + "code_tokens": 53, + "cvar95_kl": 16.74529266357422, + "delta_nll": 6.0843281745910645, + "max_kl": 20.723461151123047, + "mean_kl": 5.812427043914795, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.18867924809455872 + }, + { + "candidate_nll": 4.801464080810547, + "code_tokens": 51, + "cvar95_kl": 10.116583824157715, + "delta_nll": 4.423172950744629, + "max_kl": 10.287872314453125, + "mean_kl": 4.263458728790283, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.19607843458652496 + }, + { + "candidate_nll": 8.403035163879395, + "code_tokens": 36, + "cvar95_kl": 17.49567985534668, + "delta_nll": 6.710879802703857, + "max_kl": 19.026796340942383, + "mean_kl": 7.305871963500977, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.110510349273682, + "code_tokens": 24, + "cvar95_kl": 14.319328308105469, + "delta_nll": 4.549757957458496, + "max_kl": 16.183975219726562, + "mean_kl": 5.0233893394470215, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 5.715150833129883, + "code_tokens": 39, + "cvar95_kl": 14.749557495117188, + "delta_nll": 4.400816440582275, + "max_kl": 17.40807342529297, + "mean_kl": 4.847113609313965, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 8.089132308959961, + "code_tokens": 84, + "cvar95_kl": 16.800724029541016, + "delta_nll": 7.390719890594482, + "max_kl": 19.03764533996582, + "mean_kl": 7.405287265777588, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 6.721980571746826, + "code_tokens": 32, + "cvar95_kl": 17.09701156616211, + "delta_nll": 5.799212455749512, + "max_kl": 17.409950256347656, + "mean_kl": 5.754696846008301, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 6.091429710388184, + "code_tokens": 19, + "cvar95_kl": 12.71577262878418, + "delta_nll": 4.781266212463379, + "max_kl": 12.71577262878418, + "mean_kl": 4.936309814453125, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 6.672041416168213, + "code_tokens": 50, + "cvar95_kl": 13.012763977050781, + "delta_nll": 5.854064464569092, + "max_kl": 13.272233963012695, + "mean_kl": 5.973624229431152, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 6.98530912399292, + "code_tokens": 71, + "cvar95_kl": 15.91196346282959, + "delta_nll": 6.485495567321777, + "max_kl": 16.4097900390625, + "mean_kl": 6.460043430328369, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.18309858441352844 + }, + { + "candidate_nll": 7.047937870025635, + "code_tokens": 28, + "cvar95_kl": 11.685052871704102, + "delta_nll": 5.659969329833984, + "max_kl": 12.974746704101562, + "mean_kl": 5.907883644104004, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.6830525398254395, + "code_tokens": 76, + "cvar95_kl": 15.278136253356934, + "delta_nll": 5.813791275024414, + "max_kl": 19.812965393066406, + "mean_kl": 5.664852142333984, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.1315789520740509 + }, + { + "candidate_nll": 6.604979515075684, + "code_tokens": 34, + "cvar95_kl": 13.694269180297852, + "delta_nll": 5.642394065856934, + "max_kl": 13.897171020507812, + "mean_kl": 5.752444267272949, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 5.053215503692627, + "code_tokens": 11, + "cvar95_kl": 8.381197929382324, + "delta_nll": 2.9058237075805664, + "max_kl": 8.381197929382324, + "mean_kl": 3.149045705795288, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 6.872141361236572, + "code_tokens": 31, + "cvar95_kl": 13.81723403930664, + "delta_nll": 5.455873966217041, + "max_kl": 14.405634880065918, + "mean_kl": 5.835296154022217, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 7.215824604034424, + "code_tokens": 118, + "cvar95_kl": 14.0382661819458, + "delta_nll": 6.369048118591309, + "max_kl": 15.307075500488281, + "mean_kl": 6.498455047607422, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.11864406615495682 + }, + { + "candidate_nll": 8.093099594116211, + "code_tokens": 60, + "cvar95_kl": 17.5562801361084, + "delta_nll": 7.317403793334961, + "max_kl": 19.346017837524414, + "mean_kl": 7.151413440704346, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 7.2205071449279785, + "code_tokens": 47, + "cvar95_kl": 15.797688484191895, + "delta_nll": 6.325057029724121, + "max_kl": 17.93082618713379, + "mean_kl": 6.425187587738037, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.12765957415103912 + }, + { + "candidate_nll": 8.257096290588379, + "code_tokens": 84, + "cvar95_kl": 16.45967674255371, + "delta_nll": 7.105082035064697, + "max_kl": 19.12945556640625, + "mean_kl": 7.231865882873535, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 5.834633827209473, + "code_tokens": 27, + "cvar95_kl": 12.061792373657227, + "delta_nll": 4.840607643127441, + "max_kl": 12.418611526489258, + "mean_kl": 5.109142780303955, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 5.356016635894775, + "code_tokens": 19, + "cvar95_kl": 10.865323066711426, + "delta_nll": 3.744109630584717, + "max_kl": 10.865323066711426, + "mean_kl": 4.1366658210754395, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 7.725261688232422, + "code_tokens": 114, + "cvar95_kl": 16.47029685974121, + "delta_nll": 6.952249050140381, + "max_kl": 23.567359924316406, + "mean_kl": 7.044285774230957, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.05263157933950424 + }, + { + "candidate_nll": 7.008149147033691, + "code_tokens": 40, + "cvar95_kl": 12.833202362060547, + "delta_nll": 5.2802557945251465, + "max_kl": 13.46038818359375, + "mean_kl": 5.590648174285889, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 6.082307815551758, + "code_tokens": 25, + "cvar95_kl": 9.203156471252441, + "delta_nll": 4.596629619598389, + "max_kl": 9.253789901733398, + "mean_kl": 5.032571792602539, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.717785835266113, + "code_tokens": 23, + "cvar95_kl": 10.398738861083984, + "delta_nll": 3.947096824645996, + "max_kl": 10.88022232055664, + "mean_kl": 3.986232280731201, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 8.404343605041504, + "code_tokens": 77, + "cvar95_kl": 15.905930519104004, + "delta_nll": 7.387993812561035, + "max_kl": 17.07470703125, + "mean_kl": 7.294581890106201, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.11688311398029327 + }, + { + "candidate_nll": 4.982236862182617, + "code_tokens": 19, + "cvar95_kl": 13.548501968383789, + "delta_nll": 3.8745713233947754, + "max_kl": 13.548501968383789, + "mean_kl": 3.461730480194092, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 5.40920877456665, + "code_tokens": 37, + "cvar95_kl": 12.62231159210205, + "delta_nll": 4.425964832305908, + "max_kl": 13.764900207519531, + "mean_kl": 5.0552592277526855, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 5.75615930557251, + "code_tokens": 33, + "cvar95_kl": 13.484342575073242, + "delta_nll": 4.372994422912598, + "max_kl": 15.642751693725586, + "mean_kl": 4.644308090209961, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 6.9848761558532715, + "code_tokens": 95, + "cvar95_kl": 15.296972274780273, + "delta_nll": 6.3250346183776855, + "max_kl": 17.073169708251953, + "mean_kl": 6.2538676261901855, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 9.188053131103516, + "code_tokens": 125, + "cvar95_kl": 20.678083419799805, + "delta_nll": 8.17489242553711, + "max_kl": 26.2041015625, + "mean_kl": 8.30583381652832, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.12800000607967377 + }, + { + "candidate_nll": 9.183415412902832, + "code_tokens": 168, + "cvar95_kl": 18.149625778198242, + "delta_nll": 8.470367431640625, + "max_kl": 22.336986541748047, + "mean_kl": 8.265302658081055, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.1071428582072258 + }, + { + "candidate_nll": 6.07883358001709, + "code_tokens": 31, + "cvar95_kl": 12.88611888885498, + "delta_nll": 4.763431549072266, + "max_kl": 13.095601081848145, + "mean_kl": 5.094316005706787, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 8.263697624206543, + "code_tokens": 83, + "cvar95_kl": 17.417490005493164, + "delta_nll": 7.501990795135498, + "max_kl": 19.339702606201172, + "mean_kl": 7.365175724029541, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.09638553857803345 + }, + { + "candidate_nll": 6.021536827087402, + "code_tokens": 50, + "cvar95_kl": 12.370884895324707, + "delta_nll": 5.32159423828125, + "max_kl": 13.641308784484863, + "mean_kl": 5.285440444946289, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 6.167575359344482, + "code_tokens": 69, + "cvar95_kl": 13.122328758239746, + "delta_nll": 5.800393581390381, + "max_kl": 14.744141578674316, + "mean_kl": 5.6994476318359375, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.14492753148078918 + }, + { + "candidate_nll": 6.372243881225586, + "code_tokens": 50, + "cvar95_kl": 13.75304889678955, + "delta_nll": 5.287347793579102, + "max_kl": 14.115933418273926, + "mean_kl": 5.5893402099609375, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.2199999988079071 + }, + { + "candidate_nll": 9.79059886932373, + "code_tokens": 161, + "cvar95_kl": 19.18048858642578, + "delta_nll": 8.78681755065918, + "max_kl": 22.38246726989746, + "mean_kl": 8.884422302246094, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.07453416287899017 + }, + { + "candidate_nll": 6.147061347961426, + "code_tokens": 53, + "cvar95_kl": 12.51416301727295, + "delta_nll": 5.49128532409668, + "max_kl": 14.243257522583008, + "mean_kl": 5.475334167480469, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.16981132328510284 + }, + { + "candidate_nll": 9.690007209777832, + "code_tokens": 97, + "cvar95_kl": 18.884456634521484, + "delta_nll": 9.244479179382324, + "max_kl": 19.99490737915039, + "mean_kl": 9.142219543457031, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.11340206116437912 + }, + { + "candidate_nll": 4.896730422973633, + "code_tokens": 22, + "cvar95_kl": 10.66364860534668, + "delta_nll": 3.5829174518585205, + "max_kl": 11.0335111618042, + "mean_kl": 3.987413167953491, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 7.500644207000732, + "code_tokens": 34, + "cvar95_kl": 18.181716918945312, + "delta_nll": 6.631740093231201, + "max_kl": 18.38522720336914, + "mean_kl": 6.78463077545166, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 7.20500373840332, + "code_tokens": 81, + "cvar95_kl": 17.02579116821289, + "delta_nll": 6.346360683441162, + "max_kl": 19.6846866607666, + "mean_kl": 6.685362339019775, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.12345679104328156 + }, + { + "candidate_nll": 4.228307247161865, + "code_tokens": 17, + "cvar95_kl": 11.686140060424805, + "delta_nll": 2.9662039279937744, + "max_kl": 11.686140060424805, + "mean_kl": 3.095698118209839, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 7.390003681182861, + "code_tokens": 181, + "cvar95_kl": 17.95045280456543, + "delta_nll": 6.564746379852295, + "max_kl": 21.75688362121582, + "mean_kl": 6.82565450668335, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.10497237741947174 + }, + { + "candidate_nll": 6.002388000488281, + "code_tokens": 63, + "cvar95_kl": 13.834821701049805, + "delta_nll": 5.021979331970215, + "max_kl": 17.773006439208984, + "mean_kl": 5.346158981323242, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 5.720582485198975, + "code_tokens": 35, + "cvar95_kl": 10.918184280395508, + "delta_nll": 5.027059555053711, + "max_kl": 11.54118537902832, + "mean_kl": 4.891810417175293, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 4.557638168334961, + "code_tokens": 25, + "cvar95_kl": 11.164588928222656, + "delta_nll": 3.7528507709503174, + "max_kl": 13.020818710327148, + "mean_kl": 3.714855670928955, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 6.245771408081055, + "code_tokens": 60, + "cvar95_kl": 15.190094947814941, + "delta_nll": 5.157405853271484, + "max_kl": 17.567943572998047, + "mean_kl": 5.67853307723999, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.782344818115234, + "code_tokens": 86, + "cvar95_kl": 17.838539123535156, + "delta_nll": 7.870697975158691, + "max_kl": 19.921340942382812, + "mean_kl": 7.631186008453369, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.151162788271904 + }, + { + "candidate_nll": 7.07009744644165, + "code_tokens": 40, + "cvar95_kl": 13.842477798461914, + "delta_nll": 6.143931865692139, + "max_kl": 14.315739631652832, + "mean_kl": 6.227888584136963, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.880716800689697, + "code_tokens": 27, + "cvar95_kl": 11.557903289794922, + "delta_nll": 5.317156791687012, + "max_kl": 11.984046936035156, + "mean_kl": 5.346262454986572, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.712613582611084, + "code_tokens": 52, + "cvar95_kl": 16.302522659301758, + "delta_nll": 5.623030662536621, + "max_kl": 20.411540985107422, + "mean_kl": 5.623936176300049, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.17307692766189575 + }, + { + "candidate_nll": 8.746241569519043, + "code_tokens": 71, + "cvar95_kl": 19.48733901977539, + "delta_nll": 8.00436782836914, + "max_kl": 22.44593048095703, + "mean_kl": 8.13500690460205, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.1267605572938919 + }, + { + "candidate_nll": 6.524031162261963, + "code_tokens": 25, + "cvar95_kl": 11.967222213745117, + "delta_nll": 5.422179222106934, + "max_kl": 12.257397651672363, + "mean_kl": 5.520922660827637, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.677319526672363, + "code_tokens": 103, + "cvar95_kl": 15.73502254486084, + "delta_nll": 7.082823753356934, + "max_kl": 18.542821884155273, + "mean_kl": 7.130124092102051, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.11650485545396805 + }, + { + "candidate_nll": 9.2035551071167, + "code_tokens": 116, + "cvar95_kl": 20.498456954956055, + "delta_nll": 8.146017074584961, + "max_kl": 24.68031120300293, + "mean_kl": 8.077812194824219, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.12931033968925476 + }, + { + "candidate_nll": 9.002032279968262, + "code_tokens": 107, + "cvar95_kl": 15.79393482208252, + "delta_nll": 8.353686332702637, + "max_kl": 18.22105598449707, + "mean_kl": 8.272685050964355, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.10280373692512512 + }, + { + "candidate_nll": 6.425909042358398, + "code_tokens": 54, + "cvar95_kl": 12.25799560546875, + "delta_nll": 5.482871055603027, + "max_kl": 12.738582611083984, + "mean_kl": 5.519399642944336, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.8600382804870605, + "code_tokens": 46, + "cvar95_kl": 13.359199523925781, + "delta_nll": 6.2266459465026855, + "max_kl": 15.317357063293457, + "mean_kl": 6.185436725616455, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.15217390656471252 + }, + { + "candidate_nll": 7.747014999389648, + "code_tokens": 170, + "cvar95_kl": 19.08658218383789, + "delta_nll": 7.417868137359619, + "max_kl": 24.84630012512207, + "mean_kl": 7.3473124504089355, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.10588235408067703 + }, + { + "candidate_nll": 7.280138969421387, + "code_tokens": 45, + "cvar95_kl": 12.725850105285645, + "delta_nll": 5.853446960449219, + "max_kl": 13.655635833740234, + "mean_kl": 5.842446327209473, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 5.928070068359375, + "code_tokens": 24, + "cvar95_kl": 9.08138656616211, + "delta_nll": 4.519766807556152, + "max_kl": 9.666980743408203, + "mean_kl": 4.3002777099609375, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.894879341125488, + "code_tokens": 78, + "cvar95_kl": 17.632667541503906, + "delta_nll": 5.732919692993164, + "max_kl": 20.634016036987305, + "mean_kl": 6.099061012268066, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 6.243806838989258, + "code_tokens": 33, + "cvar95_kl": 15.34309196472168, + "delta_nll": 5.613249778747559, + "max_kl": 15.988541603088379, + "mean_kl": 5.450748443603516, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 6.715158462524414, + "code_tokens": 32, + "cvar95_kl": 13.616884231567383, + "delta_nll": 5.599488735198975, + "max_kl": 16.403324127197266, + "mean_kl": 5.524694919586182, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 6.331667423248291, + "code_tokens": 77, + "cvar95_kl": 12.989389419555664, + "delta_nll": 5.342917442321777, + "max_kl": 13.994707107543945, + "mean_kl": 5.431491374969482, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 9.539262771606445, + "code_tokens": 222, + "cvar95_kl": 17.803709030151367, + "delta_nll": 9.117012023925781, + "max_kl": 20.401744842529297, + "mean_kl": 9.014388084411621, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.054054055362939835 + }, + { + "candidate_nll": 7.594597339630127, + "code_tokens": 86, + "cvar95_kl": 16.05810546875, + "delta_nll": 6.546002388000488, + "max_kl": 19.470806121826172, + "mean_kl": 6.61421012878418, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.11627907305955887 + }, + { + "candidate_nll": 6.157800197601318, + "code_tokens": 31, + "cvar95_kl": 13.67531681060791, + "delta_nll": 5.8237504959106445, + "max_kl": 14.517333030700684, + "mean_kl": 5.894681453704834, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 4.832849502563477, + "code_tokens": 34, + "cvar95_kl": 14.013256072998047, + "delta_nll": 4.321249485015869, + "max_kl": 16.205324172973633, + "mean_kl": 4.243719577789307, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 6.300419330596924, + "code_tokens": 38, + "cvar95_kl": 13.6538667678833, + "delta_nll": 4.912959098815918, + "max_kl": 14.725404739379883, + "mean_kl": 5.166810989379883, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 8.606120109558105, + "code_tokens": 93, + "cvar95_kl": 17.485523223876953, + "delta_nll": 7.643075466156006, + "max_kl": 19.62320327758789, + "mean_kl": 7.572474956512451, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.13978494703769684 + }, + { + "candidate_nll": 8.373787879943848, + "code_tokens": 70, + "cvar95_kl": 19.158784866333008, + "delta_nll": 7.595182418823242, + "max_kl": 20.255348205566406, + "mean_kl": 7.601937294006348, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 8.573891639709473, + "code_tokens": 67, + "cvar95_kl": 19.363628387451172, + "delta_nll": 7.763399600982666, + "max_kl": 20.93771743774414, + "mean_kl": 7.933835029602051, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.11940298229455948 + }, + { + "candidate_nll": 6.85093355178833, + "code_tokens": 66, + "cvar95_kl": 12.422243118286133, + "delta_nll": 5.56480598449707, + "max_kl": 13.553519248962402, + "mean_kl": 5.820588111877441, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 5.822625637054443, + "code_tokens": 34, + "cvar95_kl": 9.325614929199219, + "delta_nll": 4.810331344604492, + "max_kl": 10.003253936767578, + "mean_kl": 4.337172985076904, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 6.158653259277344, + "code_tokens": 64, + "cvar95_kl": 12.188623428344727, + "delta_nll": 5.248627662658691, + "max_kl": 14.50631332397461, + "mean_kl": 5.370842933654785, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.140625 + }, + { + "candidate_nll": 7.528486728668213, + "code_tokens": 56, + "cvar95_kl": 15.238049507141113, + "delta_nll": 6.218131065368652, + "max_kl": 16.96524429321289, + "mean_kl": 6.565185546875, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.689460277557373, + "code_tokens": 21, + "cvar95_kl": 11.045279502868652, + "delta_nll": 3.564377784729004, + "max_kl": 12.51699161529541, + "mean_kl": 3.479917526245117, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 6.20604133605957, + "code_tokens": 60, + "cvar95_kl": 12.642890930175781, + "delta_nll": 5.540700912475586, + "max_kl": 14.780373573303223, + "mean_kl": 5.769472599029541, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.11666666716337204 + }, + { + "candidate_nll": 7.656118392944336, + "code_tokens": 29, + "cvar95_kl": 20.35753059387207, + "delta_nll": 6.302342414855957, + "max_kl": 22.450647354125977, + "mean_kl": 6.479215621948242, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 6.6550421714782715, + "code_tokens": 34, + "cvar95_kl": 16.88304328918457, + "delta_nll": 5.932997226715088, + "max_kl": 16.99598503112793, + "mean_kl": 5.6819329261779785, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 8.889690399169922, + "code_tokens": 88, + "cvar95_kl": 17.99001693725586, + "delta_nll": 7.970582962036133, + "max_kl": 20.228857040405273, + "mean_kl": 8.200961112976074, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.09090909361839294 + }, + { + "candidate_nll": 5.6042280197143555, + "code_tokens": 17, + "cvar95_kl": 9.046873092651367, + "delta_nll": 5.061004638671875, + "max_kl": 9.046873092651367, + "mean_kl": 4.566992282867432, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 6.077763080596924, + "code_tokens": 35, + "cvar95_kl": 11.606023788452148, + "delta_nll": 4.187215805053711, + "max_kl": 12.28648567199707, + "mean_kl": 4.864452838897705, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.11428571492433548 + }, + { + "candidate_nll": 6.271656036376953, + "code_tokens": 48, + "cvar95_kl": 13.226001739501953, + "delta_nll": 5.3818793296813965, + "max_kl": 13.77161693572998, + "mean_kl": 5.385038375854492, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.755795955657959, + "code_tokens": 130, + "cvar95_kl": 13.906326293945312, + "delta_nll": 6.765523433685303, + "max_kl": 16.8376407623291, + "mean_kl": 6.559697151184082, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.16923077404499054 + }, + { + "candidate_nll": 9.499576568603516, + "code_tokens": 65, + "cvar95_kl": 18.435426712036133, + "delta_nll": 8.404637336730957, + "max_kl": 21.21596908569336, + "mean_kl": 8.39793872833252, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.13846154510974884 + }, + { + "candidate_nll": 8.299006462097168, + "code_tokens": 145, + "cvar95_kl": 15.856008529663086, + "delta_nll": 7.423991680145264, + "max_kl": 17.370222091674805, + "mean_kl": 7.592480659484863, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.08275862038135529 + }, + { + "candidate_nll": 9.093622207641602, + "code_tokens": 80, + "cvar95_kl": 17.155447006225586, + "delta_nll": 8.377923011779785, + "max_kl": 18.29189682006836, + "mean_kl": 8.281046867370605, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.08749999850988388 + }, + { + "candidate_nll": 6.09935998916626, + "code_tokens": 70, + "cvar95_kl": 12.32916259765625, + "delta_nll": 5.517223358154297, + "max_kl": 12.785040855407715, + "mean_kl": 5.771823883056641, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.15714286267757416 + }, + { + "candidate_nll": 5.54110050201416, + "code_tokens": 25, + "cvar95_kl": 8.395151138305664, + "delta_nll": 4.052811622619629, + "max_kl": 8.68779182434082, + "mean_kl": 4.099954128265381, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 7.414370059967041, + "code_tokens": 50, + "cvar95_kl": 15.269986152648926, + "delta_nll": 5.759986877441406, + "max_kl": 17.42552375793457, + "mean_kl": 6.399582386016846, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 6.141435146331787, + "code_tokens": 48, + "cvar95_kl": 13.928746223449707, + "delta_nll": 5.343392372131348, + "max_kl": 15.390373229980469, + "mean_kl": 5.483839511871338, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.947845935821533, + "code_tokens": 104, + "cvar95_kl": 17.555137634277344, + "delta_nll": 7.603773593902588, + "max_kl": 20.019712448120117, + "mean_kl": 7.481783866882324, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.10576923191547394 + }, + { + "candidate_nll": 7.147290229797363, + "code_tokens": 68, + "cvar95_kl": 13.593790054321289, + "delta_nll": 6.157296180725098, + "max_kl": 15.621122360229492, + "mean_kl": 6.2218241691589355, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.1617647111415863 + }, + { + "candidate_nll": 7.654260635375977, + "code_tokens": 145, + "cvar95_kl": 15.291488647460938, + "delta_nll": 6.899348258972168, + "max_kl": 16.5147762298584, + "mean_kl": 6.94154167175293, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.12413793057203293 + }, + { + "candidate_nll": 4.895870208740234, + "code_tokens": 41, + "cvar95_kl": 11.418740272521973, + "delta_nll": 4.050982475280762, + "max_kl": 12.78597640991211, + "mean_kl": 4.516117572784424, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.24390244483947754 + }, + { + "candidate_nll": 7.476015567779541, + "code_tokens": 53, + "cvar95_kl": 13.626815795898438, + "delta_nll": 6.714893341064453, + "max_kl": 14.4298677444458, + "mean_kl": 6.700835227966309, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.11320754885673523 + }, + { + "candidate_nll": 7.837395191192627, + "code_tokens": 58, + "cvar95_kl": 16.060564041137695, + "delta_nll": 7.045165538787842, + "max_kl": 16.988540649414062, + "mean_kl": 6.701242923736572, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 6.0314106941223145, + "code_tokens": 24, + "cvar95_kl": 12.793469429016113, + "delta_nll": 4.299350738525391, + "max_kl": 13.800666809082031, + "mean_kl": 4.931140422821045, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.339982032775879, + "code_tokens": 89, + "cvar95_kl": 18.24087905883789, + "delta_nll": 7.348128318786621, + "max_kl": 18.80072784423828, + "mean_kl": 7.7861809730529785, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.14606741070747375 + }, + { + "candidate_nll": 7.631298542022705, + "code_tokens": 101, + "cvar95_kl": 15.322218894958496, + "delta_nll": 7.146222114562988, + "max_kl": 18.581615447998047, + "mean_kl": 7.074252128601074, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.07920791953802109 + }, + { + "candidate_nll": 6.817539691925049, + "code_tokens": 33, + "cvar95_kl": 12.870338439941406, + "delta_nll": 5.359982013702393, + "max_kl": 13.393421173095703, + "mean_kl": 5.787983417510986, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 8.135756492614746, + "code_tokens": 50, + "cvar95_kl": 15.278292655944824, + "delta_nll": 7.089090824127197, + "max_kl": 17.636022567749023, + "mean_kl": 7.186122894287109, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 6.3604631423950195, + "code_tokens": 58, + "cvar95_kl": 12.051200866699219, + "delta_nll": 4.951710224151611, + "max_kl": 13.509994506835938, + "mean_kl": 4.814277172088623, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.2586206793785095 + }, + { + "candidate_nll": 6.494304656982422, + "code_tokens": 62, + "cvar95_kl": 15.21762466430664, + "delta_nll": 5.515145778656006, + "max_kl": 17.940370559692383, + "mean_kl": 5.679433345794678, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.17741934955120087 + }, + { + "candidate_nll": 7.677577495574951, + "code_tokens": 42, + "cvar95_kl": 13.918449401855469, + "delta_nll": 6.98588752746582, + "max_kl": 14.357190132141113, + "mean_kl": 6.982532978057861, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.0714285746216774 + }, + { + "candidate_nll": 4.0707197189331055, + "code_tokens": 15, + "cvar95_kl": 10.07060432434082, + "delta_nll": 3.1459786891937256, + "max_kl": 10.07060432434082, + "mean_kl": 3.439279794692993, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.445966720581055, + "code_tokens": 57, + "cvar95_kl": 13.896645545959473, + "delta_nll": 5.707184791564941, + "max_kl": 15.76910400390625, + "mean_kl": 6.053256988525391, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.14035087823867798 + }, + { + "candidate_nll": 6.258063793182373, + "code_tokens": 58, + "cvar95_kl": 14.955857276916504, + "delta_nll": 5.2145280838012695, + "max_kl": 18.502126693725586, + "mean_kl": 5.201476573944092, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 6.550996780395508, + "code_tokens": 45, + "cvar95_kl": 17.236413955688477, + "delta_nll": 5.679025650024414, + "max_kl": 22.608091354370117, + "mean_kl": 6.011916160583496, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.663221836090088, + "code_tokens": 32, + "cvar95_kl": 9.87942886352539, + "delta_nll": 3.6339914798736572, + "max_kl": 11.264080047607422, + "mean_kl": 4.737403869628906, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.0625 + }, + { + "candidate_nll": 5.885008335113525, + "code_tokens": 39, + "cvar95_kl": 13.051095008850098, + "delta_nll": 4.564342021942139, + "max_kl": 14.441055297851562, + "mean_kl": 5.305434703826904, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 8.819962501525879, + "code_tokens": 136, + "cvar95_kl": 18.101539611816406, + "delta_nll": 8.409201622009277, + "max_kl": 20.879758834838867, + "mean_kl": 8.272472381591797, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.07352941483259201 + }, + { + "candidate_nll": 7.282601833343506, + "code_tokens": 50, + "cvar95_kl": 16.972551345825195, + "delta_nll": 6.061845302581787, + "max_kl": 18.802457809448242, + "mean_kl": 6.075840950012207, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.437401294708252, + "code_tokens": 34, + "cvar95_kl": 15.008551597595215, + "delta_nll": 6.57148551940918, + "max_kl": 15.893885612487793, + "mean_kl": 6.5401787757873535, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.11764705926179886 + }, + { + "candidate_nll": 7.449322700500488, + "code_tokens": 44, + "cvar95_kl": 13.78484058380127, + "delta_nll": 6.270532608032227, + "max_kl": 14.232701301574707, + "mean_kl": 6.317654609680176, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.15909090638160706 + }, + { + "candidate_nll": 5.4574971199035645, + "code_tokens": 36, + "cvar95_kl": 13.313512802124023, + "delta_nll": 4.024075508117676, + "max_kl": 14.761033058166504, + "mean_kl": 4.460270404815674, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.656991481781006, + "code_tokens": 46, + "cvar95_kl": 15.26767349243164, + "delta_nll": 5.85383415222168, + "max_kl": 19.873085021972656, + "mean_kl": 5.749712944030762, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.239130437374115 + }, + { + "candidate_nll": 7.740879058837891, + "code_tokens": 48, + "cvar95_kl": 16.516746520996094, + "delta_nll": 6.560539245605469, + "max_kl": 17.374542236328125, + "mean_kl": 6.973575115203857, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 5.77192497253418, + "code_tokens": 25, + "cvar95_kl": 13.128108978271484, + "delta_nll": 4.876513957977295, + "max_kl": 13.228632926940918, + "mean_kl": 4.280581474304199, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 7.70359468460083, + "code_tokens": 69, + "cvar95_kl": 13.762032508850098, + "delta_nll": 6.199315547943115, + "max_kl": 14.832937240600586, + "mean_kl": 6.545238494873047, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.11594203114509583 + }, + { + "candidate_nll": 6.172492027282715, + "code_tokens": 60, + "cvar95_kl": 14.605008125305176, + "delta_nll": 5.207034111022949, + "max_kl": 18.38483428955078, + "mean_kl": 5.71514892578125, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.11666666716337204 + }, + { + "candidate_nll": 6.37539005279541, + "code_tokens": 49, + "cvar95_kl": 12.075594902038574, + "delta_nll": 5.460912704467773, + "max_kl": 13.685662269592285, + "mean_kl": 5.3191118240356445, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.16326530277729034 + }, + { + "candidate_nll": 6.607344150543213, + "code_tokens": 44, + "cvar95_kl": 13.079177856445312, + "delta_nll": 4.906816005706787, + "max_kl": 15.922648429870605, + "mean_kl": 4.708566665649414, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.20454545319080353 + }, + { + "candidate_nll": 8.425280570983887, + "code_tokens": 81, + "cvar95_kl": 15.543128967285156, + "delta_nll": 7.2798662185668945, + "max_kl": 17.628868103027344, + "mean_kl": 7.543609142303467, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.09876543283462524 + }, + { + "candidate_nll": 6.308358192443848, + "code_tokens": 26, + "cvar95_kl": 12.574920654296875, + "delta_nll": 4.8716230392456055, + "max_kl": 13.24545669555664, + "mean_kl": 5.493600845336914, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 7.336829662322998, + "code_tokens": 32, + "cvar95_kl": 11.7181396484375, + "delta_nll": 6.456908702850342, + "max_kl": 11.730353355407715, + "mean_kl": 6.430055141448975, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 6.473125457763672, + "code_tokens": 48, + "cvar95_kl": 15.900141716003418, + "delta_nll": 5.525383472442627, + "max_kl": 16.51864242553711, + "mean_kl": 6.253688335418701, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 9.42063045501709, + "code_tokens": 180, + "cvar95_kl": 18.094924926757812, + "delta_nll": 8.721309661865234, + "max_kl": 21.42770767211914, + "mean_kl": 8.682064056396484, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.803012847900391, + "code_tokens": 38, + "cvar95_kl": 12.541892051696777, + "delta_nll": 5.08125114440918, + "max_kl": 13.503870964050293, + "mean_kl": 5.028451442718506, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 7.564737796783447, + "code_tokens": 90, + "cvar95_kl": 15.51514720916748, + "delta_nll": 6.869676113128662, + "max_kl": 19.01331329345703, + "mean_kl": 6.994110107421875, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 5.97198486328125, + "code_tokens": 44, + "cvar95_kl": 14.285653114318848, + "delta_nll": 5.397354602813721, + "max_kl": 15.366844177246094, + "mean_kl": 5.619800090789795, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.20454545319080353 + }, + { + "candidate_nll": 6.791094779968262, + "code_tokens": 60, + "cvar95_kl": 15.77185344696045, + "delta_nll": 6.047194480895996, + "max_kl": 16.862268447875977, + "mean_kl": 6.4205145835876465, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 6.410240650177002, + "code_tokens": 69, + "cvar95_kl": 12.740572929382324, + "delta_nll": 5.407442569732666, + "max_kl": 14.224863052368164, + "mean_kl": 5.712657451629639, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.10144927352666855 + }, + { + "candidate_nll": 7.648650646209717, + "code_tokens": 71, + "cvar95_kl": 16.91039276123047, + "delta_nll": 7.131088733673096, + "max_kl": 18.206113815307617, + "mean_kl": 7.508502960205078, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.11267605423927307 + }, + { + "candidate_nll": 5.106801986694336, + "code_tokens": 22, + "cvar95_kl": 9.893945693969727, + "delta_nll": 3.864128828048706, + "max_kl": 11.477739334106445, + "mean_kl": 3.8686602115631104, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 8.936052322387695, + "code_tokens": 215, + "cvar95_kl": 17.724082946777344, + "delta_nll": 8.382272720336914, + "max_kl": 24.085630416870117, + "mean_kl": 8.377118110656738, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.060465116053819656 + }, + { + "candidate_nll": 7.4004106521606445, + "code_tokens": 26, + "cvar95_kl": 13.000633239746094, + "delta_nll": 6.29762077331543, + "max_kl": 13.424379348754883, + "mean_kl": 5.583054542541504, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 6.636744022369385, + "code_tokens": 33, + "cvar95_kl": 13.313827514648438, + "delta_nll": 5.768497467041016, + "max_kl": 13.508152961730957, + "mean_kl": 5.920689582824707, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 4.41438627243042, + "code_tokens": 18, + "cvar95_kl": 10.516018867492676, + "delta_nll": 3.478238105773926, + "max_kl": 10.516018867492676, + "mean_kl": 3.925140857696533, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 6.982627868652344, + "code_tokens": 71, + "cvar95_kl": 13.06805419921875, + "delta_nll": 5.702566146850586, + "max_kl": 13.92884635925293, + "mean_kl": 5.890363693237305, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.14084507524967194 + }, + { + "candidate_nll": 6.269446849822998, + "code_tokens": 107, + "cvar95_kl": 13.62496566772461, + "delta_nll": 5.4880452156066895, + "max_kl": 16.019271850585938, + "mean_kl": 5.641359806060791, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.13084112107753754 + }, + { + "candidate_nll": 7.886298179626465, + "code_tokens": 40, + "cvar95_kl": 15.529233932495117, + "delta_nll": 5.831273555755615, + "max_kl": 16.033126831054688, + "mean_kl": 6.775275230407715, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 6.043203830718994, + "code_tokens": 32, + "cvar95_kl": 13.894096374511719, + "delta_nll": 4.6789069175720215, + "max_kl": 14.361711502075195, + "mean_kl": 4.559809684753418, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 7.7560858726501465, + "code_tokens": 130, + "cvar95_kl": 14.5603609085083, + "delta_nll": 7.056832313537598, + "max_kl": 19.763208389282227, + "mean_kl": 6.980265140533447, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.10769230872392654 + }, + { + "candidate_nll": 6.555789947509766, + "code_tokens": 27, + "cvar95_kl": 10.588420867919922, + "delta_nll": 5.0843682289123535, + "max_kl": 10.633716583251953, + "mean_kl": 4.8612213134765625, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.425975799560547, + "code_tokens": 42, + "cvar95_kl": 13.361842155456543, + "delta_nll": 6.453035354614258, + "max_kl": 14.045574188232422, + "mean_kl": 6.252964019775391, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 5.148980617523193, + "code_tokens": 30, + "cvar95_kl": 10.963231086730957, + "delta_nll": 4.403467178344727, + "max_kl": 12.020306587219238, + "mean_kl": 4.661140441894531, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 7.816669940948486, + "code_tokens": 58, + "cvar95_kl": 15.445307731628418, + "delta_nll": 6.39743709564209, + "max_kl": 17.433626174926758, + "mean_kl": 6.319615364074707, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 6.295745372772217, + "code_tokens": 34, + "cvar95_kl": 13.309207916259766, + "delta_nll": 5.065518379211426, + "max_kl": 15.674260139465332, + "mean_kl": 5.224404811859131, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 9.27071475982666, + "code_tokens": 208, + "cvar95_kl": 16.894346237182617, + "delta_nll": 8.569252014160156, + "max_kl": 19.82659339904785, + "mean_kl": 8.587056159973145, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.04326923191547394 + }, + { + "candidate_nll": 5.523287296295166, + "code_tokens": 83, + "cvar95_kl": 11.76159381866455, + "delta_nll": 4.6243815422058105, + "max_kl": 12.595597267150879, + "mean_kl": 4.479137897491455, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.21686747670173645 + }, + { + "candidate_nll": 9.362483978271484, + "code_tokens": 170, + "cvar95_kl": 17.117692947387695, + "delta_nll": 8.83992862701416, + "max_kl": 19.109392166137695, + "mean_kl": 8.609871864318848, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.04117647185921669 + }, + { + "candidate_nll": 5.508368492126465, + "code_tokens": 40, + "cvar95_kl": 17.979896545410156, + "delta_nll": 4.726947784423828, + "max_kl": 21.315608978271484, + "mean_kl": 5.205855846405029, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 6.8739914894104, + "code_tokens": 31, + "cvar95_kl": 11.136964797973633, + "delta_nll": 5.62728214263916, + "max_kl": 11.234213829040527, + "mean_kl": 5.545742511749268, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.12903225421905518 + }, + { + "candidate_nll": 6.838761806488037, + "code_tokens": 42, + "cvar95_kl": 16.67966651916504, + "delta_nll": 6.043992042541504, + "max_kl": 19.90818214416504, + "mean_kl": 6.406925678253174, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 5.937256336212158, + "code_tokens": 27, + "cvar95_kl": 11.866704940795898, + "delta_nll": 4.1681904792785645, + "max_kl": 12.037908554077148, + "mean_kl": 4.928133010864258, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 5.914498805999756, + "code_tokens": 25, + "cvar95_kl": 14.235834121704102, + "delta_nll": 3.714688301086426, + "max_kl": 15.024880409240723, + "mean_kl": 4.826309680938721, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.931127548217773, + "code_tokens": 40, + "cvar95_kl": 12.51211166381836, + "delta_nll": 4.362740993499756, + "max_kl": 13.066761016845703, + "mean_kl": 4.6199631690979, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.201029300689697, + "code_tokens": 24, + "cvar95_kl": 10.674899101257324, + "delta_nll": 4.816674709320068, + "max_kl": 10.873260498046875, + "mean_kl": 4.719874858856201, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 6.140890598297119, + "code_tokens": 28, + "cvar95_kl": 13.03745174407959, + "delta_nll": 5.258145809173584, + "max_kl": 13.288458824157715, + "mean_kl": 4.910155296325684, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.230101585388184, + "code_tokens": 71, + "cvar95_kl": 15.551570892333984, + "delta_nll": 5.751236915588379, + "max_kl": 18.100767135620117, + "mean_kl": 5.566334247589111, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.22535210847854614 + }, + { + "candidate_nll": 3.797518491744995, + "code_tokens": 24, + "cvar95_kl": 10.772237777709961, + "delta_nll": 3.268157720565796, + "max_kl": 11.142518997192383, + "mean_kl": 3.515197515487671, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 7.698220729827881, + "code_tokens": 34, + "cvar95_kl": 17.697528839111328, + "delta_nll": 6.426639556884766, + "max_kl": 17.775449752807617, + "mean_kl": 7.202352046966553, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.14705882966518402 + }, + { + "candidate_nll": 7.846887588500977, + "code_tokens": 64, + "cvar95_kl": 18.925931930541992, + "delta_nll": 6.727593421936035, + "max_kl": 21.094974517822266, + "mean_kl": 7.282514572143555, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.073118686676025, + "code_tokens": 45, + "cvar95_kl": 13.232747077941895, + "delta_nll": 5.366527557373047, + "max_kl": 14.009428024291992, + "mean_kl": 5.419180870056152, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 7.6414713859558105, + "code_tokens": 48, + "cvar95_kl": 15.570513725280762, + "delta_nll": 6.442767143249512, + "max_kl": 15.730910301208496, + "mean_kl": 6.970928192138672, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 5.928684711456299, + "code_tokens": 21, + "cvar95_kl": 13.31320571899414, + "delta_nll": 5.2284836769104, + "max_kl": 15.509464263916016, + "mean_kl": 5.059712886810303, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 5.389525890350342, + "code_tokens": 24, + "cvar95_kl": 13.372550964355469, + "delta_nll": 4.6351118087768555, + "max_kl": 16.9333553314209, + "mean_kl": 4.8432698249816895, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.239695072174072, + "code_tokens": 63, + "cvar95_kl": 14.930732727050781, + "delta_nll": 5.667767524719238, + "max_kl": 16.020509719848633, + "mean_kl": 5.84164571762085, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.1269841343164444 + }, + { + "candidate_nll": 5.420933246612549, + "code_tokens": 22, + "cvar95_kl": 10.173861503601074, + "delta_nll": 4.142174243927002, + "max_kl": 12.02816104888916, + "mean_kl": 3.902874708175659, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 5.845660209655762, + "code_tokens": 25, + "cvar95_kl": 12.817198753356934, + "delta_nll": 4.716998100280762, + "max_kl": 14.625042915344238, + "mean_kl": 4.951282501220703, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 5.435689449310303, + "code_tokens": 21, + "cvar95_kl": 11.108255386352539, + "delta_nll": 3.4753637313842773, + "max_kl": 11.729001998901367, + "mean_kl": 3.722264289855957, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.3992183208465576, + "code_tokens": 24, + "cvar95_kl": 6.574622631072998, + "delta_nll": 1.9328862428665161, + "max_kl": 7.422287464141846, + "mean_kl": 2.527939796447754, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 6.443289279937744, + "code_tokens": 15, + "cvar95_kl": 11.823312759399414, + "delta_nll": 4.649801254272461, + "max_kl": 11.823312759399414, + "mean_kl": 4.6845526695251465, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 6.431325435638428, + "code_tokens": 59, + "cvar95_kl": 14.935809135437012, + "delta_nll": 5.650468826293945, + "max_kl": 15.029162406921387, + "mean_kl": 5.842156410217285, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.16949152946472168 + }, + { + "candidate_nll": 6.340743541717529, + "code_tokens": 54, + "cvar95_kl": 15.023612976074219, + "delta_nll": 5.406625270843506, + "max_kl": 15.710550308227539, + "mean_kl": 5.555948734283447, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.20370370149612427 + }, + { + "candidate_nll": 6.486109256744385, + "code_tokens": 28, + "cvar95_kl": 14.0604248046875, + "delta_nll": 4.534250259399414, + "max_kl": 15.277999877929688, + "mean_kl": 5.751652240753174, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.370389938354492, + "code_tokens": 46, + "cvar95_kl": 12.817264556884766, + "delta_nll": 5.285664081573486, + "max_kl": 14.33746337890625, + "mean_kl": 5.678009033203125, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 5.86468505859375, + "code_tokens": 27, + "cvar95_kl": 11.691720008850098, + "delta_nll": 4.349862575531006, + "max_kl": 12.517875671386719, + "mean_kl": 5.035086154937744, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 7.161983966827393, + "code_tokens": 97, + "cvar95_kl": 14.082858085632324, + "delta_nll": 5.904553413391113, + "max_kl": 16.169307708740234, + "mean_kl": 5.9708356857299805, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.0824742242693901 + }, + { + "candidate_nll": 5.42823600769043, + "code_tokens": 24, + "cvar95_kl": 12.647750854492188, + "delta_nll": 4.626003742218018, + "max_kl": 13.365960121154785, + "mean_kl": 4.364198684692383, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 6.415546894073486, + "code_tokens": 30, + "cvar95_kl": 11.084076881408691, + "delta_nll": 5.150398254394531, + "max_kl": 11.801158905029297, + "mean_kl": 5.699494361877441, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.923188209533691, + "code_tokens": 38, + "cvar95_kl": 17.551712036132812, + "delta_nll": 6.490118503570557, + "max_kl": 18.348251342773438, + "mean_kl": 7.303981781005859, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 4.6543169021606445, + "code_tokens": 21, + "cvar95_kl": 9.186378479003906, + "delta_nll": 3.1665377616882324, + "max_kl": 9.24585247039795, + "mean_kl": 3.098698854446411, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 7.913503646850586, + "code_tokens": 65, + "cvar95_kl": 17.016338348388672, + "delta_nll": 7.376542091369629, + "max_kl": 18.24526596069336, + "mean_kl": 7.210206508636475, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.13846154510974884 + }, + { + "candidate_nll": 5.245248794555664, + "code_tokens": 27, + "cvar95_kl": 11.403945922851562, + "delta_nll": 4.46072244644165, + "max_kl": 11.541254997253418, + "mean_kl": 4.515115261077881, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.589524745941162, + "code_tokens": 35, + "cvar95_kl": 18.488971710205078, + "delta_nll": 5.865708827972412, + "max_kl": 19.491151809692383, + "mean_kl": 6.287327289581299, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.22857142984867096 + }, + { + "candidate_nll": 8.141219139099121, + "code_tokens": 75, + "cvar95_kl": 16.829547882080078, + "delta_nll": 7.2166948318481445, + "max_kl": 21.940902709960938, + "mean_kl": 7.464452266693115, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.06666667014360428 + }, + { + "candidate_nll": 5.071516036987305, + "code_tokens": 25, + "cvar95_kl": 10.982147216796875, + "delta_nll": 3.4297804832458496, + "max_kl": 12.098268508911133, + "mean_kl": 4.292150497436523, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 7.608098983764648, + "code_tokens": 52, + "cvar95_kl": 14.858756065368652, + "delta_nll": 6.684110164642334, + "max_kl": 15.727328300476074, + "mean_kl": 7.0392045974731445, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.13461539149284363 + }, + { + "candidate_nll": 8.212756156921387, + "code_tokens": 110, + "cvar95_kl": 17.110822677612305, + "delta_nll": 7.602096080780029, + "max_kl": 19.4195499420166, + "mean_kl": 7.715754985809326, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.1090909093618393 + }, + { + "candidate_nll": 5.497500419616699, + "code_tokens": 25, + "cvar95_kl": 11.401530265808105, + "delta_nll": 4.264678478240967, + "max_kl": 12.607242584228516, + "mean_kl": 4.320770740509033, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.380268096923828, + "code_tokens": 56, + "cvar95_kl": 14.081089973449707, + "delta_nll": 6.363282203674316, + "max_kl": 15.52476978302002, + "mean_kl": 6.392178058624268, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 9.350468635559082, + "code_tokens": 108, + "cvar95_kl": 15.68614673614502, + "delta_nll": 7.7966203689575195, + "max_kl": 19.81787872314453, + "mean_kl": 7.642419815063477, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.06481481343507767 + }, + { + "candidate_nll": 6.683440685272217, + "code_tokens": 47, + "cvar95_kl": 12.018020629882812, + "delta_nll": 5.311700820922852, + "max_kl": 14.819555282592773, + "mean_kl": 5.454356670379639, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.10638298094272614 + }, + { + "candidate_nll": 6.777721881866455, + "code_tokens": 102, + "cvar95_kl": 12.925331115722656, + "delta_nll": 5.779313087463379, + "max_kl": 14.8582124710083, + "mean_kl": 5.882768630981445, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.6222381591796875, + "code_tokens": 18, + "cvar95_kl": 8.563258171081543, + "delta_nll": 3.996612548828125, + "max_kl": 8.563258171081543, + "mean_kl": 4.282686233520508, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.352535724639893, + "code_tokens": 28, + "cvar95_kl": 11.131919860839844, + "delta_nll": 4.252871513366699, + "max_kl": 11.819252014160156, + "mean_kl": 4.554657459259033, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.785792827606201, + "code_tokens": 30, + "cvar95_kl": 10.68307113647461, + "delta_nll": 4.242392063140869, + "max_kl": 11.16035270690918, + "mean_kl": 4.597148418426514, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 7.001905918121338, + "code_tokens": 48, + "cvar95_kl": 16.630590438842773, + "delta_nll": 6.526788711547852, + "max_kl": 17.60735511779785, + "mean_kl": 6.484403133392334, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 6.574094772338867, + "code_tokens": 59, + "cvar95_kl": 16.067018508911133, + "delta_nll": 6.037079811096191, + "max_kl": 16.70628547668457, + "mean_kl": 5.963759422302246, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.3050847351551056 + }, + { + "candidate_nll": 8.758148193359375, + "code_tokens": 102, + "cvar95_kl": 18.15504264831543, + "delta_nll": 7.715954303741455, + "max_kl": 23.615097045898438, + "mean_kl": 8.0086030960083, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.10784313827753067 + }, + { + "candidate_nll": 6.343189239501953, + "code_tokens": 36, + "cvar95_kl": 12.17881965637207, + "delta_nll": 5.602823734283447, + "max_kl": 12.724900245666504, + "mean_kl": 5.29435396194458, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.587445259094238, + "code_tokens": 51, + "cvar95_kl": 15.536567687988281, + "delta_nll": 6.698513984680176, + "max_kl": 16.592670440673828, + "mean_kl": 6.412707805633545, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 6.45650577545166, + "code_tokens": 62, + "cvar95_kl": 13.660927772521973, + "delta_nll": 5.210826873779297, + "max_kl": 16.17994499206543, + "mean_kl": 5.591305732727051, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.17741934955120087 + }, + { + "candidate_nll": 6.522958278656006, + "code_tokens": 39, + "cvar95_kl": 17.150596618652344, + "delta_nll": 5.946676254272461, + "max_kl": 18.909086227416992, + "mean_kl": 6.048393726348877, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 7.506277561187744, + "code_tokens": 76, + "cvar95_kl": 17.544414520263672, + "delta_nll": 6.766592979431152, + "max_kl": 20.798768997192383, + "mean_kl": 7.041098117828369, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.07894736528396606 + }, + { + "candidate_nll": 8.952858924865723, + "code_tokens": 58, + "cvar95_kl": 19.655427932739258, + "delta_nll": 7.448207855224609, + "max_kl": 21.152908325195312, + "mean_kl": 7.441876411437988, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.06896551698446274 + }, + { + "candidate_nll": 7.839152812957764, + "code_tokens": 93, + "cvar95_kl": 14.963563919067383, + "delta_nll": 6.711757659912109, + "max_kl": 17.253890991210938, + "mean_kl": 6.711872100830078, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.11827956885099411 + }, + { + "candidate_nll": 7.177638530731201, + "code_tokens": 45, + "cvar95_kl": 13.79207992553711, + "delta_nll": 5.82239294052124, + "max_kl": 17.056055068969727, + "mean_kl": 5.561323165893555, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.15555556118488312 + }, + { + "candidate_nll": 8.92570972442627, + "code_tokens": 209, + "cvar95_kl": 18.784008026123047, + "delta_nll": 8.368734359741211, + "max_kl": 24.247800827026367, + "mean_kl": 8.550400733947754, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.0717703327536583 + }, + { + "candidate_nll": 7.3287353515625, + "code_tokens": 46, + "cvar95_kl": 15.27669620513916, + "delta_nll": 6.360267639160156, + "max_kl": 16.567081451416016, + "mean_kl": 6.006220817565918, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.239130437374115 + }, + { + "candidate_nll": 7.534998893737793, + "code_tokens": 39, + "cvar95_kl": 14.883451461791992, + "delta_nll": 6.814211845397949, + "max_kl": 16.834794998168945, + "mean_kl": 6.710022926330566, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.12820513546466827 + }, + { + "candidate_nll": 5.278916835784912, + "code_tokens": 25, + "cvar95_kl": 9.001657485961914, + "delta_nll": 4.337221622467041, + "max_kl": 9.570595741271973, + "mean_kl": 4.354117393493652, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 6.257866859436035, + "code_tokens": 28, + "cvar95_kl": 12.638752937316895, + "delta_nll": 5.08267068862915, + "max_kl": 14.40915584564209, + "mean_kl": 5.140161991119385, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.88982629776001, + "code_tokens": 84, + "cvar95_kl": 14.239367485046387, + "delta_nll": 5.635903835296631, + "max_kl": 15.838350296020508, + "mean_kl": 6.2950825691223145, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 5.741642475128174, + "code_tokens": 34, + "cvar95_kl": 12.333768844604492, + "delta_nll": 4.9293622970581055, + "max_kl": 12.622369766235352, + "mean_kl": 5.033898830413818, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 5.774861812591553, + "code_tokens": 28, + "cvar95_kl": 11.688264846801758, + "delta_nll": 4.519814491271973, + "max_kl": 11.924370765686035, + "mean_kl": 4.4343581199646, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 7.91856050491333, + "code_tokens": 92, + "cvar95_kl": 15.073140144348145, + "delta_nll": 6.8197150230407715, + "max_kl": 16.082775115966797, + "mean_kl": 6.9616289138793945, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.14130434393882751 + }, + { + "candidate_nll": 8.62783145904541, + "code_tokens": 185, + "cvar95_kl": 17.109342575073242, + "delta_nll": 7.897716999053955, + "max_kl": 18.842086791992188, + "mean_kl": 7.802414417266846, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.11891891807317734 + }, + { + "candidate_nll": 7.002346038818359, + "code_tokens": 106, + "cvar95_kl": 15.674643516540527, + "delta_nll": 6.649351596832275, + "max_kl": 19.364788055419922, + "mean_kl": 6.665431976318359, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.09433962404727936 + }, + { + "candidate_nll": 9.70639419555664, + "code_tokens": 64, + "cvar95_kl": 16.14527702331543, + "delta_nll": 8.163840293884277, + "max_kl": 18.367246627807617, + "mean_kl": 7.934805393218994, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.078125 + }, + { + "candidate_nll": 7.12065315246582, + "code_tokens": 74, + "cvar95_kl": 15.036201477050781, + "delta_nll": 6.00986909866333, + "max_kl": 16.38163948059082, + "mean_kl": 6.492608547210693, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.09459459781646729 + }, + { + "candidate_nll": 6.057010173797607, + "code_tokens": 27, + "cvar95_kl": 12.310911178588867, + "delta_nll": 5.069606304168701, + "max_kl": 12.559101104736328, + "mean_kl": 5.306956768035889, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 9.534830093383789, + "code_tokens": 72, + "cvar95_kl": 19.35692596435547, + "delta_nll": 8.910992622375488, + "max_kl": 20.71329689025879, + "mean_kl": 9.20770263671875, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 7.461167812347412, + "code_tokens": 62, + "cvar95_kl": 15.068127632141113, + "delta_nll": 6.527865409851074, + "max_kl": 17.384857177734375, + "mean_kl": 6.302343845367432, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.14516128599643707 + }, + { + "candidate_nll": 7.305637836456299, + "code_tokens": 42, + "cvar95_kl": 15.51807689666748, + "delta_nll": 6.607471466064453, + "max_kl": 15.813321113586426, + "mean_kl": 6.689149379730225, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 9.615011215209961, + "code_tokens": 238, + "cvar95_kl": 19.386554718017578, + "delta_nll": 8.727678298950195, + "max_kl": 24.87630844116211, + "mean_kl": 8.95933723449707, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.06722689419984818 + }, + { + "candidate_nll": 6.567033767700195, + "code_tokens": 68, + "cvar95_kl": 14.550935745239258, + "delta_nll": 5.652522087097168, + "max_kl": 14.842965126037598, + "mean_kl": 5.966420650482178, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.13235294818878174 + }, + { + "candidate_nll": 7.7570905685424805, + "code_tokens": 83, + "cvar95_kl": 17.343191146850586, + "delta_nll": 6.772940635681152, + "max_kl": 18.743072509765625, + "mean_kl": 6.965506076812744, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.1204819306731224 + }, + { + "candidate_nll": 8.665519714355469, + "code_tokens": 64, + "cvar95_kl": 15.612076759338379, + "delta_nll": 7.401752471923828, + "max_kl": 17.00750732421875, + "mean_kl": 7.660492420196533, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.15625 + }, + { + "candidate_nll": 6.75815486907959, + "code_tokens": 37, + "cvar95_kl": 15.101064682006836, + "delta_nll": 6.101355075836182, + "max_kl": 18.104393005371094, + "mean_kl": 6.055484294891357, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 6.623212814331055, + "code_tokens": 27, + "cvar95_kl": 11.195484161376953, + "delta_nll": 5.113285541534424, + "max_kl": 11.41458797454834, + "mean_kl": 5.245691776275635, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.319132328033447, + "code_tokens": 40, + "cvar95_kl": 19.030973434448242, + "delta_nll": 5.42741584777832, + "max_kl": 19.383922576904297, + "mean_kl": 5.609898090362549, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.686306953430176, + "code_tokens": 28, + "cvar95_kl": 11.852622985839844, + "delta_nll": 4.688534259796143, + "max_kl": 12.657559394836426, + "mean_kl": 4.701098442077637, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.461175441741943, + "code_tokens": 59, + "cvar95_kl": 12.625126838684082, + "delta_nll": 4.606545925140381, + "max_kl": 13.27686595916748, + "mean_kl": 4.9737114906311035, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.18644067645072937 + }, + { + "candidate_nll": 6.405722618103027, + "code_tokens": 57, + "cvar95_kl": 14.157058715820312, + "delta_nll": 5.83572244644165, + "max_kl": 15.247002601623535, + "mean_kl": 5.766705513000488, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.19298245012760162 + }, + { + "candidate_nll": 6.590610504150391, + "code_tokens": 25, + "cvar95_kl": 12.052542686462402, + "delta_nll": 5.20667028427124, + "max_kl": 14.729951858520508, + "mean_kl": 4.706333637237549, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 6.613687515258789, + "code_tokens": 48, + "cvar95_kl": 12.116596221923828, + "delta_nll": 5.866535186767578, + "max_kl": 13.321507453918457, + "mean_kl": 6.0590500831604, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 5.586003303527832, + "code_tokens": 35, + "cvar95_kl": 13.011125564575195, + "delta_nll": 4.724227428436279, + "max_kl": 14.087244987487793, + "mean_kl": 4.858110427856445, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 5.825017929077148, + "code_tokens": 30, + "cvar95_kl": 12.62651252746582, + "delta_nll": 5.1803107261657715, + "max_kl": 13.623579978942871, + "mean_kl": 4.955433368682861, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 6.8328328132629395, + "code_tokens": 49, + "cvar95_kl": 12.834601402282715, + "delta_nll": 5.7097554206848145, + "max_kl": 13.867623329162598, + "mean_kl": 5.9504523277282715, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.12244898080825806 + }, + { + "candidate_nll": 7.913901329040527, + "code_tokens": 98, + "cvar95_kl": 15.344587326049805, + "delta_nll": 6.897684574127197, + "max_kl": 19.85374641418457, + "mean_kl": 6.9821343421936035, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.09183673560619354 + }, + { + "candidate_nll": 5.249199867248535, + "code_tokens": 25, + "cvar95_kl": 9.601848602294922, + "delta_nll": 3.8683934211730957, + "max_kl": 10.703309059143066, + "mean_kl": 4.008198261260986, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.18779182434082, + "code_tokens": 27, + "cvar95_kl": 11.703886032104492, + "delta_nll": 4.410068511962891, + "max_kl": 11.99855899810791, + "mean_kl": 4.214145660400391, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 5.1193389892578125, + "code_tokens": 20, + "cvar95_kl": 10.457448959350586, + "delta_nll": 4.074254035949707, + "max_kl": 10.457448959350586, + "mean_kl": 4.8105010986328125, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 9.285944938659668, + "code_tokens": 253, + "cvar95_kl": 18.317493438720703, + "delta_nll": 8.59327507019043, + "max_kl": 20.34368324279785, + "mean_kl": 8.678815841674805, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.09486165642738342 + }, + { + "candidate_nll": 6.401477813720703, + "code_tokens": 55, + "cvar95_kl": 11.129254341125488, + "delta_nll": 5.572357654571533, + "max_kl": 11.763513565063477, + "mean_kl": 5.745357513427734, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.12727272510528564 + }, + { + "candidate_nll": 5.132615089416504, + "code_tokens": 31, + "cvar95_kl": 8.954893112182617, + "delta_nll": 3.926116704940796, + "max_kl": 9.8966646194458, + "mean_kl": 4.0538530349731445, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 5.915590286254883, + "code_tokens": 35, + "cvar95_kl": 15.19198226928711, + "delta_nll": 5.211344242095947, + "max_kl": 17.521167755126953, + "mean_kl": 5.112227439880371, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.2571428716182709 + }, + { + "candidate_nll": 5.690140724182129, + "code_tokens": 61, + "cvar95_kl": 13.977404594421387, + "delta_nll": 4.887272357940674, + "max_kl": 15.811525344848633, + "mean_kl": 5.003320217132568, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.2786885201931 + }, + { + "candidate_nll": 6.503567695617676, + "code_tokens": 55, + "cvar95_kl": 14.518689155578613, + "delta_nll": 5.548850059509277, + "max_kl": 15.887625694274902, + "mean_kl": 5.969390869140625, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.145454540848732 + }, + { + "candidate_nll": 7.528797626495361, + "code_tokens": 108, + "cvar95_kl": 15.355332374572754, + "delta_nll": 6.0218305587768555, + "max_kl": 16.735321044921875, + "mean_kl": 6.361968040466309, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.10185185074806213 + }, + { + "candidate_nll": 6.304082870483398, + "code_tokens": 58, + "cvar95_kl": 12.867156028747559, + "delta_nll": 5.016777038574219, + "max_kl": 13.43659782409668, + "mean_kl": 5.027947425842285, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 7.362357139587402, + "code_tokens": 91, + "cvar95_kl": 16.178142547607422, + "delta_nll": 6.534902095794678, + "max_kl": 18.964996337890625, + "mean_kl": 6.680121898651123, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.09890110045671463 + }, + { + "candidate_nll": 7.690730571746826, + "code_tokens": 105, + "cvar95_kl": 15.192550659179688, + "delta_nll": 6.926990985870361, + "max_kl": 17.34478759765625, + "mean_kl": 7.272013187408447, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 6.6122002601623535, + "code_tokens": 49, + "cvar95_kl": 12.58714771270752, + "delta_nll": 5.964947700500488, + "max_kl": 14.018486976623535, + "mean_kl": 5.735738277435303, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.20408163964748383 + }, + { + "candidate_nll": 6.437350749969482, + "code_tokens": 27, + "cvar95_kl": 14.585317611694336, + "delta_nll": 4.9751877784729, + "max_kl": 15.366471290588379, + "mean_kl": 5.451574325561523, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 5.833014011383057, + "code_tokens": 25, + "cvar95_kl": 12.894765853881836, + "delta_nll": 4.792637348175049, + "max_kl": 13.129022598266602, + "mean_kl": 5.049171447753906, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.502359390258789, + "code_tokens": 98, + "cvar95_kl": 15.1676025390625, + "delta_nll": 6.207304954528809, + "max_kl": 15.956512451171875, + "mean_kl": 6.574571132659912, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 4.793135643005371, + "code_tokens": 23, + "cvar95_kl": 10.407403945922852, + "delta_nll": 4.1433539390563965, + "max_kl": 10.665365219116211, + "mean_kl": 4.138439655303955, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 6.445960521697998, + "code_tokens": 31, + "cvar95_kl": 13.770552635192871, + "delta_nll": 4.580586910247803, + "max_kl": 14.413590431213379, + "mean_kl": 6.007326602935791, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 6.0803914070129395, + "code_tokens": 36, + "cvar95_kl": 15.296991348266602, + "delta_nll": 5.332298278808594, + "max_kl": 15.310137748718262, + "mean_kl": 5.357290744781494, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.941542625427246, + "code_tokens": 38, + "cvar95_kl": 14.618003845214844, + "delta_nll": 5.507864952087402, + "max_kl": 14.671835899353027, + "mean_kl": 5.761796474456787, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 4.908812999725342, + "code_tokens": 25, + "cvar95_kl": 9.396129608154297, + "delta_nll": 3.3027231693267822, + "max_kl": 10.063189506530762, + "mean_kl": 3.0861237049102783, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.933952808380127, + "code_tokens": 26, + "cvar95_kl": 11.644186019897461, + "delta_nll": 5.054503440856934, + "max_kl": 12.182103157043457, + "mean_kl": 5.134393692016602, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 7.461406230926514, + "code_tokens": 62, + "cvar95_kl": 15.78101921081543, + "delta_nll": 6.718964099884033, + "max_kl": 16.038639068603516, + "mean_kl": 6.7185258865356445, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.12903225421905518 + }, + { + "candidate_nll": 7.183714866638184, + "code_tokens": 36, + "cvar95_kl": 15.377983093261719, + "delta_nll": 5.44519567489624, + "max_kl": 16.061323165893555, + "mean_kl": 6.182030200958252, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 9.819849014282227, + "code_tokens": 119, + "cvar95_kl": 18.916423797607422, + "delta_nll": 8.489259719848633, + "max_kl": 23.443788528442383, + "mean_kl": 8.582754135131836, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.06722689419984818 + }, + { + "candidate_nll": 3.9261722564697266, + "code_tokens": 37, + "cvar95_kl": 8.674562454223633, + "delta_nll": 3.613464593887329, + "max_kl": 9.888249397277832, + "mean_kl": 3.336153745651245, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 4.85445499420166, + "code_tokens": 28, + "cvar95_kl": 10.229500770568848, + "delta_nll": 4.07172155380249, + "max_kl": 10.782140731811523, + "mean_kl": 4.363497257232666, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 10.073695182800293, + "code_tokens": 143, + "cvar95_kl": 19.155473709106445, + "delta_nll": 9.50734806060791, + "max_kl": 21.033519744873047, + "mean_kl": 9.696727752685547, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.0559440553188324 + }, + { + "candidate_nll": 6.4149322509765625, + "code_tokens": 29, + "cvar95_kl": 11.947306632995605, + "delta_nll": 4.632626056671143, + "max_kl": 12.424151420593262, + "mean_kl": 4.841989517211914, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 5.075470924377441, + "code_tokens": 28, + "cvar95_kl": 8.861678123474121, + "delta_nll": 4.0029826164245605, + "max_kl": 9.516868591308594, + "mean_kl": 4.020075798034668, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 7.42070198059082, + "code_tokens": 30, + "cvar95_kl": 13.422024726867676, + "delta_nll": 6.1566643714904785, + "max_kl": 14.445511817932129, + "mean_kl": 6.384533882141113, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.1547722816467285, + "code_tokens": 85, + "cvar95_kl": 17.64000129699707, + "delta_nll": 6.732962131500244, + "max_kl": 19.558971405029297, + "mean_kl": 6.708840847015381, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.1411764770746231 + }, + { + "candidate_nll": 7.366747856140137, + "code_tokens": 44, + "cvar95_kl": 13.365382194519043, + "delta_nll": 6.389504432678223, + "max_kl": 18.48972511291504, + "mean_kl": 6.1912150382995605, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.13636364042758942 + }, + { + "candidate_nll": 5.871638298034668, + "code_tokens": 19, + "cvar95_kl": 12.051054954528809, + "delta_nll": 5.067535877227783, + "max_kl": 12.051054954528809, + "mean_kl": 4.953860282897949, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 7.536548137664795, + "code_tokens": 113, + "cvar95_kl": 14.14492416381836, + "delta_nll": 6.7991251945495605, + "max_kl": 16.67455291748047, + "mean_kl": 6.659552574157715, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.13274335861206055 + }, + { + "candidate_nll": 6.5961785316467285, + "code_tokens": 40, + "cvar95_kl": 10.876379013061523, + "delta_nll": 5.120984077453613, + "max_kl": 11.421456336975098, + "mean_kl": 5.295270919799805, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 7.756985664367676, + "code_tokens": 51, + "cvar95_kl": 13.46002197265625, + "delta_nll": 6.150325775146484, + "max_kl": 14.301965713500977, + "mean_kl": 6.65886926651001, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.0784313753247261 + }, + { + "candidate_nll": 6.727102756500244, + "code_tokens": 55, + "cvar95_kl": 12.618531227111816, + "delta_nll": 5.14607048034668, + "max_kl": 13.464410781860352, + "mean_kl": 5.552209377288818, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.2181818187236786 + }, + { + "candidate_nll": 6.615050792694092, + "code_tokens": 27, + "cvar95_kl": 14.061850547790527, + "delta_nll": 5.484982490539551, + "max_kl": 14.600873947143555, + "mean_kl": 5.507162570953369, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.040569305419922, + "code_tokens": 32, + "cvar95_kl": 12.40976333618164, + "delta_nll": 5.20076322555542, + "max_kl": 13.1023588180542, + "mean_kl": 5.567838668823242, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 7.635012626647949, + "code_tokens": 64, + "cvar95_kl": 13.011015892028809, + "delta_nll": 6.982474327087402, + "max_kl": 13.179544448852539, + "mean_kl": 6.665050029754639, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.140625 + }, + { + "candidate_nll": 7.388963222503662, + "code_tokens": 35, + "cvar95_kl": 16.231088638305664, + "delta_nll": 6.339308738708496, + "max_kl": 17.409896850585938, + "mean_kl": 6.2020392417907715, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 8.091447830200195, + "code_tokens": 152, + "cvar95_kl": 16.839876174926758, + "delta_nll": 7.6396002769470215, + "max_kl": 18.486286163330078, + "mean_kl": 7.4408440589904785, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.087840557098389, + "code_tokens": 77, + "cvar95_kl": 14.049373626708984, + "delta_nll": 5.8265061378479, + "max_kl": 16.891666412353516, + "mean_kl": 6.026370048522949, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.19480518996715546 + }, + { + "candidate_nll": 7.5876593589782715, + "code_tokens": 77, + "cvar95_kl": 15.612993240356445, + "delta_nll": 7.019378185272217, + "max_kl": 17.608076095581055, + "mean_kl": 6.9508843421936035, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 5.69223690032959, + "code_tokens": 71, + "cvar95_kl": 15.785207748413086, + "delta_nll": 5.130147933959961, + "max_kl": 17.981666564941406, + "mean_kl": 5.170380115509033, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.22535210847854614 + }, + { + "candidate_nll": 8.952704429626465, + "code_tokens": 112, + "cvar95_kl": 17.304824829101562, + "delta_nll": 8.074325561523438, + "max_kl": 19.0963077545166, + "mean_kl": 7.976065158843994, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 8.42145824432373, + "code_tokens": 111, + "cvar95_kl": 16.82485008239746, + "delta_nll": 7.701554298400879, + "max_kl": 19.610469818115234, + "mean_kl": 7.957353591918945, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.09009008854627609 + }, + { + "candidate_nll": 5.466435432434082, + "code_tokens": 62, + "cvar95_kl": 11.850017547607422, + "delta_nll": 4.678539276123047, + "max_kl": 13.239222526550293, + "mean_kl": 4.8919219970703125, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.24193547666072845 + }, + { + "candidate_nll": 5.754313945770264, + "code_tokens": 52, + "cvar95_kl": 14.410789489746094, + "delta_nll": 5.143733978271484, + "max_kl": 15.66426944732666, + "mean_kl": 5.140294075012207, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.787608623504639, + "code_tokens": 24, + "cvar95_kl": 9.888026237487793, + "delta_nll": 3.8923821449279785, + "max_kl": 10.372011184692383, + "mean_kl": 4.461191177368164, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.227962493896484, + "code_tokens": 48, + "cvar95_kl": 13.28017520904541, + "delta_nll": 5.336192607879639, + "max_kl": 14.37435531616211, + "mean_kl": 5.639005184173584, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 8.254581451416016, + "code_tokens": 98, + "cvar95_kl": 19.022464752197266, + "delta_nll": 7.204288482666016, + "max_kl": 22.15456771850586, + "mean_kl": 7.365209579467773, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.12244898080825806 + }, + { + "candidate_nll": 7.500246524810791, + "code_tokens": 48, + "cvar95_kl": 12.867466926574707, + "delta_nll": 6.507560729980469, + "max_kl": 13.058128356933594, + "mean_kl": 6.417051792144775, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 5.90140962600708, + "code_tokens": 56, + "cvar95_kl": 14.278271675109863, + "delta_nll": 4.955414295196533, + "max_kl": 18.07775115966797, + "mean_kl": 5.1698760986328125, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.1964285671710968 + }, + { + "candidate_nll": 7.23591947555542, + "code_tokens": 88, + "cvar95_kl": 14.820775985717773, + "delta_nll": 6.634642601013184, + "max_kl": 15.36046028137207, + "mean_kl": 6.545860290527344, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.17045454680919647 + }, + { + "candidate_nll": 8.273067474365234, + "code_tokens": 50, + "cvar95_kl": 15.802258491516113, + "delta_nll": 6.639537334442139, + "max_kl": 16.34769630432129, + "mean_kl": 6.802567005157471, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 6.517818450927734, + "code_tokens": 85, + "cvar95_kl": 15.090581893920898, + "delta_nll": 6.080831527709961, + "max_kl": 16.65238380432129, + "mean_kl": 6.229759216308594, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.0941176488995552 + }, + { + "candidate_nll": 5.860525608062744, + "code_tokens": 27, + "cvar95_kl": 9.318595886230469, + "delta_nll": 4.786312580108643, + "max_kl": 9.406341552734375, + "mean_kl": 4.915737628936768, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 6.545654296875, + "code_tokens": 48, + "cvar95_kl": 15.140477180480957, + "delta_nll": 5.428035736083984, + "max_kl": 16.508012771606445, + "mean_kl": 6.11161470413208, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 7.892729759216309, + "code_tokens": 29, + "cvar95_kl": 15.543217658996582, + "delta_nll": 6.244882583618164, + "max_kl": 17.325485229492188, + "mean_kl": 6.507449150085449, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 7.32188081741333, + "code_tokens": 63, + "cvar95_kl": 14.76348876953125, + "delta_nll": 6.383665561676025, + "max_kl": 17.104995727539062, + "mean_kl": 6.901912689208984, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.1269841343164444 + }, + { + "candidate_nll": 7.499311923980713, + "code_tokens": 89, + "cvar95_kl": 16.880441665649414, + "delta_nll": 6.824253082275391, + "max_kl": 18.31840705871582, + "mean_kl": 7.053842544555664, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.17977528274059296 + }, + { + "candidate_nll": 9.231328010559082, + "code_tokens": 133, + "cvar95_kl": 17.38821029663086, + "delta_nll": 7.887049674987793, + "max_kl": 21.18796730041504, + "mean_kl": 8.066020965576172, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.10526315867900848 + }, + { + "candidate_nll": 5.3140482902526855, + "code_tokens": 25, + "cvar95_kl": 11.619026184082031, + "delta_nll": 4.365511417388916, + "max_kl": 11.97139835357666, + "mean_kl": 4.294548988342285, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 6.391116142272949, + "code_tokens": 33, + "cvar95_kl": 13.320865631103516, + "delta_nll": 5.353816032409668, + "max_kl": 14.365342140197754, + "mean_kl": 5.570664882659912, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 6.627377510070801, + "code_tokens": 93, + "cvar95_kl": 16.12345314025879, + "delta_nll": 5.870602607727051, + "max_kl": 17.953266143798828, + "mean_kl": 6.034734725952148, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.1827957034111023 + }, + { + "candidate_nll": 8.179693222045898, + "code_tokens": 101, + "cvar95_kl": 16.198509216308594, + "delta_nll": 7.211293697357178, + "max_kl": 19.57101821899414, + "mean_kl": 7.211629867553711, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.09900990128517151 + }, + { + "candidate_nll": 7.446458339691162, + "code_tokens": 152, + "cvar95_kl": 15.213964462280273, + "delta_nll": 6.88169527053833, + "max_kl": 18.253053665161133, + "mean_kl": 6.9112725257873535, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.1189985275268555, + "code_tokens": 33, + "cvar95_kl": 13.898028373718262, + "delta_nll": 5.718127250671387, + "max_kl": 14.907177925109863, + "mean_kl": 6.129914283752441, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.1515151560306549 + }, + { + "candidate_nll": 6.846905708312988, + "code_tokens": 29, + "cvar95_kl": 15.502946853637695, + "delta_nll": 5.560442924499512, + "max_kl": 17.28907012939453, + "mean_kl": 5.882711887359619, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 6.970116138458252, + "code_tokens": 27, + "cvar95_kl": 16.372779846191406, + "delta_nll": 5.5120463371276855, + "max_kl": 19.4992618560791, + "mean_kl": 6.034868240356445, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 5.47191858291626, + "code_tokens": 33, + "cvar95_kl": 13.152055740356445, + "delta_nll": 5.16187858581543, + "max_kl": 13.301405906677246, + "mean_kl": 5.055593490600586, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 5.514010906219482, + "code_tokens": 27, + "cvar95_kl": 14.367959022521973, + "delta_nll": 4.7176127433776855, + "max_kl": 16.60899543762207, + "mean_kl": 5.0425238609313965, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.145252704620361, + "code_tokens": 84, + "cvar95_kl": 15.319562911987305, + "delta_nll": 6.441359043121338, + "max_kl": 16.092954635620117, + "mean_kl": 6.543799877166748, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 7.265289783477783, + "code_tokens": 103, + "cvar95_kl": 14.1566801071167, + "delta_nll": 6.219820022583008, + "max_kl": 15.796046257019043, + "mean_kl": 6.197361469268799, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.1650485396385193 + }, + { + "candidate_nll": 7.677601337432861, + "code_tokens": 88, + "cvar95_kl": 16.139453887939453, + "delta_nll": 6.761253833770752, + "max_kl": 16.830949783325195, + "mean_kl": 6.588618755340576, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.23863635957241058 + }, + { + "candidate_nll": 8.084244728088379, + "code_tokens": 62, + "cvar95_kl": 14.630484580993652, + "delta_nll": 6.775313854217529, + "max_kl": 18.651079177856445, + "mean_kl": 6.610351085662842, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 6.443145275115967, + "code_tokens": 24, + "cvar95_kl": 12.257050514221191, + "delta_nll": 4.906436443328857, + "max_kl": 12.723618507385254, + "mean_kl": 5.0205979347229, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.082009315490723, + "code_tokens": 34, + "cvar95_kl": 17.798452377319336, + "delta_nll": 6.743556976318359, + "max_kl": 19.620832443237305, + "mean_kl": 6.110645294189453, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 6.534377098083496, + "code_tokens": 43, + "cvar95_kl": 15.334261894226074, + "delta_nll": 5.814631462097168, + "max_kl": 15.997147560119629, + "mean_kl": 5.531414031982422, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.23255814611911774 + }, + { + "candidate_nll": 8.24865436553955, + "code_tokens": 54, + "cvar95_kl": 15.002738952636719, + "delta_nll": 6.7593488693237305, + "max_kl": 15.685047149658203, + "mean_kl": 6.90505313873291, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.461955547332764, + "code_tokens": 50, + "cvar95_kl": 14.434246063232422, + "delta_nll": 5.732069969177246, + "max_kl": 16.21602439880371, + "mean_kl": 5.635706901550293, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.2199999988079071 + }, + { + "candidate_nll": 8.303750038146973, + "code_tokens": 235, + "cvar95_kl": 16.713088989257812, + "delta_nll": 7.546239376068115, + "max_kl": 18.338376998901367, + "mean_kl": 7.807841777801514, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.08085106313228607 + }, + { + "candidate_nll": 5.914539813995361, + "code_tokens": 33, + "cvar95_kl": 11.824909210205078, + "delta_nll": 3.8858847618103027, + "max_kl": 14.381505012512207, + "mean_kl": 4.723475456237793, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 7.199466228485107, + "code_tokens": 45, + "cvar95_kl": 14.636801719665527, + "delta_nll": 6.212964057922363, + "max_kl": 18.842662811279297, + "mean_kl": 6.245413780212402, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.741953372955322, + "code_tokens": 28, + "cvar95_kl": 13.334175109863281, + "delta_nll": 6.118993282318115, + "max_kl": 14.028539657592773, + "mean_kl": 5.849698066711426, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 6.0031561851501465, + "code_tokens": 49, + "cvar95_kl": 12.906373023986816, + "delta_nll": 5.324892044067383, + "max_kl": 14.15864086151123, + "mean_kl": 5.53861141204834, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.16326530277729034 + }, + { + "candidate_nll": 8.37089729309082, + "code_tokens": 107, + "cvar95_kl": 18.711265563964844, + "delta_nll": 8.121481895446777, + "max_kl": 21.75229263305664, + "mean_kl": 7.973795413970947, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.08411215245723724 + }, + { + "candidate_nll": 7.396381378173828, + "code_tokens": 48, + "cvar95_kl": 15.519913673400879, + "delta_nll": 6.175471305847168, + "max_kl": 15.985645294189453, + "mean_kl": 6.529460906982422, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.787487983703613, + "code_tokens": 40, + "cvar95_kl": 12.641828536987305, + "delta_nll": 4.947947025299072, + "max_kl": 12.91968059539795, + "mean_kl": 4.933192253112793, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.4923553466796875, + "code_tokens": 46, + "cvar95_kl": 12.791705131530762, + "delta_nll": 6.022871494293213, + "max_kl": 15.166008949279785, + "mean_kl": 6.009113311767578, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.08695652335882187 + }, + { + "candidate_nll": 7.386579990386963, + "code_tokens": 31, + "cvar95_kl": 12.489130973815918, + "delta_nll": 6.344130516052246, + "max_kl": 12.655963897705078, + "mean_kl": 5.570866107940674, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.29032257199287415 + }, + { + "candidate_nll": 6.759082317352295, + "code_tokens": 58, + "cvar95_kl": 15.403461456298828, + "delta_nll": 6.342597961425781, + "max_kl": 16.52808952331543, + "mean_kl": 6.069666385650635, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.320929050445557, + "code_tokens": 49, + "cvar95_kl": 15.664395332336426, + "delta_nll": 6.203159332275391, + "max_kl": 18.762025833129883, + "mean_kl": 6.775157928466797, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.10204081982374191 + }, + { + "candidate_nll": 3.232093334197998, + "code_tokens": 18, + "cvar95_kl": 14.379911422729492, + "delta_nll": 2.2854998111724854, + "max_kl": 14.379911422729492, + "mean_kl": 3.154895782470703, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 5.073698997497559, + "code_tokens": 28, + "cvar95_kl": 11.90218734741211, + "delta_nll": 3.914287567138672, + "max_kl": 13.154596328735352, + "mean_kl": 4.482165336608887, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.245492935180664, + "code_tokens": 57, + "cvar95_kl": 12.89111042022705, + "delta_nll": 5.5034260749816895, + "max_kl": 16.255178451538086, + "mean_kl": 5.587789535522461, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 6.817921161651611, + "code_tokens": 88, + "cvar95_kl": 14.746416091918945, + "delta_nll": 6.11781120300293, + "max_kl": 16.41497039794922, + "mean_kl": 6.060349941253662, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.14772726595401764 + }, + { + "candidate_nll": 5.407069683074951, + "code_tokens": 23, + "cvar95_kl": 11.483325958251953, + "delta_nll": 4.208282947540283, + "max_kl": 12.135886192321777, + "mean_kl": 4.716247081756592, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 7.6528167724609375, + "code_tokens": 40, + "cvar95_kl": 16.883859634399414, + "delta_nll": 6.393359184265137, + "max_kl": 18.0887451171875, + "mean_kl": 6.76287841796875, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.923940181732178, + "code_tokens": 20, + "cvar95_kl": 9.243882179260254, + "delta_nll": 3.7627251148223877, + "max_kl": 9.243882179260254, + "mean_kl": 4.033858776092529, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 5.75247859954834, + "code_tokens": 63, + "cvar95_kl": 11.926373481750488, + "delta_nll": 4.736692905426025, + "max_kl": 12.885865211486816, + "mean_kl": 4.818276882171631, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.2063492089509964 + }, + { + "candidate_nll": 5.258952617645264, + "code_tokens": 39, + "cvar95_kl": 10.66903305053711, + "delta_nll": 4.306271553039551, + "max_kl": 10.957941055297852, + "mean_kl": 4.206771373748779, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 7.157126426696777, + "code_tokens": 50, + "cvar95_kl": 14.397866249084473, + "delta_nll": 5.7452616691589355, + "max_kl": 16.995994567871094, + "mean_kl": 5.919834136962891, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 7.629418849945068, + "code_tokens": 63, + "cvar95_kl": 14.569668769836426, + "delta_nll": 6.6619415283203125, + "max_kl": 15.140783309936523, + "mean_kl": 6.862249374389648, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 6.595024585723877, + "code_tokens": 46, + "cvar95_kl": 13.6791410446167, + "delta_nll": 5.622462749481201, + "max_kl": 14.106607437133789, + "mean_kl": 5.625805854797363, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.239130437374115 + }, + { + "candidate_nll": 5.193181991577148, + "code_tokens": 25, + "cvar95_kl": 10.163742065429688, + "delta_nll": 4.51428747177124, + "max_kl": 11.125225067138672, + "mean_kl": 4.593459606170654, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.109882354736328, + "code_tokens": 111, + "cvar95_kl": 15.676238059997559, + "delta_nll": 6.295616626739502, + "max_kl": 16.351335525512695, + "mean_kl": 6.329699993133545, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.12612612545490265 + }, + { + "candidate_nll": 7.638617515563965, + "code_tokens": 71, + "cvar95_kl": 17.203418731689453, + "delta_nll": 6.910059928894043, + "max_kl": 21.755584716796875, + "mean_kl": 6.950325012207031, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.0845070406794548 + }, + { + "candidate_nll": 5.30342435836792, + "code_tokens": 28, + "cvar95_kl": 11.06049919128418, + "delta_nll": 3.9327006340026855, + "max_kl": 13.407916069030762, + "mean_kl": 4.159602165222168, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.288679122924805, + "code_tokens": 25, + "cvar95_kl": 12.721880912780762, + "delta_nll": 5.088223457336426, + "max_kl": 13.641603469848633, + "mean_kl": 5.226037502288818, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 7.778939723968506, + "code_tokens": 40, + "cvar95_kl": 14.702555656433105, + "delta_nll": 6.476167678833008, + "max_kl": 14.861016273498535, + "mean_kl": 6.7368974685668945, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 8.119486808776855, + "code_tokens": 177, + "cvar95_kl": 18.2999210357666, + "delta_nll": 7.075042724609375, + "max_kl": 19.510295867919922, + "mean_kl": 7.475830078125, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.10169491171836853 + }, + { + "candidate_nll": 7.888127326965332, + "code_tokens": 51, + "cvar95_kl": 16.540817260742188, + "delta_nll": 6.7969489097595215, + "max_kl": 18.894746780395508, + "mean_kl": 7.058737754821777, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.13725490868091583 + }, + { + "candidate_nll": 7.799062728881836, + "code_tokens": 69, + "cvar95_kl": 13.987031936645508, + "delta_nll": 7.344602108001709, + "max_kl": 14.264241218566895, + "mean_kl": 7.235646724700928, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.08695652335882187 + }, + { + "candidate_nll": 5.743449687957764, + "code_tokens": 26, + "cvar95_kl": 16.10639190673828, + "delta_nll": 4.995189666748047, + "max_kl": 18.000364303588867, + "mean_kl": 5.134572505950928, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 5.761554718017578, + "code_tokens": 24, + "cvar95_kl": 9.834070205688477, + "delta_nll": 4.243215560913086, + "max_kl": 9.874164581298828, + "mean_kl": 4.755529403686523, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.063486099243164, + "code_tokens": 15, + "cvar95_kl": 11.676645278930664, + "delta_nll": 4.913010120391846, + "max_kl": 11.676645278930664, + "mean_kl": 4.971171855926514, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.012362957000732, + "code_tokens": 43, + "cvar95_kl": 14.487759590148926, + "delta_nll": 5.863014221191406, + "max_kl": 15.91156005859375, + "mean_kl": 6.201606750488281, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 4.9097185134887695, + "code_tokens": 18, + "cvar95_kl": 11.513174057006836, + "delta_nll": 3.8173928260803223, + "max_kl": 11.513174057006836, + "mean_kl": 4.4886908531188965, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.673968315124512, + "code_tokens": 49, + "cvar95_kl": 16.82981300354004, + "delta_nll": 6.945974826812744, + "max_kl": 18.047313690185547, + "mean_kl": 6.624238967895508, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.16326530277729034 + }, + { + "candidate_nll": 7.276628017425537, + "code_tokens": 59, + "cvar95_kl": 14.715937614440918, + "delta_nll": 6.408872127532959, + "max_kl": 16.21517562866211, + "mean_kl": 6.391209602355957, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.1355932205915451 + }, + { + "candidate_nll": 8.991841316223145, + "code_tokens": 221, + "cvar95_kl": 18.530044555664062, + "delta_nll": 8.406575202941895, + "max_kl": 21.95897102355957, + "mean_kl": 8.552567481994629, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.07692307978868484 + }, + { + "candidate_nll": 6.916408061981201, + "code_tokens": 25, + "cvar95_kl": 16.195913314819336, + "delta_nll": 5.714534759521484, + "max_kl": 17.553709030151367, + "mean_kl": 6.192701816558838, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 6.633866310119629, + "code_tokens": 36, + "cvar95_kl": 12.680106163024902, + "delta_nll": 5.909945487976074, + "max_kl": 13.129993438720703, + "mean_kl": 6.319777488708496, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 9.531496047973633, + "code_tokens": 158, + "cvar95_kl": 19.727760314941406, + "delta_nll": 8.887351036071777, + "max_kl": 24.45259666442871, + "mean_kl": 8.925276756286621, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.08227848261594772 + }, + { + "candidate_nll": 9.315840721130371, + "code_tokens": 92, + "cvar95_kl": 18.23031997680664, + "delta_nll": 8.00030517578125, + "max_kl": 18.876235961914062, + "mean_kl": 7.811603546142578, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.14130434393882751 + }, + { + "candidate_nll": 8.82265567779541, + "code_tokens": 102, + "cvar95_kl": 17.290529251098633, + "delta_nll": 7.908613204956055, + "max_kl": 20.62548065185547, + "mean_kl": 7.950940132141113, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.0882352963089943 + }, + { + "candidate_nll": 9.069695472717285, + "code_tokens": 171, + "cvar95_kl": 18.297332763671875, + "delta_nll": 8.459725379943848, + "max_kl": 22.073253631591797, + "mean_kl": 8.541522979736328, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.05847953259944916 + }, + { + "candidate_nll": 6.1485395431518555, + "code_tokens": 40, + "cvar95_kl": 12.62686538696289, + "delta_nll": 4.943675518035889, + "max_kl": 12.702066421508789, + "mean_kl": 5.54106330871582, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 8.188200950622559, + "code_tokens": 48, + "cvar95_kl": 17.820632934570312, + "delta_nll": 6.9521870613098145, + "max_kl": 22.379179000854492, + "mean_kl": 7.138439178466797, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.744607925415039, + "code_tokens": 24, + "cvar95_kl": 11.187292098999023, + "delta_nll": 5.139687538146973, + "max_kl": 11.546053886413574, + "mean_kl": 5.37273645401001, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.4897894859313965, + "code_tokens": 48, + "cvar95_kl": 12.582972526550293, + "delta_nll": 6.502685546875, + "max_kl": 13.531739234924316, + "mean_kl": 6.48695182800293, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 6.705573558807373, + "code_tokens": 25, + "cvar95_kl": 12.428618431091309, + "delta_nll": 5.318690299987793, + "max_kl": 13.255191802978516, + "mean_kl": 5.082554817199707, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 6.045449733734131, + "code_tokens": 30, + "cvar95_kl": 12.071916580200195, + "delta_nll": 4.5467047691345215, + "max_kl": 14.20003604888916, + "mean_kl": 4.228212833404541, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 5.573200702667236, + "code_tokens": 19, + "cvar95_kl": 11.921090126037598, + "delta_nll": 4.146382808685303, + "max_kl": 11.921090126037598, + "mean_kl": 4.167760848999023, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 4.469455718994141, + "code_tokens": 12, + "cvar95_kl": 9.683591842651367, + "delta_nll": 3.390383243560791, + "max_kl": 9.683591842651367, + "mean_kl": 3.7022716999053955, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.877119541168213, + "code_tokens": 37, + "cvar95_kl": 14.53328800201416, + "delta_nll": 5.536809921264648, + "max_kl": 15.737703323364258, + "mean_kl": 6.104064464569092, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 4.401162624359131, + "code_tokens": 33, + "cvar95_kl": 12.681415557861328, + "delta_nll": 3.195732831954956, + "max_kl": 14.064916610717773, + "mean_kl": 3.8941752910614014, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 7.439024448394775, + "code_tokens": 94, + "cvar95_kl": 18.908557891845703, + "delta_nll": 6.586914539337158, + "max_kl": 22.752878189086914, + "mean_kl": 6.877234935760498, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.1808510571718216 + }, + { + "candidate_nll": 7.913916110992432, + "code_tokens": 90, + "cvar95_kl": 16.04769515991211, + "delta_nll": 7.289151191711426, + "max_kl": 17.112163543701172, + "mean_kl": 6.959145545959473, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.15555556118488312 + }, + { + "candidate_nll": 9.372567176818848, + "code_tokens": 47, + "cvar95_kl": 19.3505916595459, + "delta_nll": 7.924803256988525, + "max_kl": 24.485092163085938, + "mean_kl": 7.802027225494385, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.12765957415103912 + }, + { + "candidate_nll": 7.837671279907227, + "code_tokens": 61, + "cvar95_kl": 14.860335350036621, + "delta_nll": 6.91383695602417, + "max_kl": 18.29848289489746, + "mean_kl": 7.058842658996582, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.09836065769195557 + }, + { + "candidate_nll": 7.278967380523682, + "code_tokens": 39, + "cvar95_kl": 14.741548538208008, + "delta_nll": 6.055452346801758, + "max_kl": 16.738733291625977, + "mean_kl": 6.528321266174316, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 8.307541847229004, + "code_tokens": 152, + "cvar95_kl": 17.26683235168457, + "delta_nll": 7.375380039215088, + "max_kl": 19.289033889770508, + "mean_kl": 7.9517292976379395, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.05921052768826485 + }, + { + "candidate_nll": 8.467917442321777, + "code_tokens": 100, + "cvar95_kl": 17.342504501342773, + "delta_nll": 7.498935222625732, + "max_kl": 20.089366912841797, + "mean_kl": 7.636419296264648, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.10999999940395355 + }, + { + "candidate_nll": 9.029674530029297, + "code_tokens": 89, + "cvar95_kl": 20.83284568786621, + "delta_nll": 8.408746719360352, + "max_kl": 23.827774047851562, + "mean_kl": 8.358925819396973, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.10112359374761581 + }, + { + "candidate_nll": 6.702725410461426, + "code_tokens": 40, + "cvar95_kl": 16.079551696777344, + "delta_nll": 5.391724109649658, + "max_kl": 18.246734619140625, + "mean_kl": 5.517543792724609, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 6.910059928894043, + "code_tokens": 76, + "cvar95_kl": 16.0003719329834, + "delta_nll": 6.208492755889893, + "max_kl": 17.821443557739258, + "mean_kl": 6.197665691375732, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 6.734744548797607, + "code_tokens": 56, + "cvar95_kl": 15.434666633605957, + "delta_nll": 5.485757350921631, + "max_kl": 19.012434005737305, + "mean_kl": 6.047554969787598, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.210289001464844, + "code_tokens": 34, + "cvar95_kl": 11.561746597290039, + "delta_nll": 4.720858573913574, + "max_kl": 11.742725372314453, + "mean_kl": 4.934038162231445, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 8.498247146606445, + "code_tokens": 96, + "cvar95_kl": 18.00456428527832, + "delta_nll": 7.820135593414307, + "max_kl": 20.192535400390625, + "mean_kl": 7.579460620880127, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.1145833358168602 + }, + { + "candidate_nll": 8.167725563049316, + "code_tokens": 451, + "cvar95_kl": 16.581106185913086, + "delta_nll": 7.469030380249023, + "max_kl": 20.32306671142578, + "mean_kl": 7.388213157653809, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.06430155038833618 + }, + { + "candidate_nll": 6.979404926300049, + "code_tokens": 37, + "cvar95_kl": 15.302664756774902, + "delta_nll": 5.367094993591309, + "max_kl": 15.39319896697998, + "mean_kl": 5.973320007324219, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 6.981492042541504, + "code_tokens": 42, + "cvar95_kl": 14.88740062713623, + "delta_nll": 5.739893913269043, + "max_kl": 16.3580379486084, + "mean_kl": 6.734771728515625, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 7.320530414581299, + "code_tokens": 33, + "cvar95_kl": 15.07326889038086, + "delta_nll": 6.303774356842041, + "max_kl": 15.626194953918457, + "mean_kl": 5.996128559112549, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.189610481262207, + "code_tokens": 43, + "cvar95_kl": 11.992717742919922, + "delta_nll": 5.263558864593506, + "max_kl": 13.065299987792969, + "mean_kl": 5.476806163787842, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.20930232107639313 + }, + { + "candidate_nll": 6.4565839767456055, + "code_tokens": 73, + "cvar95_kl": 16.56178092956543, + "delta_nll": 5.720147609710693, + "max_kl": 20.89815902709961, + "mean_kl": 5.75248384475708, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.16438356041908264 + }, + { + "candidate_nll": 6.314355373382568, + "code_tokens": 15, + "cvar95_kl": 13.249364852905273, + "delta_nll": 5.40187931060791, + "max_kl": 13.249364852905273, + "mean_kl": 4.947488307952881, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.949474334716797, + "code_tokens": 34, + "cvar95_kl": 17.501005172729492, + "delta_nll": 5.690394401550293, + "max_kl": 18.235502243041992, + "mean_kl": 6.1915812492370605, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 8.013801574707031, + "code_tokens": 136, + "cvar95_kl": 15.473977088928223, + "delta_nll": 6.905318260192871, + "max_kl": 18.229930877685547, + "mean_kl": 6.821324825286865, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.11029411852359772 + }, + { + "candidate_nll": 5.503498554229736, + "code_tokens": 17, + "cvar95_kl": 11.023889541625977, + "delta_nll": 4.257342338562012, + "max_kl": 11.023889541625977, + "mean_kl": 4.5598602294921875, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.4372992515563965, + "code_tokens": 33, + "cvar95_kl": 13.765380859375, + "delta_nll": 5.64424467086792, + "max_kl": 14.270291328430176, + "mean_kl": 5.7638936042785645, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 6.095071315765381, + "code_tokens": 38, + "cvar95_kl": 13.591787338256836, + "delta_nll": 5.25262451171875, + "max_kl": 14.332977294921875, + "mean_kl": 5.0865397453308105, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 6.392592430114746, + "code_tokens": 61, + "cvar95_kl": 11.867210388183594, + "delta_nll": 5.995067596435547, + "max_kl": 12.751730918884277, + "mean_kl": 6.2026848793029785, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.1147540956735611 + }, + { + "candidate_nll": 7.002776622772217, + "code_tokens": 143, + "cvar95_kl": 13.100314140319824, + "delta_nll": 6.267056941986084, + "max_kl": 15.566120147705078, + "mean_kl": 6.319276809692383, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.139860138297081 + }, + { + "candidate_nll": 7.212360858917236, + "code_tokens": 41, + "cvar95_kl": 14.955130577087402, + "delta_nll": 6.123294830322266, + "max_kl": 16.862258911132812, + "mean_kl": 6.552040100097656, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.1463414579629898 + }, + { + "candidate_nll": 7.193619728088379, + "code_tokens": 67, + "cvar95_kl": 17.356595993041992, + "delta_nll": 6.828938007354736, + "max_kl": 20.344186782836914, + "mean_kl": 6.948001861572266, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.10447761416435242 + }, + { + "candidate_nll": 7.548043727874756, + "code_tokens": 76, + "cvar95_kl": 12.892610549926758, + "delta_nll": 6.157783508300781, + "max_kl": 14.684167861938477, + "mean_kl": 6.117018699645996, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.14473684132099152 + }, + { + "candidate_nll": 8.200398445129395, + "code_tokens": 137, + "cvar95_kl": 16.5550594329834, + "delta_nll": 7.564545631408691, + "max_kl": 17.82728385925293, + "mean_kl": 7.498002529144287, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.11678832024335861 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2339", + "rounding": "stochastic", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 9.758928298950195, + "median": 5.743535280227661, + "minimum": 1.5858736038208008, + "q25": 4.946879148483276, + "q75": 6.712530732154846 + }, + "task_macro": { + "candidate_nll": 6.841310597896576, + "delta_nll": 5.827264015436173, + "mean_kl": 5.956130279302597, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.1837331698164344 + }, + "token_weighted": { + "candidate_nll": 7.459651947021484, + "cvar95_kl": 16.67877960205078, + "delta_nll": 6.542814254760742, + "max_kl": 26.2041015625, + "mean_kl": 6.644874572753906, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.14796322584152222 + } + }, + "read_risk_l0_stochastic_seed_2340": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.529829532623291, + "macro_mean_kl": 4.654094039916992, + "macro_top1_agreement": 0.2604804055094719, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 5.607088274002075, + "macro_mean_kl": 5.778263929367065, + "macro_top1_agreement": 0.18080501413345337, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 6.1582082138061525, + "macro_mean_kl": 6.257300514221192, + "macro_top1_agreement": 0.16003973731398582, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 7.426259628295899, + "macro_mean_kl": 7.473003723144531, + "macro_top1_agreement": 0.11327533903717994, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 7.945725917816162, + "code_tokens": 98, + "cvar95_kl": 15.896183967590332, + "delta_nll": 7.218870162963867, + "max_kl": 18.407466888427734, + "mean_kl": 7.410557746887207, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.13265305757522583 + }, + { + "candidate_nll": 7.086247444152832, + "code_tokens": 19, + "cvar95_kl": 16.164852142333984, + "delta_nll": 5.9268341064453125, + "max_kl": 16.164852142333984, + "mean_kl": 6.096107006072998, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 7.59185791015625, + "code_tokens": 39, + "cvar95_kl": 14.698465347290039, + "delta_nll": 6.576508522033691, + "max_kl": 15.135161399841309, + "mean_kl": 6.460233688354492, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 5.81123161315918, + "code_tokens": 24, + "cvar95_kl": 11.742298126220703, + "delta_nll": 4.3649420738220215, + "max_kl": 13.993202209472656, + "mean_kl": 4.761748313903809, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.303531646728516, + "code_tokens": 25, + "cvar95_kl": 16.216670989990234, + "delta_nll": 5.895773410797119, + "max_kl": 16.61733055114746, + "mean_kl": 5.656062602996826, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.201058387756348, + "code_tokens": 55, + "cvar95_kl": 17.633180618286133, + "delta_nll": 7.305446624755859, + "max_kl": 24.455251693725586, + "mean_kl": 7.210232257843018, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.1090909093618393 + }, + { + "candidate_nll": 5.146166801452637, + "code_tokens": 16, + "cvar95_kl": 13.53948974609375, + "delta_nll": 4.316726207733154, + "max_kl": 13.53948974609375, + "mean_kl": 4.29443359375, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 11.536279678344727, + "code_tokens": 199, + "cvar95_kl": 19.996627807617188, + "delta_nll": 10.314386367797852, + "max_kl": 23.049842834472656, + "mean_kl": 10.34400749206543, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.04522613063454628 + }, + { + "candidate_nll": 7.1557207107543945, + "code_tokens": 28, + "cvar95_kl": 13.279447555541992, + "delta_nll": 5.589642524719238, + "max_kl": 14.4442777633667, + "mean_kl": 5.347708225250244, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.961407661437988, + "code_tokens": 87, + "cvar95_kl": 14.8566312789917, + "delta_nll": 6.088425159454346, + "max_kl": 17.345321655273438, + "mean_kl": 5.982692718505859, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.14942528307437897 + }, + { + "candidate_nll": 7.046088695526123, + "code_tokens": 35, + "cvar95_kl": 15.964597702026367, + "delta_nll": 5.813124656677246, + "max_kl": 19.777856826782227, + "mean_kl": 6.094069957733154, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 7.176762104034424, + "code_tokens": 62, + "cvar95_kl": 16.08839988708496, + "delta_nll": 6.53747034072876, + "max_kl": 20.523738861083984, + "mean_kl": 6.469990253448486, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.27419355511665344 + }, + { + "candidate_nll": 6.683027267456055, + "code_tokens": 58, + "cvar95_kl": 14.271475791931152, + "delta_nll": 5.577969551086426, + "max_kl": 15.383337020874023, + "mean_kl": 5.528463363647461, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 7.018076419830322, + "code_tokens": 79, + "cvar95_kl": 16.937503814697266, + "delta_nll": 5.859028339385986, + "max_kl": 19.37666130065918, + "mean_kl": 5.916988372802734, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.15189872682094574 + }, + { + "candidate_nll": 6.490160942077637, + "code_tokens": 58, + "cvar95_kl": 13.038939476013184, + "delta_nll": 5.397041320800781, + "max_kl": 13.885763168334961, + "mean_kl": 5.606827259063721, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 5.690226078033447, + "code_tokens": 48, + "cvar95_kl": 12.866801261901855, + "delta_nll": 4.52802848815918, + "max_kl": 16.75507926940918, + "mean_kl": 4.855568885803223, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 6.443325519561768, + "code_tokens": 37, + "cvar95_kl": 12.303128242492676, + "delta_nll": 5.317893028259277, + "max_kl": 13.956622123718262, + "mean_kl": 5.24326753616333, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 6.8628830909729, + "code_tokens": 61, + "cvar95_kl": 13.191947937011719, + "delta_nll": 6.155476093292236, + "max_kl": 14.516740798950195, + "mean_kl": 6.175937652587891, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.14754098653793335 + }, + { + "candidate_nll": 7.6969709396362305, + "code_tokens": 77, + "cvar95_kl": 17.545181274414062, + "delta_nll": 6.92725133895874, + "max_kl": 17.790517807006836, + "mean_kl": 6.644804000854492, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.16883116960525513 + }, + { + "candidate_nll": 9.98511028289795, + "code_tokens": 98, + "cvar95_kl": 17.957660675048828, + "delta_nll": 8.963407516479492, + "max_kl": 21.165735244750977, + "mean_kl": 9.225478172302246, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.0714285746216774 + }, + { + "candidate_nll": 9.269207954406738, + "code_tokens": 162, + "cvar95_kl": 18.82483673095703, + "delta_nll": 8.684057235717773, + "max_kl": 22.23560905456543, + "mean_kl": 8.747114181518555, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.09259258955717087 + }, + { + "candidate_nll": 8.296299934387207, + "code_tokens": 103, + "cvar95_kl": 17.763742446899414, + "delta_nll": 7.5518341064453125, + "max_kl": 20.32332420349121, + "mean_kl": 7.573688507080078, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.10679611563682556 + }, + { + "candidate_nll": 7.7683305740356445, + "code_tokens": 73, + "cvar95_kl": 16.50806427001953, + "delta_nll": 6.449359893798828, + "max_kl": 18.80563735961914, + "mean_kl": 6.513814926147461, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.15068493783473969 + }, + { + "candidate_nll": 9.06551456451416, + "code_tokens": 119, + "cvar95_kl": 19.369552612304688, + "delta_nll": 8.082540512084961, + "max_kl": 24.519207000732422, + "mean_kl": 8.548442840576172, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.10084033757448196 + }, + { + "candidate_nll": 3.376668691635132, + "code_tokens": 17, + "cvar95_kl": 8.15190315246582, + "delta_nll": 2.187145233154297, + "max_kl": 8.15190315246582, + "mean_kl": 3.164494752883911, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 6.534257411956787, + "code_tokens": 56, + "cvar95_kl": 13.889361381530762, + "delta_nll": 5.108555793762207, + "max_kl": 15.918265342712402, + "mean_kl": 5.230093955993652, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.0892857164144516 + }, + { + "candidate_nll": 7.520979404449463, + "code_tokens": 57, + "cvar95_kl": 14.918351173400879, + "delta_nll": 7.053618907928467, + "max_kl": 16.518280029296875, + "mean_kl": 6.967004776000977, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.302680015563965, + "code_tokens": 63, + "cvar95_kl": 14.549047470092773, + "delta_nll": 6.6699347496032715, + "max_kl": 17.155574798583984, + "mean_kl": 6.547885894775391, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 9.606975555419922, + "code_tokens": 200, + "cvar95_kl": 19.221988677978516, + "delta_nll": 8.76904010772705, + "max_kl": 22.904020309448242, + "mean_kl": 8.899686813354492, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.05999999865889549 + }, + { + "candidate_nll": 6.970822811126709, + "code_tokens": 30, + "cvar95_kl": 12.798995971679688, + "delta_nll": 5.886620044708252, + "max_kl": 14.37397289276123, + "mean_kl": 5.509890556335449, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 5.814707279205322, + "code_tokens": 28, + "cvar95_kl": 13.167102813720703, + "delta_nll": 5.1983561515808105, + "max_kl": 14.194252014160156, + "mean_kl": 5.007128715515137, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 5.498969554901123, + "code_tokens": 26, + "cvar95_kl": 11.649707794189453, + "delta_nll": 4.728013038635254, + "max_kl": 12.843340873718262, + "mean_kl": 4.895074844360352, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 7.164824962615967, + "code_tokens": 54, + "cvar95_kl": 13.832489967346191, + "delta_nll": 6.331883430480957, + "max_kl": 14.476320266723633, + "mean_kl": 6.284614086151123, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.538082122802734, + "code_tokens": 47, + "cvar95_kl": 14.690825462341309, + "delta_nll": 5.581346035003662, + "max_kl": 16.98521614074707, + "mean_kl": 5.917759895324707, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.1702127605676651 + }, + { + "candidate_nll": 8.035472869873047, + "code_tokens": 93, + "cvar95_kl": 15.809000015258789, + "delta_nll": 7.300943374633789, + "max_kl": 16.15323257446289, + "mean_kl": 7.164092063903809, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.10752688348293304 + }, + { + "candidate_nll": 4.676515102386475, + "code_tokens": 30, + "cvar95_kl": 8.431849479675293, + "delta_nll": 3.817140579223633, + "max_kl": 8.539586067199707, + "mean_kl": 3.9591166973114014, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.67697811126709, + "code_tokens": 89, + "cvar95_kl": 13.161036491394043, + "delta_nll": 5.6084747314453125, + "max_kl": 14.337300300598145, + "mean_kl": 5.655031204223633, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.12359550595283508 + }, + { + "candidate_nll": 6.973338603973389, + "code_tokens": 70, + "cvar95_kl": 14.098828315734863, + "delta_nll": 6.009089469909668, + "max_kl": 16.786500930786133, + "mean_kl": 5.950063705444336, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.22857142984867096 + }, + { + "candidate_nll": 5.035398006439209, + "code_tokens": 26, + "cvar95_kl": 10.231618881225586, + "delta_nll": 3.422170877456665, + "max_kl": 10.432791709899902, + "mean_kl": 4.289181232452393, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 6.318871974945068, + "code_tokens": 48, + "cvar95_kl": 14.046198844909668, + "delta_nll": 5.3611650466918945, + "max_kl": 17.829790115356445, + "mean_kl": 5.795448303222656, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 5.4732346534729, + "code_tokens": 29, + "cvar95_kl": 10.07953929901123, + "delta_nll": 4.604086875915527, + "max_kl": 10.237546920776367, + "mean_kl": 4.838384628295898, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 6.522350788116455, + "code_tokens": 17, + "cvar95_kl": 14.348174095153809, + "delta_nll": 5.4091877937316895, + "max_kl": 14.348174095153809, + "mean_kl": 5.006411075592041, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.376233100891113, + "code_tokens": 37, + "cvar95_kl": 14.797456741333008, + "delta_nll": 5.456017017364502, + "max_kl": 14.913376808166504, + "mean_kl": 5.698887825012207, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 7.668627738952637, + "code_tokens": 121, + "cvar95_kl": 17.031265258789062, + "delta_nll": 7.157226085662842, + "max_kl": 19.967878341674805, + "mean_kl": 7.242094039916992, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.10743801295757294 + }, + { + "candidate_nll": 6.366820335388184, + "code_tokens": 30, + "cvar95_kl": 13.146636962890625, + "delta_nll": 4.9679765701293945, + "max_kl": 14.615886688232422, + "mean_kl": 4.991009712219238, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.040926456451416, + "code_tokens": 74, + "cvar95_kl": 14.660947799682617, + "delta_nll": 5.142454624176025, + "max_kl": 15.444988250732422, + "mean_kl": 5.480735778808594, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 5.937473297119141, + "code_tokens": 53, + "cvar95_kl": 14.046551704406738, + "delta_nll": 5.103049278259277, + "max_kl": 14.759676933288574, + "mean_kl": 5.0830912590026855, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.15094339847564697 + }, + { + "candidate_nll": 4.668146133422852, + "code_tokens": 21, + "cvar95_kl": 8.44596004486084, + "delta_nll": 2.8087503910064697, + "max_kl": 9.288627624511719, + "mean_kl": 3.5570385456085205, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.75728178024292, + "code_tokens": 20, + "cvar95_kl": 11.201504707336426, + "delta_nll": 3.3948681354522705, + "max_kl": 11.201504707336426, + "mean_kl": 3.546494960784912, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 8.150979995727539, + "code_tokens": 122, + "cvar95_kl": 16.638042449951172, + "delta_nll": 7.331981658935547, + "max_kl": 18.353479385375977, + "mean_kl": 7.385437965393066, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.10655737668275833 + }, + { + "candidate_nll": 7.635036945343018, + "code_tokens": 94, + "cvar95_kl": 15.547398567199707, + "delta_nll": 6.794897079467773, + "max_kl": 18.332210540771484, + "mean_kl": 6.5591607093811035, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.11702127754688263 + }, + { + "candidate_nll": 3.367004156112671, + "code_tokens": 12, + "cvar95_kl": 7.005953311920166, + "delta_nll": 2.3017024993896484, + "max_kl": 7.005953311920166, + "mean_kl": 2.288357734680176, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.5833333134651184 + }, + { + "candidate_nll": 6.6283278465271, + "code_tokens": 38, + "cvar95_kl": 14.072693824768066, + "delta_nll": 5.121927738189697, + "max_kl": 14.88714599609375, + "mean_kl": 5.563094615936279, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.10526315867900848 + }, + { + "candidate_nll": 7.397514820098877, + "code_tokens": 58, + "cvar95_kl": 14.226428031921387, + "delta_nll": 6.492727756500244, + "max_kl": 14.493650436401367, + "mean_kl": 6.138402462005615, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.08620689809322357 + }, + { + "candidate_nll": 6.807084083557129, + "code_tokens": 54, + "cvar95_kl": 13.502957344055176, + "delta_nll": 6.210820198059082, + "max_kl": 14.355481147766113, + "mean_kl": 6.241952419281006, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 5.989890098571777, + "code_tokens": 40, + "cvar95_kl": 15.081245422363281, + "delta_nll": 5.058350086212158, + "max_kl": 15.785704612731934, + "mean_kl": 5.162514686584473, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 6.54926061630249, + "code_tokens": 115, + "cvar95_kl": 13.918118476867676, + "delta_nll": 6.036714553833008, + "max_kl": 15.621550559997559, + "mean_kl": 5.945743560791016, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.156521737575531 + }, + { + "candidate_nll": 6.950506687164307, + "code_tokens": 60, + "cvar95_kl": 16.80495262145996, + "delta_nll": 6.4258928298950195, + "max_kl": 17.572059631347656, + "mean_kl": 6.814407825469971, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.18333333730697632 + }, + { + "candidate_nll": 8.70337963104248, + "code_tokens": 124, + "cvar95_kl": 16.015161514282227, + "delta_nll": 7.933392524719238, + "max_kl": 18.54594612121582, + "mean_kl": 7.902065753936768, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.11290322244167328 + }, + { + "candidate_nll": 8.85893440246582, + "code_tokens": 91, + "cvar95_kl": 17.058340072631836, + "delta_nll": 8.08818531036377, + "max_kl": 17.417049407958984, + "mean_kl": 8.131766319274902, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.05494505539536476 + }, + { + "candidate_nll": 8.149469375610352, + "code_tokens": 125, + "cvar95_kl": 16.13266372680664, + "delta_nll": 7.329552173614502, + "max_kl": 18.15972137451172, + "mean_kl": 7.50166130065918, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.1120000034570694 + }, + { + "candidate_nll": 4.3069610595703125, + "code_tokens": 26, + "cvar95_kl": 8.909371376037598, + "delta_nll": 3.171084403991699, + "max_kl": 9.285487174987793, + "mean_kl": 3.5309042930603027, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 7.731120586395264, + "code_tokens": 24, + "cvar95_kl": 12.705826759338379, + "delta_nll": 5.490695476531982, + "max_kl": 13.355690956115723, + "mean_kl": 5.410254001617432, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 9.830209732055664, + "code_tokens": 180, + "cvar95_kl": 18.464027404785156, + "delta_nll": 8.976027488708496, + "max_kl": 22.785741806030273, + "mean_kl": 8.86313247680664, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.08888889104127884 + }, + { + "candidate_nll": 8.060062408447266, + "code_tokens": 40, + "cvar95_kl": 18.052146911621094, + "delta_nll": 6.685254096984863, + "max_kl": 18.32403564453125, + "mean_kl": 6.625109672546387, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 7.497457504272461, + "code_tokens": 72, + "cvar95_kl": 15.46969223022461, + "delta_nll": 6.522460460662842, + "max_kl": 17.452007293701172, + "mean_kl": 6.745613098144531, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.0972222238779068 + }, + { + "candidate_nll": 4.424750328063965, + "code_tokens": 20, + "cvar95_kl": 7.5703840255737305, + "delta_nll": 2.8471791744232178, + "max_kl": 7.5703840255737305, + "mean_kl": 3.129254102706909, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.959151268005371, + "code_tokens": 76, + "cvar95_kl": 16.4499454498291, + "delta_nll": 5.968506813049316, + "max_kl": 18.99936294555664, + "mean_kl": 6.15907096862793, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.14473684132099152 + }, + { + "candidate_nll": 7.04449462890625, + "code_tokens": 42, + "cvar95_kl": 15.559196472167969, + "delta_nll": 5.67275333404541, + "max_kl": 18.52060317993164, + "mean_kl": 6.03004264831543, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 3.9153897762298584, + "code_tokens": 28, + "cvar95_kl": 11.848109245300293, + "delta_nll": 3.3321666717529297, + "max_kl": 14.265363693237305, + "mean_kl": 3.3578941822052, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.4642857015132904 + }, + { + "candidate_nll": 8.175276756286621, + "code_tokens": 58, + "cvar95_kl": 17.275461196899414, + "delta_nll": 7.1417999267578125, + "max_kl": 18.857982635498047, + "mean_kl": 7.781470775604248, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.0517241396009922 + }, + { + "candidate_nll": 7.0295891761779785, + "code_tokens": 25, + "cvar95_kl": 14.279532432556152, + "delta_nll": 6.160939693450928, + "max_kl": 15.117413520812988, + "mean_kl": 5.761580944061279, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 8.41842269897461, + "code_tokens": 88, + "cvar95_kl": 15.879216194152832, + "delta_nll": 7.4534149169921875, + "max_kl": 19.365957260131836, + "mean_kl": 7.538500785827637, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.10227272659540176 + }, + { + "candidate_nll": 6.593632221221924, + "code_tokens": 42, + "cvar95_kl": 14.506057739257812, + "delta_nll": 5.4661149978637695, + "max_kl": 18.681884765625, + "mean_kl": 5.328307151794434, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 7.822828769683838, + "code_tokens": 24, + "cvar95_kl": 15.479490280151367, + "delta_nll": 7.003277778625488, + "max_kl": 16.93381690979004, + "mean_kl": 6.532806873321533, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 4.305536270141602, + "code_tokens": 24, + "cvar95_kl": 10.52962875366211, + "delta_nll": 3.6206648349761963, + "max_kl": 11.092144012451172, + "mean_kl": 3.650953531265259, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 8.480262756347656, + "code_tokens": 43, + "cvar95_kl": 16.859052658081055, + "delta_nll": 7.413083553314209, + "max_kl": 17.732078552246094, + "mean_kl": 7.177937984466553, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.20930232107639313 + }, + { + "candidate_nll": 6.340559482574463, + "code_tokens": 24, + "cvar95_kl": 10.772621154785156, + "delta_nll": 5.454339981079102, + "max_kl": 10.772699356079102, + "mean_kl": 4.983367443084717, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 3.4573378562927246, + "code_tokens": 14, + "cvar95_kl": 8.070866584777832, + "delta_nll": 2.144712209701538, + "max_kl": 8.070866584777832, + "mean_kl": 3.1268928050994873, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 5.62486457824707, + "code_tokens": 41, + "cvar95_kl": 12.896904945373535, + "delta_nll": 5.133821487426758, + "max_kl": 16.09858512878418, + "mean_kl": 4.89450740814209, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.17073170840740204 + }, + { + "candidate_nll": 6.033371925354004, + "code_tokens": 32, + "cvar95_kl": 12.542804718017578, + "delta_nll": 5.045032024383545, + "max_kl": 13.77367877960205, + "mean_kl": 5.262771129608154, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 6.633744239807129, + "code_tokens": 53, + "cvar95_kl": 16.57927131652832, + "delta_nll": 5.872049331665039, + "max_kl": 17.910245895385742, + "mean_kl": 5.764554500579834, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 5.568831920623779, + "code_tokens": 51, + "cvar95_kl": 13.846016883850098, + "delta_nll": 5.190540790557861, + "max_kl": 14.365521430969238, + "mean_kl": 5.020407199859619, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 6.91562557220459, + "code_tokens": 36, + "cvar95_kl": 14.131309509277344, + "delta_nll": 5.223470211029053, + "max_kl": 14.402656555175781, + "mean_kl": 6.196619033813477, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.293663024902344, + "code_tokens": 24, + "cvar95_kl": 13.232110977172852, + "delta_nll": 3.732910633087158, + "max_kl": 14.319419860839844, + "mean_kl": 3.9736149311065674, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 6.754380226135254, + "code_tokens": 39, + "cvar95_kl": 14.03863525390625, + "delta_nll": 5.4400458335876465, + "max_kl": 14.187812805175781, + "mean_kl": 5.791140079498291, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 8.480989456176758, + "code_tokens": 84, + "cvar95_kl": 17.505176544189453, + "delta_nll": 7.782577037811279, + "max_kl": 20.380849838256836, + "mean_kl": 7.898966312408447, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 6.341352462768555, + "code_tokens": 32, + "cvar95_kl": 13.133983612060547, + "delta_nll": 5.41858434677124, + "max_kl": 13.231185913085938, + "mean_kl": 5.401313304901123, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 5.1882758140563965, + "code_tokens": 19, + "cvar95_kl": 9.889150619506836, + "delta_nll": 3.878112316131592, + "max_kl": 9.889150619506836, + "mean_kl": 4.349624156951904, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 7.030200958251953, + "code_tokens": 50, + "cvar95_kl": 14.505355834960938, + "delta_nll": 6.212224006652832, + "max_kl": 15.834488868713379, + "mean_kl": 5.8623948097229, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.2199999988079071 + }, + { + "candidate_nll": 7.222283363342285, + "code_tokens": 71, + "cvar95_kl": 14.494232177734375, + "delta_nll": 6.722469806671143, + "max_kl": 14.999835014343262, + "mean_kl": 6.964103698730469, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.09859155118465424 + }, + { + "candidate_nll": 6.122281074523926, + "code_tokens": 28, + "cvar95_kl": 9.486889839172363, + "delta_nll": 4.734312534332275, + "max_kl": 10.095641136169434, + "mean_kl": 4.601841449737549, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.865962028503418, + "code_tokens": 76, + "cvar95_kl": 19.253814697265625, + "delta_nll": 5.996700763702393, + "max_kl": 22.17168426513672, + "mean_kl": 6.033330917358398, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 5.9624409675598145, + "code_tokens": 34, + "cvar95_kl": 12.761094093322754, + "delta_nll": 4.999855995178223, + "max_kl": 12.836151123046875, + "mean_kl": 5.190700531005859, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 4.6816086769104, + "code_tokens": 11, + "cvar95_kl": 7.527197360992432, + "delta_nll": 2.53421688079834, + "max_kl": 7.527197360992432, + "mean_kl": 2.654057264328003, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 6.6416096687316895, + "code_tokens": 31, + "cvar95_kl": 14.634297370910645, + "delta_nll": 5.225342273712158, + "max_kl": 16.78049087524414, + "mean_kl": 5.194468975067139, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 6.821870803833008, + "code_tokens": 118, + "cvar95_kl": 14.3765230178833, + "delta_nll": 5.975094795227051, + "max_kl": 15.634370803833008, + "mean_kl": 6.303758144378662, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.1355932205915451 + }, + { + "candidate_nll": 8.079778671264648, + "code_tokens": 60, + "cvar95_kl": 18.416202545166016, + "delta_nll": 7.304082870483398, + "max_kl": 19.955551147460938, + "mean_kl": 7.156256198883057, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 6.002645492553711, + "code_tokens": 47, + "cvar95_kl": 12.942679405212402, + "delta_nll": 5.1071953773498535, + "max_kl": 14.372639656066895, + "mean_kl": 5.31866979598999, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.1489361673593521 + }, + { + "candidate_nll": 9.416496276855469, + "code_tokens": 84, + "cvar95_kl": 18.948856353759766, + "delta_nll": 8.264481544494629, + "max_kl": 19.574520111083984, + "mean_kl": 8.656302452087402, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 5.761261940002441, + "code_tokens": 27, + "cvar95_kl": 14.233955383300781, + "delta_nll": 4.76723575592041, + "max_kl": 15.80260944366455, + "mean_kl": 5.222951889038086, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.4479804039001465, + "code_tokens": 19, + "cvar95_kl": 16.130245208740234, + "delta_nll": 4.836073398590088, + "max_kl": 16.130245208740234, + "mean_kl": 4.6856913566589355, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 7.907400608062744, + "code_tokens": 114, + "cvar95_kl": 14.967144012451172, + "delta_nll": 7.134387969970703, + "max_kl": 16.650287628173828, + "mean_kl": 7.173661231994629, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.07894736528396606 + }, + { + "candidate_nll": 7.412554740905762, + "code_tokens": 40, + "cvar95_kl": 13.963188171386719, + "delta_nll": 5.684661388397217, + "max_kl": 15.963077545166016, + "mean_kl": 5.6255903244018555, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 6.776309967041016, + "code_tokens": 25, + "cvar95_kl": 9.863590240478516, + "delta_nll": 5.2906317710876465, + "max_kl": 10.055238723754883, + "mean_kl": 5.0812788009643555, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 5.517434597015381, + "code_tokens": 23, + "cvar95_kl": 12.117416381835938, + "delta_nll": 4.746745586395264, + "max_kl": 12.49785327911377, + "mean_kl": 4.688004970550537, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 7.9940571784973145, + "code_tokens": 77, + "cvar95_kl": 17.249282836914062, + "delta_nll": 6.977707862854004, + "max_kl": 18.546146392822266, + "mean_kl": 7.072796821594238, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.12987013161182404 + }, + { + "candidate_nll": 4.78372859954834, + "code_tokens": 19, + "cvar95_kl": 12.62852954864502, + "delta_nll": 3.676063060760498, + "max_kl": 12.62852954864502, + "mean_kl": 3.66679310798645, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 5.142388343811035, + "code_tokens": 37, + "cvar95_kl": 9.954858779907227, + "delta_nll": 4.159144401550293, + "max_kl": 10.157228469848633, + "mean_kl": 4.6051859855651855, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 6.651440620422363, + "code_tokens": 33, + "cvar95_kl": 13.766603469848633, + "delta_nll": 5.268275737762451, + "max_kl": 14.344676971435547, + "mean_kl": 5.354461193084717, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 6.142171859741211, + "code_tokens": 95, + "cvar95_kl": 13.834988594055176, + "delta_nll": 5.482330322265625, + "max_kl": 17.526512145996094, + "mean_kl": 5.350018501281738, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.23157894611358643 + }, + { + "candidate_nll": 9.08430004119873, + "code_tokens": 125, + "cvar95_kl": 17.419191360473633, + "delta_nll": 8.071139335632324, + "max_kl": 19.307126998901367, + "mean_kl": 7.962515354156494, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.1120000034570694 + }, + { + "candidate_nll": 9.381268501281738, + "code_tokens": 168, + "cvar95_kl": 17.334503173828125, + "delta_nll": 8.668220520019531, + "max_kl": 21.0098876953125, + "mean_kl": 8.511295318603516, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.0773809552192688 + }, + { + "candidate_nll": 6.470621585845947, + "code_tokens": 31, + "cvar95_kl": 17.443098068237305, + "delta_nll": 5.155219554901123, + "max_kl": 18.13502311706543, + "mean_kl": 5.39203405380249, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 8.20223331451416, + "code_tokens": 83, + "cvar95_kl": 18.606388092041016, + "delta_nll": 7.440526485443115, + "max_kl": 22.566837310791016, + "mean_kl": 7.3162312507629395, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.14457830786705017 + }, + { + "candidate_nll": 6.222802639007568, + "code_tokens": 50, + "cvar95_kl": 15.729897499084473, + "delta_nll": 5.522860050201416, + "max_kl": 20.821619033813477, + "mean_kl": 5.754718780517578, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 6.629868984222412, + "code_tokens": 69, + "cvar95_kl": 15.336938858032227, + "delta_nll": 6.2626872062683105, + "max_kl": 17.314664840698242, + "mean_kl": 6.122338771820068, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 7.66568660736084, + "code_tokens": 50, + "cvar95_kl": 15.838849067687988, + "delta_nll": 6.5807905197143555, + "max_kl": 18.174306869506836, + "mean_kl": 6.948843002319336, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 10.077898025512695, + "code_tokens": 161, + "cvar95_kl": 18.912843704223633, + "delta_nll": 9.074116706848145, + "max_kl": 21.143110275268555, + "mean_kl": 8.933684349060059, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.06832297891378403 + }, + { + "candidate_nll": 7.146995544433594, + "code_tokens": 53, + "cvar95_kl": 13.860424041748047, + "delta_nll": 6.491219520568848, + "max_kl": 14.79946231842041, + "mean_kl": 6.580214023590088, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.16981132328510284 + }, + { + "candidate_nll": 9.406006813049316, + "code_tokens": 97, + "cvar95_kl": 18.87972640991211, + "delta_nll": 8.960478782653809, + "max_kl": 22.423992156982422, + "mean_kl": 8.578316688537598, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.11340206116437912 + }, + { + "candidate_nll": 5.115037441253662, + "code_tokens": 22, + "cvar95_kl": 9.433389663696289, + "delta_nll": 3.80122447013855, + "max_kl": 11.015423774719238, + "mean_kl": 3.9210100173950195, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 6.4200968742370605, + "code_tokens": 34, + "cvar95_kl": 14.788166046142578, + "delta_nll": 5.551192760467529, + "max_kl": 15.553927421569824, + "mean_kl": 5.403462886810303, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.526176452636719, + "code_tokens": 81, + "cvar95_kl": 13.722704887390137, + "delta_nll": 5.6675333976745605, + "max_kl": 15.721179008483887, + "mean_kl": 5.909884452819824, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.09876543283462524 + }, + { + "candidate_nll": 3.946226119995117, + "code_tokens": 17, + "cvar95_kl": 11.121929168701172, + "delta_nll": 2.6841228008270264, + "max_kl": 11.121929168701172, + "mean_kl": 2.9546749591827393, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 7.385559558868408, + "code_tokens": 181, + "cvar95_kl": 17.896848678588867, + "delta_nll": 6.560302257537842, + "max_kl": 24.047876358032227, + "mean_kl": 6.739127159118652, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.14917127788066864 + }, + { + "candidate_nll": 6.850899696350098, + "code_tokens": 63, + "cvar95_kl": 13.490833282470703, + "delta_nll": 5.870491027832031, + "max_kl": 14.204180717468262, + "mean_kl": 6.037862300872803, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.1587301641702652 + }, + { + "candidate_nll": 5.135871887207031, + "code_tokens": 35, + "cvar95_kl": 11.034538269042969, + "delta_nll": 4.442348957061768, + "max_kl": 11.58460807800293, + "mean_kl": 4.449858665466309, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.95594596862793, + "code_tokens": 25, + "cvar95_kl": 10.701911926269531, + "delta_nll": 4.151158332824707, + "max_kl": 11.84148120880127, + "mean_kl": 3.9529993534088135, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 6.727231025695801, + "code_tokens": 60, + "cvar95_kl": 13.86137866973877, + "delta_nll": 5.6388654708862305, + "max_kl": 14.952767372131348, + "mean_kl": 6.398132801055908, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 8.775725364685059, + "code_tokens": 86, + "cvar95_kl": 17.023845672607422, + "delta_nll": 7.864078521728516, + "max_kl": 20.28631019592285, + "mean_kl": 7.625515460968018, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.09302325546741486 + }, + { + "candidate_nll": 8.07038688659668, + "code_tokens": 40, + "cvar95_kl": 16.8223934173584, + "delta_nll": 7.144221305847168, + "max_kl": 17.148508071899414, + "mean_kl": 7.009854793548584, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 5.040236473083496, + "code_tokens": 27, + "cvar95_kl": 10.131795883178711, + "delta_nll": 3.4766762256622314, + "max_kl": 11.285076141357422, + "mean_kl": 3.471076011657715, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.441808700561523, + "code_tokens": 52, + "cvar95_kl": 14.448826789855957, + "delta_nll": 6.3522257804870605, + "max_kl": 14.995779991149902, + "mean_kl": 6.891955375671387, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.17307692766189575 + }, + { + "candidate_nll": 8.655117988586426, + "code_tokens": 71, + "cvar95_kl": 21.44781494140625, + "delta_nll": 7.913243770599365, + "max_kl": 26.590444564819336, + "mean_kl": 7.899590969085693, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.14084507524967194 + }, + { + "candidate_nll": 5.937989711761475, + "code_tokens": 25, + "cvar95_kl": 11.853952407836914, + "delta_nll": 4.836137771606445, + "max_kl": 13.121256828308105, + "mean_kl": 4.774961948394775, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.177437782287598, + "code_tokens": 103, + "cvar95_kl": 17.095247268676758, + "delta_nll": 7.582942008972168, + "max_kl": 19.969484329223633, + "mean_kl": 7.345445156097412, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.12621359527111053 + }, + { + "candidate_nll": 7.9952239990234375, + "code_tokens": 116, + "cvar95_kl": 18.523584365844727, + "delta_nll": 6.937685966491699, + "max_kl": 23.41889762878418, + "mean_kl": 6.962935447692871, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.12931033968925476 + }, + { + "candidate_nll": 8.852417945861816, + "code_tokens": 107, + "cvar95_kl": 17.415542602539062, + "delta_nll": 8.204071998596191, + "max_kl": 18.556264877319336, + "mean_kl": 8.442631721496582, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.09345794469118118 + }, + { + "candidate_nll": 6.470614433288574, + "code_tokens": 54, + "cvar95_kl": 14.508312225341797, + "delta_nll": 5.527576446533203, + "max_kl": 15.388725280761719, + "mean_kl": 5.630562782287598, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.24074074625968933 + }, + { + "candidate_nll": 6.113170623779297, + "code_tokens": 46, + "cvar95_kl": 12.886042594909668, + "delta_nll": 5.479778289794922, + "max_kl": 14.093743324279785, + "mean_kl": 5.421306610107422, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 8.698545455932617, + "code_tokens": 170, + "cvar95_kl": 20.179244995117188, + "delta_nll": 8.369399070739746, + "max_kl": 22.369979858398438, + "mean_kl": 8.315814971923828, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.0941176488995552 + }, + { + "candidate_nll": 7.397311687469482, + "code_tokens": 45, + "cvar95_kl": 12.99881649017334, + "delta_nll": 5.970619201660156, + "max_kl": 16.97552490234375, + "mean_kl": 5.569902420043945, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 6.178398609161377, + "code_tokens": 24, + "cvar95_kl": 12.854195594787598, + "delta_nll": 4.770095348358154, + "max_kl": 14.333632469177246, + "mean_kl": 4.962522983551025, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.452371597290039, + "code_tokens": 78, + "cvar95_kl": 14.837325096130371, + "delta_nll": 5.290411949157715, + "max_kl": 18.91684341430664, + "mean_kl": 5.479903221130371, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.12820513546466827 + }, + { + "candidate_nll": 5.908323764801025, + "code_tokens": 33, + "cvar95_kl": 12.341875076293945, + "delta_nll": 5.277766704559326, + "max_kl": 13.116974830627441, + "mean_kl": 4.884599208831787, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 7.88120174407959, + "code_tokens": 32, + "cvar95_kl": 15.688665390014648, + "delta_nll": 6.76553201675415, + "max_kl": 16.707632064819336, + "mean_kl": 6.403326034545898, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.296677589416504, + "code_tokens": 77, + "cvar95_kl": 13.900920867919922, + "delta_nll": 6.30792760848999, + "max_kl": 14.846196174621582, + "mean_kl": 6.377599716186523, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.15584415197372437 + }, + { + "candidate_nll": 8.873444557189941, + "code_tokens": 222, + "cvar95_kl": 19.775392532348633, + "delta_nll": 8.451193809509277, + "max_kl": 22.783096313476562, + "mean_kl": 8.405142784118652, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.09459459781646729 + }, + { + "candidate_nll": 8.870773315429688, + "code_tokens": 86, + "cvar95_kl": 17.412418365478516, + "delta_nll": 7.822178363800049, + "max_kl": 20.02899169921875, + "mean_kl": 7.906457424163818, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.08139535039663315 + }, + { + "candidate_nll": 4.537069797515869, + "code_tokens": 31, + "cvar95_kl": 10.35316276550293, + "delta_nll": 4.203020095825195, + "max_kl": 12.050481796264648, + "mean_kl": 4.1203813552856445, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 5.001067638397217, + "code_tokens": 34, + "cvar95_kl": 13.283382415771484, + "delta_nll": 4.489467620849609, + "max_kl": 14.214451789855957, + "mean_kl": 4.542171478271484, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 7.253037452697754, + "code_tokens": 38, + "cvar95_kl": 13.718513488769531, + "delta_nll": 5.865577220916748, + "max_kl": 14.291269302368164, + "mean_kl": 6.098138809204102, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.10526315867900848 + }, + { + "candidate_nll": 7.480920314788818, + "code_tokens": 93, + "cvar95_kl": 16.264171600341797, + "delta_nll": 6.517875671386719, + "max_kl": 23.037782669067383, + "mean_kl": 6.502486705780029, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.08602150529623032 + }, + { + "candidate_nll": 7.635690689086914, + "code_tokens": 70, + "cvar95_kl": 16.25494956970215, + "delta_nll": 6.857085227966309, + "max_kl": 18.674331665039062, + "mean_kl": 7.0344672203063965, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.11428571492433548 + }, + { + "candidate_nll": 6.906419277191162, + "code_tokens": 67, + "cvar95_kl": 16.37234878540039, + "delta_nll": 6.0959272384643555, + "max_kl": 18.49078369140625, + "mean_kl": 6.012510299682617, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.17910447716712952 + }, + { + "candidate_nll": 6.242733955383301, + "code_tokens": 66, + "cvar95_kl": 11.875757217407227, + "delta_nll": 4.956606864929199, + "max_kl": 13.200735092163086, + "mean_kl": 5.1290974617004395, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 5.174868583679199, + "code_tokens": 34, + "cvar95_kl": 9.649105072021484, + "delta_nll": 4.162574291229248, + "max_kl": 10.225178718566895, + "mean_kl": 4.23341178894043, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.951855182647705, + "code_tokens": 64, + "cvar95_kl": 13.238517761230469, + "delta_nll": 5.041830062866211, + "max_kl": 14.892927169799805, + "mean_kl": 5.148162841796875, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 6.77103853225708, + "code_tokens": 56, + "cvar95_kl": 14.13158893585205, + "delta_nll": 5.4606828689575195, + "max_kl": 19.57667350769043, + "mean_kl": 5.653182506561279, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.1607142835855484 + }, + { + "candidate_nll": 6.622738838195801, + "code_tokens": 21, + "cvar95_kl": 14.023832321166992, + "delta_nll": 5.497656345367432, + "max_kl": 16.094491958618164, + "mean_kl": 5.128180027008057, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.21910285949707, + "code_tokens": 60, + "cvar95_kl": 14.567971229553223, + "delta_nll": 5.553762435913086, + "max_kl": 16.601163864135742, + "mean_kl": 5.917576789855957, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.557610034942627, + "code_tokens": 29, + "cvar95_kl": 12.790719032287598, + "delta_nll": 5.203834056854248, + "max_kl": 12.820716857910156, + "mean_kl": 5.446712493896484, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.8652729988098145, + "code_tokens": 34, + "cvar95_kl": 17.113784790039062, + "delta_nll": 7.143228054046631, + "max_kl": 19.543848037719727, + "mean_kl": 6.943400859832764, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 8.246520042419434, + "code_tokens": 88, + "cvar95_kl": 16.24700355529785, + "delta_nll": 7.3274126052856445, + "max_kl": 19.71839714050293, + "mean_kl": 7.283341884613037, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.15909090638160706 + }, + { + "candidate_nll": 5.491734981536865, + "code_tokens": 17, + "cvar95_kl": 10.644609451293945, + "delta_nll": 4.948511123657227, + "max_kl": 10.644609451293945, + "mean_kl": 4.363007068634033, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.736550331115723, + "code_tokens": 35, + "cvar95_kl": 11.282350540161133, + "delta_nll": 4.84600305557251, + "max_kl": 11.850091934204102, + "mean_kl": 4.9081854820251465, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.241963863372803, + "code_tokens": 48, + "cvar95_kl": 14.838665008544922, + "delta_nll": 6.352187156677246, + "max_kl": 16.55344009399414, + "mean_kl": 6.5219855308532715, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 8.435922622680664, + "code_tokens": 130, + "cvar95_kl": 16.377361297607422, + "delta_nll": 7.445650100708008, + "max_kl": 18.765535354614258, + "mean_kl": 7.242146015167236, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.1230769231915474 + }, + { + "candidate_nll": 9.55981159210205, + "code_tokens": 65, + "cvar95_kl": 18.44716453552246, + "delta_nll": 8.464872360229492, + "max_kl": 19.691984176635742, + "mean_kl": 8.405108451843262, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.0923076942563057 + }, + { + "candidate_nll": 9.016061782836914, + "code_tokens": 145, + "cvar95_kl": 18.482589721679688, + "delta_nll": 8.141046524047852, + "max_kl": 20.016944885253906, + "mean_kl": 8.408459663391113, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.07586207240819931 + }, + { + "candidate_nll": 9.044563293457031, + "code_tokens": 80, + "cvar95_kl": 18.33016014099121, + "delta_nll": 8.328864097595215, + "max_kl": 20.221359252929688, + "mean_kl": 8.066650390625, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.08749999850988388 + }, + { + "candidate_nll": 6.989552974700928, + "code_tokens": 70, + "cvar95_kl": 15.425893783569336, + "delta_nll": 6.407416343688965, + "max_kl": 16.882461547851562, + "mean_kl": 6.784552097320557, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.334779262542725, + "code_tokens": 25, + "cvar95_kl": 11.710289001464844, + "delta_nll": 4.846490383148193, + "max_kl": 12.215002059936523, + "mean_kl": 4.687609672546387, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 6.7939534187316895, + "code_tokens": 50, + "cvar95_kl": 13.885425567626953, + "delta_nll": 5.139570236206055, + "max_kl": 17.146818161010742, + "mean_kl": 5.788064479827881, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 7.527553558349609, + "code_tokens": 48, + "cvar95_kl": 16.445228576660156, + "delta_nll": 6.72951078414917, + "max_kl": 19.281204223632812, + "mean_kl": 6.882157802581787, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.790351390838623, + "code_tokens": 104, + "cvar95_kl": 20.341276168823242, + "delta_nll": 7.446279048919678, + "max_kl": 23.343019485473633, + "mean_kl": 7.401629447937012, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.14423076808452606 + }, + { + "candidate_nll": 6.028661727905273, + "code_tokens": 68, + "cvar95_kl": 12.51672649383545, + "delta_nll": 5.038667678833008, + "max_kl": 14.915531158447266, + "mean_kl": 5.229028701782227, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 7.034127712249756, + "code_tokens": 145, + "cvar95_kl": 15.46589469909668, + "delta_nll": 6.2792158126831055, + "max_kl": 17.274089813232422, + "mean_kl": 6.435276985168457, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.11034482717514038 + }, + { + "candidate_nll": 5.18171501159668, + "code_tokens": 41, + "cvar95_kl": 10.046242713928223, + "delta_nll": 4.336827278137207, + "max_kl": 10.959515571594238, + "mean_kl": 4.739192962646484, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.1463414579629898 + }, + { + "candidate_nll": 9.180143356323242, + "code_tokens": 53, + "cvar95_kl": 18.818880081176758, + "delta_nll": 8.419020652770996, + "max_kl": 19.769147872924805, + "mean_kl": 8.407691955566406, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.1320754736661911 + }, + { + "candidate_nll": 7.087182998657227, + "code_tokens": 58, + "cvar95_kl": 13.941725730895996, + "delta_nll": 6.294953346252441, + "max_kl": 15.858210563659668, + "mean_kl": 6.024244785308838, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.18965516984462738 + }, + { + "candidate_nll": 6.24513053894043, + "code_tokens": 24, + "cvar95_kl": 14.667718887329102, + "delta_nll": 4.513070583343506, + "max_kl": 16.18686294555664, + "mean_kl": 5.1588544845581055, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.216503143310547, + "code_tokens": 89, + "cvar95_kl": 16.048620223999023, + "delta_nll": 7.224649429321289, + "max_kl": 18.193634033203125, + "mean_kl": 7.485149383544922, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.11235955357551575 + }, + { + "candidate_nll": 8.199848175048828, + "code_tokens": 101, + "cvar95_kl": 18.65972328186035, + "delta_nll": 7.7147722244262695, + "max_kl": 23.198593139648438, + "mean_kl": 7.673948764801025, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.09900990128517151 + }, + { + "candidate_nll": 6.424108505249023, + "code_tokens": 33, + "cvar95_kl": 12.516359329223633, + "delta_nll": 4.966550827026367, + "max_kl": 14.038970947265625, + "mean_kl": 5.396251201629639, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 7.818647384643555, + "code_tokens": 50, + "cvar95_kl": 16.052085876464844, + "delta_nll": 6.771981716156006, + "max_kl": 17.116504669189453, + "mean_kl": 6.893701076507568, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 7.268710613250732, + "code_tokens": 58, + "cvar95_kl": 13.86573314666748, + "delta_nll": 5.859957695007324, + "max_kl": 15.29179573059082, + "mean_kl": 5.935037136077881, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.12068965286016464 + }, + { + "candidate_nll": 6.6865339279174805, + "code_tokens": 62, + "cvar95_kl": 13.4817533493042, + "delta_nll": 5.7073750495910645, + "max_kl": 14.443244934082031, + "mean_kl": 5.663556098937988, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.09677419066429138 + }, + { + "candidate_nll": 7.081045150756836, + "code_tokens": 42, + "cvar95_kl": 13.133261680603027, + "delta_nll": 6.389355182647705, + "max_kl": 14.881226539611816, + "mean_kl": 6.2693562507629395, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 3.9954094886779785, + "code_tokens": 15, + "cvar95_kl": 11.557120323181152, + "delta_nll": 3.0706684589385986, + "max_kl": 11.557120323181152, + "mean_kl": 3.7637083530426025, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 6.809692859649658, + "code_tokens": 57, + "cvar95_kl": 17.034212112426758, + "delta_nll": 6.070910453796387, + "max_kl": 17.698726654052734, + "mean_kl": 6.132185459136963, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 7.94233512878418, + "code_tokens": 58, + "cvar95_kl": 20.32477569580078, + "delta_nll": 6.898798942565918, + "max_kl": 21.837221145629883, + "mean_kl": 6.687595844268799, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 5.957248210906982, + "code_tokens": 45, + "cvar95_kl": 14.812076568603516, + "delta_nll": 5.085277080535889, + "max_kl": 15.794858932495117, + "mean_kl": 5.26712703704834, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 7.738927841186523, + "code_tokens": 32, + "cvar95_kl": 15.25554370880127, + "delta_nll": 5.709697723388672, + "max_kl": 16.09642791748047, + "mean_kl": 6.262681007385254, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.420883655548096, + "code_tokens": 39, + "cvar95_kl": 14.449610710144043, + "delta_nll": 6.100217342376709, + "max_kl": 14.499865531921387, + "mean_kl": 6.473056316375732, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 7.232837200164795, + "code_tokens": 136, + "cvar95_kl": 16.106534957885742, + "delta_nll": 6.822075843811035, + "max_kl": 19.957639694213867, + "mean_kl": 6.892885684967041, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.13235294818878174 + }, + { + "candidate_nll": 7.568854808807373, + "code_tokens": 50, + "cvar95_kl": 15.772099494934082, + "delta_nll": 6.348098278045654, + "max_kl": 18.305295944213867, + "mean_kl": 6.213040828704834, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 7.602375507354736, + "code_tokens": 34, + "cvar95_kl": 17.294815063476562, + "delta_nll": 6.736459732055664, + "max_kl": 17.79904556274414, + "mean_kl": 6.852627277374268, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.11764705926179886 + }, + { + "candidate_nll": 7.2490925788879395, + "code_tokens": 44, + "cvar95_kl": 13.881840705871582, + "delta_nll": 6.070302486419678, + "max_kl": 14.445178031921387, + "mean_kl": 6.427465915679932, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.09090909361839294 + }, + { + "candidate_nll": 5.622386932373047, + "code_tokens": 36, + "cvar95_kl": 11.831518173217773, + "delta_nll": 4.18896484375, + "max_kl": 12.934969902038574, + "mean_kl": 4.7474870681762695, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.530400276184082, + "code_tokens": 46, + "cvar95_kl": 13.860983848571777, + "delta_nll": 6.727242946624756, + "max_kl": 14.479036331176758, + "mean_kl": 6.773794651031494, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.08695652335882187 + }, + { + "candidate_nll": 7.678246974945068, + "code_tokens": 48, + "cvar95_kl": 14.112056732177734, + "delta_nll": 6.4979071617126465, + "max_kl": 15.582676887512207, + "mean_kl": 6.9049530029296875, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 6.095922946929932, + "code_tokens": 25, + "cvar95_kl": 11.65941333770752, + "delta_nll": 5.200511932373047, + "max_kl": 11.806264877319336, + "mean_kl": 4.468298435211182, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 7.5233659744262695, + "code_tokens": 69, + "cvar95_kl": 15.975589752197266, + "delta_nll": 6.019086837768555, + "max_kl": 18.59939193725586, + "mean_kl": 6.42559289932251, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.10144927352666855 + }, + { + "candidate_nll": 5.787787914276123, + "code_tokens": 60, + "cvar95_kl": 12.390335083007812, + "delta_nll": 4.822330474853516, + "max_kl": 14.561538696289062, + "mean_kl": 5.239902019500732, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 6.573622226715088, + "code_tokens": 49, + "cvar95_kl": 13.158638000488281, + "delta_nll": 5.659144878387451, + "max_kl": 19.048328399658203, + "mean_kl": 5.485144138336182, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.12244898080825806 + }, + { + "candidate_nll": 7.065051078796387, + "code_tokens": 44, + "cvar95_kl": 14.520142555236816, + "delta_nll": 5.364522933959961, + "max_kl": 17.408767700195312, + "mean_kl": 5.369234085083008, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.13636364042758942 + }, + { + "candidate_nll": 8.253899574279785, + "code_tokens": 81, + "cvar95_kl": 16.77047348022461, + "delta_nll": 7.108485221862793, + "max_kl": 19.535070419311523, + "mean_kl": 7.444390296936035, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 7.351226329803467, + "code_tokens": 26, + "cvar95_kl": 16.176025390625, + "delta_nll": 5.914491176605225, + "max_kl": 19.31952667236328, + "mean_kl": 5.955350399017334, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 7.865180015563965, + "code_tokens": 32, + "cvar95_kl": 15.386016845703125, + "delta_nll": 6.985259056091309, + "max_kl": 15.524510383605957, + "mean_kl": 6.713606357574463, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.15625 + }, + { + "candidate_nll": 5.6524128913879395, + "code_tokens": 48, + "cvar95_kl": 14.929228782653809, + "delta_nll": 4.7046709060668945, + "max_kl": 16.145933151245117, + "mean_kl": 5.300203800201416, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 9.771458625793457, + "code_tokens": 180, + "cvar95_kl": 18.343589782714844, + "delta_nll": 9.072137832641602, + "max_kl": 20.86124610900879, + "mean_kl": 8.989375114440918, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.08888889104127884 + }, + { + "candidate_nll": 6.007436275482178, + "code_tokens": 38, + "cvar95_kl": 13.268260955810547, + "delta_nll": 5.285674571990967, + "max_kl": 13.305304527282715, + "mean_kl": 5.074159622192383, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 8.867239952087402, + "code_tokens": 90, + "cvar95_kl": 16.45772933959961, + "delta_nll": 8.172178268432617, + "max_kl": 18.37931251525879, + "mean_kl": 8.217884063720703, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 6.380977153778076, + "code_tokens": 44, + "cvar95_kl": 14.730422019958496, + "delta_nll": 5.806346893310547, + "max_kl": 16.83793830871582, + "mean_kl": 5.824166774749756, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 6.164734840393066, + "code_tokens": 60, + "cvar95_kl": 14.431235313415527, + "delta_nll": 5.420834541320801, + "max_kl": 17.518796920776367, + "mean_kl": 5.7159743309021, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 5.84304141998291, + "code_tokens": 69, + "cvar95_kl": 10.121077537536621, + "delta_nll": 4.840243339538574, + "max_kl": 11.358295440673828, + "mean_kl": 4.907491683959961, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 7.227229595184326, + "code_tokens": 71, + "cvar95_kl": 17.441532135009766, + "delta_nll": 6.709667682647705, + "max_kl": 18.305423736572266, + "mean_kl": 6.9408979415893555, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.11267605423927307 + }, + { + "candidate_nll": 5.8947434425354, + "code_tokens": 22, + "cvar95_kl": 11.714811325073242, + "delta_nll": 4.652070045471191, + "max_kl": 13.064974784851074, + "mean_kl": 4.970773220062256, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 9.324915885925293, + "code_tokens": 215, + "cvar95_kl": 17.665220260620117, + "delta_nll": 8.771136283874512, + "max_kl": 18.958698272705078, + "mean_kl": 8.841898918151855, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.07906977087259293 + }, + { + "candidate_nll": 7.081709861755371, + "code_tokens": 26, + "cvar95_kl": 15.566751480102539, + "delta_nll": 5.978919982910156, + "max_kl": 18.20821189880371, + "mean_kl": 5.296440124511719, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 6.490296840667725, + "code_tokens": 33, + "cvar95_kl": 14.281147003173828, + "delta_nll": 5.6220502853393555, + "max_kl": 15.152495384216309, + "mean_kl": 5.185719013214111, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 5.1204142570495605, + "code_tokens": 18, + "cvar95_kl": 10.75286865234375, + "delta_nll": 4.184266090393066, + "max_kl": 10.75286865234375, + "mean_kl": 4.450979232788086, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.2522430419921875, + "code_tokens": 71, + "cvar95_kl": 16.71743392944336, + "delta_nll": 4.97218132019043, + "max_kl": 22.1542911529541, + "mean_kl": 5.413252830505371, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.22535210847854614 + }, + { + "candidate_nll": 7.222535133361816, + "code_tokens": 107, + "cvar95_kl": 16.988676071166992, + "delta_nll": 6.441133499145508, + "max_kl": 20.65082550048828, + "mean_kl": 6.098114490509033, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.13084112107753754 + }, + { + "candidate_nll": 7.058650016784668, + "code_tokens": 40, + "cvar95_kl": 14.845174789428711, + "delta_nll": 5.003625392913818, + "max_kl": 15.869563102722168, + "mean_kl": 5.999375820159912, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 6.346510410308838, + "code_tokens": 32, + "cvar95_kl": 11.413649559020996, + "delta_nll": 4.982213497161865, + "max_kl": 11.807063102722168, + "mean_kl": 5.132983207702637, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 8.267568588256836, + "code_tokens": 130, + "cvar95_kl": 13.28575325012207, + "delta_nll": 7.568315029144287, + "max_kl": 14.319347381591797, + "mean_kl": 7.422452926635742, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.07692307978868484 + }, + { + "candidate_nll": 5.449161052703857, + "code_tokens": 27, + "cvar95_kl": 7.292428493499756, + "delta_nll": 3.9777393341064453, + "max_kl": 7.8574724197387695, + "mean_kl": 3.7615773677825928, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 7.074109077453613, + "code_tokens": 42, + "cvar95_kl": 14.093789100646973, + "delta_nll": 6.101168632507324, + "max_kl": 15.994047164916992, + "mean_kl": 5.975953578948975, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.248098850250244, + "code_tokens": 30, + "cvar95_kl": 12.730339050292969, + "delta_nll": 5.502585411071777, + "max_kl": 14.530223846435547, + "mean_kl": 5.304936408996582, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.995838165283203, + "code_tokens": 58, + "cvar95_kl": 14.84289836883545, + "delta_nll": 5.576605319976807, + "max_kl": 15.61310863494873, + "mean_kl": 5.631900310516357, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 6.551945209503174, + "code_tokens": 34, + "cvar95_kl": 12.631610870361328, + "delta_nll": 5.321718215942383, + "max_kl": 14.686688423156738, + "mean_kl": 5.567238807678223, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.0882352963089943 + }, + { + "candidate_nll": 9.657171249389648, + "code_tokens": 208, + "cvar95_kl": 17.302213668823242, + "delta_nll": 8.955708503723145, + "max_kl": 19.143171310424805, + "mean_kl": 9.1058931350708, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.07211538404226303 + }, + { + "candidate_nll": 6.7491912841796875, + "code_tokens": 83, + "cvar95_kl": 12.9851655960083, + "delta_nll": 5.850285530090332, + "max_kl": 14.47718620300293, + "mean_kl": 5.778772354125977, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.15662650763988495 + }, + { + "candidate_nll": 8.450944900512695, + "code_tokens": 170, + "cvar95_kl": 15.911176681518555, + "delta_nll": 7.928389072418213, + "max_kl": 17.530845642089844, + "mean_kl": 7.866115570068359, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 4.289580821990967, + "code_tokens": 40, + "cvar95_kl": 12.897073745727539, + "delta_nll": 3.508160352706909, + "max_kl": 14.365889549255371, + "mean_kl": 4.307861328125, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 5.936556339263916, + "code_tokens": 31, + "cvar95_kl": 13.62869644165039, + "delta_nll": 4.689846992492676, + "max_kl": 16.158498764038086, + "mean_kl": 4.987310886383057, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 6.516602993011475, + "code_tokens": 42, + "cvar95_kl": 16.66765022277832, + "delta_nll": 5.721833229064941, + "max_kl": 19.54392433166504, + "mean_kl": 6.196187496185303, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 6.453505992889404, + "code_tokens": 27, + "cvar95_kl": 11.536044120788574, + "delta_nll": 4.6844401359558105, + "max_kl": 13.074862480163574, + "mean_kl": 5.124514579772949, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.296087265014648, + "code_tokens": 25, + "cvar95_kl": 12.028911590576172, + "delta_nll": 4.096276760101318, + "max_kl": 12.118614196777344, + "mean_kl": 5.052201747894287, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 6.362607002258301, + "code_tokens": 40, + "cvar95_kl": 15.711528778076172, + "delta_nll": 5.794220447540283, + "max_kl": 16.112594604492188, + "mean_kl": 5.8477888107299805, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 5.448092937469482, + "code_tokens": 24, + "cvar95_kl": 9.745088577270508, + "delta_nll": 4.0637383460998535, + "max_kl": 10.340706825256348, + "mean_kl": 3.9863250255584717, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.360782623291016, + "code_tokens": 28, + "cvar95_kl": 12.333198547363281, + "delta_nll": 5.4780378341674805, + "max_kl": 14.396811485290527, + "mean_kl": 5.223334312438965, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.042294979095459, + "code_tokens": 71, + "cvar95_kl": 14.389984130859375, + "delta_nll": 5.563430309295654, + "max_kl": 18.109642028808594, + "mean_kl": 5.452578544616699, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.19718310236930847 + }, + { + "candidate_nll": 5.492650508880615, + "code_tokens": 24, + "cvar95_kl": 10.578119277954102, + "delta_nll": 4.963289737701416, + "max_kl": 10.969165802001953, + "mean_kl": 5.069170951843262, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.772335052490234, + "code_tokens": 34, + "cvar95_kl": 16.812768936157227, + "delta_nll": 6.500753879547119, + "max_kl": 16.960323333740234, + "mean_kl": 6.8001627922058105, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.14705882966518402 + }, + { + "candidate_nll": 8.015684127807617, + "code_tokens": 64, + "cvar95_kl": 15.63918685913086, + "delta_nll": 6.896389961242676, + "max_kl": 17.329498291015625, + "mean_kl": 7.275482177734375, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.09375 + }, + { + "candidate_nll": 5.596726417541504, + "code_tokens": 45, + "cvar95_kl": 13.222251892089844, + "delta_nll": 4.890135288238525, + "max_kl": 15.762255668640137, + "mean_kl": 4.96620512008667, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 7.302375316619873, + "code_tokens": 48, + "cvar95_kl": 15.638702392578125, + "delta_nll": 6.103671073913574, + "max_kl": 16.93924903869629, + "mean_kl": 6.553588390350342, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 6.0828423500061035, + "code_tokens": 21, + "cvar95_kl": 14.358531951904297, + "delta_nll": 5.382641315460205, + "max_kl": 14.571773529052734, + "mean_kl": 5.232615947723389, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 5.086331844329834, + "code_tokens": 24, + "cvar95_kl": 13.508039474487305, + "delta_nll": 4.331917762756348, + "max_kl": 14.54066276550293, + "mean_kl": 4.599956035614014, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.863462448120117, + "code_tokens": 63, + "cvar95_kl": 17.146827697753906, + "delta_nll": 6.291535377502441, + "max_kl": 18.826435089111328, + "mean_kl": 6.076934337615967, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 7.928468704223633, + "code_tokens": 22, + "cvar95_kl": 14.011526107788086, + "delta_nll": 6.649709701538086, + "max_kl": 14.051651954650879, + "mean_kl": 5.963674545288086, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 4.846866130828857, + "code_tokens": 25, + "cvar95_kl": 9.56362533569336, + "delta_nll": 3.7182037830352783, + "max_kl": 10.582528114318848, + "mean_kl": 4.186452865600586, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 5.025755405426025, + "code_tokens": 21, + "cvar95_kl": 9.149168968200684, + "delta_nll": 3.0654296875, + "max_kl": 10.952722549438477, + "mean_kl": 3.6250243186950684, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.814821720123291, + "code_tokens": 24, + "cvar95_kl": 10.398035049438477, + "delta_nll": 3.348489761352539, + "max_kl": 11.131999969482422, + "mean_kl": 3.832204580307007, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.771554946899414, + "code_tokens": 15, + "cvar95_kl": 10.377639770507812, + "delta_nll": 3.97806715965271, + "max_kl": 10.377639770507812, + "mean_kl": 4.590856075286865, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.4669671058654785, + "code_tokens": 59, + "cvar95_kl": 13.635093688964844, + "delta_nll": 5.686110496520996, + "max_kl": 14.767053604125977, + "mean_kl": 5.838822364807129, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.20338982343673706 + }, + { + "candidate_nll": 7.370120525360107, + "code_tokens": 54, + "cvar95_kl": 15.588007926940918, + "delta_nll": 6.436002254486084, + "max_kl": 15.920820236206055, + "mean_kl": 6.502050399780273, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.09259258955717087 + }, + { + "candidate_nll": 5.556395530700684, + "code_tokens": 28, + "cvar95_kl": 13.958757400512695, + "delta_nll": 3.604536533355713, + "max_kl": 14.591316223144531, + "mean_kl": 4.772505760192871, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 6.778261184692383, + "code_tokens": 46, + "cvar95_kl": 15.621968269348145, + "delta_nll": 5.693535327911377, + "max_kl": 17.493654251098633, + "mean_kl": 6.430947303771973, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 5.219844818115234, + "code_tokens": 27, + "cvar95_kl": 10.829151153564453, + "delta_nll": 3.7050223350524902, + "max_kl": 11.948349952697754, + "mean_kl": 4.625002861022949, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 8.938897132873535, + "code_tokens": 97, + "cvar95_kl": 17.225631713867188, + "delta_nll": 7.681466579437256, + "max_kl": 18.502750396728516, + "mean_kl": 7.747902870178223, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.07216494530439377 + }, + { + "candidate_nll": 6.176458358764648, + "code_tokens": 24, + "cvar95_kl": 12.100406646728516, + "delta_nll": 5.374226093292236, + "max_kl": 12.309867858886719, + "mean_kl": 5.13083028793335, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.5981903076171875, + "code_tokens": 30, + "cvar95_kl": 15.305464744567871, + "delta_nll": 6.333041667938232, + "max_kl": 15.430707931518555, + "mean_kl": 6.416407108306885, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.8747124671936035, + "code_tokens": 38, + "cvar95_kl": 13.418859481811523, + "delta_nll": 6.441642761230469, + "max_kl": 14.239228248596191, + "mean_kl": 6.6014628410339355, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 4.900537014007568, + "code_tokens": 21, + "cvar95_kl": 10.188032150268555, + "delta_nll": 3.4127578735351562, + "max_kl": 10.702964782714844, + "mean_kl": 3.6601340770721436, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.5035319328308105, + "code_tokens": 65, + "cvar95_kl": 15.054574966430664, + "delta_nll": 5.966569900512695, + "max_kl": 18.995073318481445, + "mean_kl": 6.0340094566345215, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.1230769231915474 + }, + { + "candidate_nll": 5.812273025512695, + "code_tokens": 27, + "cvar95_kl": 12.206243515014648, + "delta_nll": 5.027746677398682, + "max_kl": 13.227521896362305, + "mean_kl": 5.361629962921143, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 7.791317462921143, + "code_tokens": 35, + "cvar95_kl": 17.935081481933594, + "delta_nll": 7.067501544952393, + "max_kl": 18.34217643737793, + "mean_kl": 7.422238826751709, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.557102203369141, + "code_tokens": 75, + "cvar95_kl": 15.179966926574707, + "delta_nll": 5.632577896118164, + "max_kl": 16.296802520751953, + "mean_kl": 5.921302318572998, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.1066666692495346 + }, + { + "candidate_nll": 5.914217948913574, + "code_tokens": 25, + "cvar95_kl": 14.13585090637207, + "delta_nll": 4.272482395172119, + "max_kl": 16.50495719909668, + "mean_kl": 4.770137786865234, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 6.323394775390625, + "code_tokens": 52, + "cvar95_kl": 12.873138427734375, + "delta_nll": 5.3994059562683105, + "max_kl": 14.396800994873047, + "mean_kl": 5.5366339683532715, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.17307692766189575 + }, + { + "candidate_nll": 7.800520896911621, + "code_tokens": 110, + "cvar95_kl": 17.468528747558594, + "delta_nll": 7.189860820770264, + "max_kl": 18.459205627441406, + "mean_kl": 7.182217121124268, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.11818181723356247 + }, + { + "candidate_nll": 5.120651721954346, + "code_tokens": 25, + "cvar95_kl": 11.97669792175293, + "delta_nll": 3.8878297805786133, + "max_kl": 12.16628646850586, + "mean_kl": 4.122009754180908, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.43115758895874, + "code_tokens": 56, + "cvar95_kl": 16.572368621826172, + "delta_nll": 6.4141716957092285, + "max_kl": 17.282278060913086, + "mean_kl": 6.456520080566406, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 9.164886474609375, + "code_tokens": 108, + "cvar95_kl": 16.110130310058594, + "delta_nll": 7.6110382080078125, + "max_kl": 18.778684616088867, + "mean_kl": 7.446301460266113, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.12037037312984467 + }, + { + "candidate_nll": 6.112306118011475, + "code_tokens": 47, + "cvar95_kl": 15.035003662109375, + "delta_nll": 4.740566253662109, + "max_kl": 17.32859992980957, + "mean_kl": 5.314882755279541, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.1489361673593521 + }, + { + "candidate_nll": 7.261061668395996, + "code_tokens": 102, + "cvar95_kl": 16.64226722717285, + "delta_nll": 6.26265287399292, + "max_kl": 19.892963409423828, + "mean_kl": 6.536538600921631, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.19607843458652496 + }, + { + "candidate_nll": 5.239745616912842, + "code_tokens": 18, + "cvar95_kl": 10.894200325012207, + "delta_nll": 3.6141200065612793, + "max_kl": 10.894200325012207, + "mean_kl": 4.305347919464111, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.213761806488037, + "code_tokens": 28, + "cvar95_kl": 12.504292488098145, + "delta_nll": 6.114097595214844, + "max_kl": 12.5715970993042, + "mean_kl": 5.990342617034912, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 5.736509323120117, + "code_tokens": 30, + "cvar95_kl": 13.462663650512695, + "delta_nll": 4.193108558654785, + "max_kl": 15.640144348144531, + "mean_kl": 4.377007961273193, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.390140056610107, + "code_tokens": 48, + "cvar95_kl": 15.24341869354248, + "delta_nll": 5.915022850036621, + "max_kl": 15.653775215148926, + "mean_kl": 5.914939880371094, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.051579475402832, + "code_tokens": 59, + "cvar95_kl": 17.50609588623047, + "delta_nll": 6.514564514160156, + "max_kl": 20.485301971435547, + "mean_kl": 6.541595935821533, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.1525423675775528 + }, + { + "candidate_nll": 8.580992698669434, + "code_tokens": 102, + "cvar95_kl": 16.112329483032227, + "delta_nll": 7.538798809051514, + "max_kl": 17.94512367248535, + "mean_kl": 7.699957370758057, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.06862745434045792 + }, + { + "candidate_nll": 5.828360080718994, + "code_tokens": 36, + "cvar95_kl": 12.883910179138184, + "delta_nll": 5.087994575500488, + "max_kl": 13.18531322479248, + "mean_kl": 5.12930154800415, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 8.732449531555176, + "code_tokens": 51, + "cvar95_kl": 17.623380661010742, + "delta_nll": 7.843518257141113, + "max_kl": 18.49416160583496, + "mean_kl": 7.408244609832764, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 6.414745330810547, + "code_tokens": 62, + "cvar95_kl": 12.577508926391602, + "delta_nll": 5.169066429138184, + "max_kl": 13.100858688354492, + "mean_kl": 5.5659661293029785, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 5.913795471191406, + "code_tokens": 39, + "cvar95_kl": 13.838386535644531, + "delta_nll": 5.337513446807861, + "max_kl": 14.394287109375, + "mean_kl": 5.609561443328857, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.10256410390138626 + }, + { + "candidate_nll": 6.548534870147705, + "code_tokens": 76, + "cvar95_kl": 14.024591445922852, + "delta_nll": 5.808850288391113, + "max_kl": 15.208357810974121, + "mean_kl": 5.806898593902588, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.1315789520740509 + }, + { + "candidate_nll": 8.257987022399902, + "code_tokens": 58, + "cvar95_kl": 16.630159378051758, + "delta_nll": 6.753335952758789, + "max_kl": 18.979272842407227, + "mean_kl": 6.651447772979736, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 8.65462875366211, + "code_tokens": 93, + "cvar95_kl": 16.637081146240234, + "delta_nll": 7.527233600616455, + "max_kl": 20.31838607788086, + "mean_kl": 7.540098190307617, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.08602150529623032 + }, + { + "candidate_nll": 7.706447124481201, + "code_tokens": 45, + "cvar95_kl": 14.483113288879395, + "delta_nll": 6.35120153427124, + "max_kl": 18.324350357055664, + "mean_kl": 6.670519828796387, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 8.43494987487793, + "code_tokens": 209, + "cvar95_kl": 17.38965606689453, + "delta_nll": 7.877974510192871, + "max_kl": 19.7731876373291, + "mean_kl": 7.985987186431885, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.110047847032547 + }, + { + "candidate_nll": 8.679492950439453, + "code_tokens": 46, + "cvar95_kl": 15.21619701385498, + "delta_nll": 7.711025238037109, + "max_kl": 16.62545394897461, + "mean_kl": 7.225772380828857, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 7.209316253662109, + "code_tokens": 39, + "cvar95_kl": 17.742788314819336, + "delta_nll": 6.488529205322266, + "max_kl": 17.817384719848633, + "mean_kl": 6.254206657409668, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 5.038835048675537, + "code_tokens": 25, + "cvar95_kl": 10.509612083435059, + "delta_nll": 4.097139835357666, + "max_kl": 10.608065605163574, + "mean_kl": 4.1454081535339355, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.430180549621582, + "code_tokens": 28, + "cvar95_kl": 16.747032165527344, + "delta_nll": 5.254984378814697, + "max_kl": 18.79953384399414, + "mean_kl": 5.48136043548584, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 7.638055801391602, + "code_tokens": 84, + "cvar95_kl": 17.113096237182617, + "delta_nll": 6.384133338928223, + "max_kl": 21.81757926940918, + "mean_kl": 6.956610679626465, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.1547619104385376 + }, + { + "candidate_nll": 7.187884330749512, + "code_tokens": 34, + "cvar95_kl": 15.88741683959961, + "delta_nll": 6.375604152679443, + "max_kl": 16.925626754760742, + "mean_kl": 6.660923480987549, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 4.825079441070557, + "code_tokens": 28, + "cvar95_kl": 7.621490478515625, + "delta_nll": 3.5700323581695557, + "max_kl": 8.115614891052246, + "mean_kl": 3.6462838649749756, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 7.876516819000244, + "code_tokens": 92, + "cvar95_kl": 14.978248596191406, + "delta_nll": 6.7776713371276855, + "max_kl": 17.40444564819336, + "mean_kl": 6.94923210144043, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.1195652186870575 + }, + { + "candidate_nll": 8.99438190460205, + "code_tokens": 185, + "cvar95_kl": 18.072059631347656, + "delta_nll": 8.264266967773438, + "max_kl": 19.19498062133789, + "mean_kl": 8.21474838256836, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.0648648664355278 + }, + { + "candidate_nll": 7.12148380279541, + "code_tokens": 106, + "cvar95_kl": 17.127473831176758, + "delta_nll": 6.768489360809326, + "max_kl": 18.72142791748047, + "mean_kl": 6.911635398864746, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.14150942862033844 + }, + { + "candidate_nll": 8.3419771194458, + "code_tokens": 64, + "cvar95_kl": 14.377845764160156, + "delta_nll": 6.799422740936279, + "max_kl": 17.282651901245117, + "mean_kl": 6.777704238891602, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.09375 + }, + { + "candidate_nll": 6.785233020782471, + "code_tokens": 74, + "cvar95_kl": 16.2027645111084, + "delta_nll": 5.6744489669799805, + "max_kl": 20.101417541503906, + "mean_kl": 5.9676923751831055, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.13513512909412384 + }, + { + "candidate_nll": 5.470483303070068, + "code_tokens": 27, + "cvar95_kl": 11.292600631713867, + "delta_nll": 4.483079433441162, + "max_kl": 12.627168655395508, + "mean_kl": 4.921891212463379, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 8.738982200622559, + "code_tokens": 72, + "cvar95_kl": 19.336278915405273, + "delta_nll": 8.115144729614258, + "max_kl": 22.611217498779297, + "mean_kl": 8.22087574005127, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 7.548574447631836, + "code_tokens": 62, + "cvar95_kl": 13.661849021911621, + "delta_nll": 6.615272045135498, + "max_kl": 14.815098762512207, + "mean_kl": 6.346026420593262, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.12903225421905518 + }, + { + "candidate_nll": 6.911891937255859, + "code_tokens": 42, + "cvar95_kl": 14.507881164550781, + "delta_nll": 6.213725566864014, + "max_kl": 15.153106689453125, + "mean_kl": 6.226941108703613, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 9.764392852783203, + "code_tokens": 238, + "cvar95_kl": 18.516157150268555, + "delta_nll": 8.877059936523438, + "max_kl": 20.35807228088379, + "mean_kl": 8.93833065032959, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.06302521377801895 + }, + { + "candidate_nll": 7.222746849060059, + "code_tokens": 68, + "cvar95_kl": 14.918595314025879, + "delta_nll": 6.308235168457031, + "max_kl": 15.063772201538086, + "mean_kl": 6.321497440338135, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.14705882966518402 + }, + { + "candidate_nll": 7.179327964782715, + "code_tokens": 83, + "cvar95_kl": 15.674354553222656, + "delta_nll": 6.195178031921387, + "max_kl": 18.851346969604492, + "mean_kl": 6.402262210845947, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.1927710771560669 + }, + { + "candidate_nll": 8.540306091308594, + "code_tokens": 64, + "cvar95_kl": 17.324148178100586, + "delta_nll": 7.276538848876953, + "max_kl": 21.06527328491211, + "mean_kl": 7.651609420776367, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.37931489944458, + "code_tokens": 37, + "cvar95_kl": 15.01427936553955, + "delta_nll": 5.722515106201172, + "max_kl": 16.98827362060547, + "mean_kl": 5.727890968322754, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.13513512909412384 + }, + { + "candidate_nll": 6.814022064208984, + "code_tokens": 27, + "cvar95_kl": 11.837930679321289, + "delta_nll": 5.3040947914123535, + "max_kl": 13.230722427368164, + "mean_kl": 4.982956409454346, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.505108833312988, + "code_tokens": 40, + "cvar95_kl": 16.635581970214844, + "delta_nll": 5.613392353057861, + "max_kl": 19.263525009155273, + "mean_kl": 5.868790149688721, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.311047554016113, + "code_tokens": 28, + "cvar95_kl": 16.0626220703125, + "delta_nll": 6.31327486038208, + "max_kl": 16.982450485229492, + "mean_kl": 6.5884528160095215, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.52803897857666, + "code_tokens": 59, + "cvar95_kl": 12.692699432373047, + "delta_nll": 4.673409461975098, + "max_kl": 15.241018295288086, + "mean_kl": 4.809003829956055, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.2711864411830902 + }, + { + "candidate_nll": 6.5417160987854, + "code_tokens": 57, + "cvar95_kl": 13.315577507019043, + "delta_nll": 5.971715927124023, + "max_kl": 14.563706398010254, + "mean_kl": 5.876420974731445, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.17543859779834747 + }, + { + "candidate_nll": 7.211860179901123, + "code_tokens": 25, + "cvar95_kl": 16.175024032592773, + "delta_nll": 5.827919960021973, + "max_kl": 17.597049713134766, + "mean_kl": 5.76457405090332, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.246300220489502, + "code_tokens": 48, + "cvar95_kl": 15.121705055236816, + "delta_nll": 5.499147891998291, + "max_kl": 16.310564041137695, + "mean_kl": 5.87150764465332, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 7.182501316070557, + "code_tokens": 35, + "cvar95_kl": 14.371108055114746, + "delta_nll": 6.320725440979004, + "max_kl": 15.774658203125, + "mean_kl": 6.173677921295166, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.895477771759033, + "code_tokens": 30, + "cvar95_kl": 10.32376480102539, + "delta_nll": 4.250770568847656, + "max_kl": 10.950892448425293, + "mean_kl": 4.195818901062012, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.23333333432674408 + }, + { + "candidate_nll": 7.68110990524292, + "code_tokens": 49, + "cvar95_kl": 13.557181358337402, + "delta_nll": 6.558032512664795, + "max_kl": 14.382635116577148, + "mean_kl": 6.58962869644165, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.16326530277729034 + }, + { + "candidate_nll": 7.7593207359313965, + "code_tokens": 98, + "cvar95_kl": 15.846819877624512, + "delta_nll": 6.743103981018066, + "max_kl": 17.359466552734375, + "mean_kl": 6.983160495758057, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 4.6789469718933105, + "code_tokens": 25, + "cvar95_kl": 9.499639511108398, + "delta_nll": 3.298140525817871, + "max_kl": 10.645829200744629, + "mean_kl": 3.6075079441070557, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 6.032371997833252, + "code_tokens": 27, + "cvar95_kl": 17.616134643554688, + "delta_nll": 5.254648685455322, + "max_kl": 18.177783966064453, + "mean_kl": 5.371174335479736, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 4.369905948638916, + "code_tokens": 20, + "cvar95_kl": 10.441977500915527, + "delta_nll": 3.3248212337493896, + "max_kl": 10.441977500915527, + "mean_kl": 4.264588832855225, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 9.9822359085083, + "code_tokens": 253, + "cvar95_kl": 18.169784545898438, + "delta_nll": 9.289566040039062, + "max_kl": 22.081905364990234, + "mean_kl": 9.195657730102539, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.05928853899240494 + }, + { + "candidate_nll": 6.482024669647217, + "code_tokens": 55, + "cvar95_kl": 15.345934867858887, + "delta_nll": 5.652904510498047, + "max_kl": 20.039621353149414, + "mean_kl": 5.664531707763672, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.23636363446712494 + }, + { + "candidate_nll": 6.333980560302734, + "code_tokens": 31, + "cvar95_kl": 10.31614875793457, + "delta_nll": 5.1274824142456055, + "max_kl": 10.769848823547363, + "mean_kl": 5.1271138191223145, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 5.929033279418945, + "code_tokens": 35, + "cvar95_kl": 15.448509216308594, + "delta_nll": 5.22478723526001, + "max_kl": 15.535297393798828, + "mean_kl": 5.217130184173584, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.2571428716182709 + }, + { + "candidate_nll": 6.359667778015137, + "code_tokens": 61, + "cvar95_kl": 12.189936637878418, + "delta_nll": 5.556799411773682, + "max_kl": 13.540264129638672, + "mean_kl": 5.566935062408447, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.19672131538391113 + }, + { + "candidate_nll": 6.839692115783691, + "code_tokens": 55, + "cvar95_kl": 13.155415534973145, + "delta_nll": 5.884974479675293, + "max_kl": 14.254890441894531, + "mean_kl": 6.0974931716918945, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.1090909093618393 + }, + { + "candidate_nll": 8.277275085449219, + "code_tokens": 108, + "cvar95_kl": 15.839046478271484, + "delta_nll": 6.770308017730713, + "max_kl": 20.45664405822754, + "mean_kl": 6.849660396575928, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 6.224024295806885, + "code_tokens": 58, + "cvar95_kl": 11.289955139160156, + "delta_nll": 4.936717987060547, + "max_kl": 12.145856857299805, + "mean_kl": 4.9731669425964355, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.731349468231201, + "code_tokens": 91, + "cvar95_kl": 17.396732330322266, + "delta_nll": 6.903894424438477, + "max_kl": 18.941513061523438, + "mean_kl": 6.77350378036499, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.10989011079072952 + }, + { + "candidate_nll": 8.502848625183105, + "code_tokens": 105, + "cvar95_kl": 17.57210350036621, + "delta_nll": 7.739109039306641, + "max_kl": 24.752368927001953, + "mean_kl": 8.0310697555542, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.12380952388048172 + }, + { + "candidate_nll": 6.596067905426025, + "code_tokens": 49, + "cvar95_kl": 13.700627326965332, + "delta_nll": 5.94881534576416, + "max_kl": 15.207971572875977, + "mean_kl": 5.542850971221924, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.22448979318141937 + }, + { + "candidate_nll": 5.448166370391846, + "code_tokens": 27, + "cvar95_kl": 10.667242050170898, + "delta_nll": 3.9860033988952637, + "max_kl": 12.107257843017578, + "mean_kl": 4.145949363708496, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 6.224318027496338, + "code_tokens": 25, + "cvar95_kl": 13.913007736206055, + "delta_nll": 5.18394136428833, + "max_kl": 15.11604118347168, + "mean_kl": 5.1885085105896, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.11802864074707, + "code_tokens": 98, + "cvar95_kl": 18.632564544677734, + "delta_nll": 6.82297420501709, + "max_kl": 22.85083770751953, + "mean_kl": 7.045065879821777, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.0714285746216774 + }, + { + "candidate_nll": 5.22553014755249, + "code_tokens": 23, + "cvar95_kl": 10.783465385437012, + "delta_nll": 4.575748443603516, + "max_kl": 10.813262939453125, + "mean_kl": 4.406945705413818, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 6.142210483551025, + "code_tokens": 31, + "cvar95_kl": 14.380949974060059, + "delta_nll": 4.27683687210083, + "max_kl": 14.998478889465332, + "mean_kl": 5.339545249938965, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 6.780340194702148, + "code_tokens": 36, + "cvar95_kl": 12.821520805358887, + "delta_nll": 6.032247066497803, + "max_kl": 12.886221885681152, + "mean_kl": 5.607838153839111, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.1245927810668945, + "code_tokens": 38, + "cvar95_kl": 17.008880615234375, + "delta_nll": 5.690915107727051, + "max_kl": 18.961849212646484, + "mean_kl": 6.602321147918701, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.1315789520740509 + }, + { + "candidate_nll": 5.768430233001709, + "code_tokens": 25, + "cvar95_kl": 11.466361999511719, + "delta_nll": 4.16234016418457, + "max_kl": 13.829218864440918, + "mean_kl": 4.004995822906494, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 4.448824405670166, + "code_tokens": 26, + "cvar95_kl": 9.852022171020508, + "delta_nll": 3.5693752765655518, + "max_kl": 10.88119888305664, + "mean_kl": 3.523958921432495, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 6.695723533630371, + "code_tokens": 62, + "cvar95_kl": 14.97152042388916, + "delta_nll": 5.953281402587891, + "max_kl": 17.9086971282959, + "mean_kl": 5.9481940269470215, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.17741934955120087 + }, + { + "candidate_nll": 7.645513534545898, + "code_tokens": 36, + "cvar95_kl": 13.667764663696289, + "delta_nll": 5.906994342803955, + "max_kl": 13.976835250854492, + "mean_kl": 6.328986167907715, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 11.227980613708496, + "code_tokens": 119, + "cvar95_kl": 20.900482177734375, + "delta_nll": 9.897391319274902, + "max_kl": 23.28335952758789, + "mean_kl": 9.920970916748047, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.08403361588716507 + }, + { + "candidate_nll": 4.499349117279053, + "code_tokens": 37, + "cvar95_kl": 10.900077819824219, + "delta_nll": 4.186641693115234, + "max_kl": 11.087789535522461, + "mean_kl": 3.9038915634155273, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 6.95881986618042, + "code_tokens": 28, + "cvar95_kl": 13.669307708740234, + "delta_nll": 6.17608642578125, + "max_kl": 13.843303680419922, + "mean_kl": 6.45206880569458, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 8.869391441345215, + "code_tokens": 143, + "cvar95_kl": 17.025514602661133, + "delta_nll": 8.303044319152832, + "max_kl": 19.058120727539062, + "mean_kl": 8.439316749572754, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.0559440553188324 + }, + { + "candidate_nll": 7.800201892852783, + "code_tokens": 29, + "cvar95_kl": 13.855948448181152, + "delta_nll": 6.017895698547363, + "max_kl": 14.188302993774414, + "mean_kl": 6.718564987182617, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 5.972645282745361, + "code_tokens": 28, + "cvar95_kl": 13.56666374206543, + "delta_nll": 4.9001569747924805, + "max_kl": 16.035968780517578, + "mean_kl": 4.636019706726074, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 7.161046504974365, + "code_tokens": 30, + "cvar95_kl": 14.012748718261719, + "delta_nll": 5.897008895874023, + "max_kl": 14.442832946777344, + "mean_kl": 5.834114074707031, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 7.929535388946533, + "code_tokens": 85, + "cvar95_kl": 18.425378799438477, + "delta_nll": 7.507725238800049, + "max_kl": 20.195838928222656, + "mean_kl": 7.390705585479736, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.0941176488995552 + }, + { + "candidate_nll": 9.024380683898926, + "code_tokens": 44, + "cvar95_kl": 16.823211669921875, + "delta_nll": 8.047137260437012, + "max_kl": 18.442358016967773, + "mean_kl": 7.943384170532227, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.06818182021379471 + }, + { + "candidate_nll": 6.648914337158203, + "code_tokens": 19, + "cvar95_kl": 13.910045623779297, + "delta_nll": 5.844811916351318, + "max_kl": 13.910045623779297, + "mean_kl": 5.115932464599609, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 7.414926528930664, + "code_tokens": 113, + "cvar95_kl": 15.560555458068848, + "delta_nll": 6.67750358581543, + "max_kl": 17.887048721313477, + "mean_kl": 6.485905647277832, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.16814158856868744 + }, + { + "candidate_nll": 5.9278974533081055, + "code_tokens": 40, + "cvar95_kl": 12.349303245544434, + "delta_nll": 4.45270299911499, + "max_kl": 12.837043762207031, + "mean_kl": 5.000565528869629, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 8.12266731262207, + "code_tokens": 51, + "cvar95_kl": 17.692960739135742, + "delta_nll": 6.516007423400879, + "max_kl": 23.551660537719727, + "mean_kl": 7.041337966918945, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.1568627506494522 + }, + { + "candidate_nll": 6.759530544281006, + "code_tokens": 55, + "cvar95_kl": 14.923659324645996, + "delta_nll": 5.178498268127441, + "max_kl": 15.464252471923828, + "mean_kl": 5.630533695220947, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 5.21919059753418, + "code_tokens": 27, + "cvar95_kl": 11.629581451416016, + "delta_nll": 4.089122295379639, + "max_kl": 12.954608917236328, + "mean_kl": 3.875945568084717, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 6.439744472503662, + "code_tokens": 32, + "cvar95_kl": 12.800460815429688, + "delta_nll": 5.59993839263916, + "max_kl": 13.598000526428223, + "mean_kl": 5.642062664031982, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.15625 + }, + { + "candidate_nll": 7.785398006439209, + "code_tokens": 64, + "cvar95_kl": 14.615548133850098, + "delta_nll": 7.13286018371582, + "max_kl": 16.39180564880371, + "mean_kl": 6.646759033203125, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.203125 + }, + { + "candidate_nll": 9.080845832824707, + "code_tokens": 35, + "cvar95_kl": 15.447818756103516, + "delta_nll": 8.0311918258667, + "max_kl": 16.39253807067871, + "mean_kl": 7.702136993408203, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.08571428805589676 + }, + { + "candidate_nll": 9.322888374328613, + "code_tokens": 152, + "cvar95_kl": 20.29839324951172, + "delta_nll": 8.871040344238281, + "max_kl": 26.18801498413086, + "mean_kl": 8.770977973937988, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.07236842066049576 + }, + { + "candidate_nll": 6.907682418823242, + "code_tokens": 77, + "cvar95_kl": 15.888350486755371, + "delta_nll": 5.646347999572754, + "max_kl": 21.808700561523438, + "mean_kl": 5.770454406738281, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.15584415197372437 + }, + { + "candidate_nll": 7.490671157836914, + "code_tokens": 77, + "cvar95_kl": 15.736324310302734, + "delta_nll": 6.922389984130859, + "max_kl": 17.79918098449707, + "mean_kl": 6.852675437927246, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.11688311398029327 + }, + { + "candidate_nll": 5.5519609451293945, + "code_tokens": 71, + "cvar95_kl": 14.793689727783203, + "delta_nll": 4.989871978759766, + "max_kl": 18.8124942779541, + "mean_kl": 5.021596908569336, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.2112676054239273 + }, + { + "candidate_nll": 7.867068767547607, + "code_tokens": 112, + "cvar95_kl": 16.7785587310791, + "delta_nll": 6.988689422607422, + "max_kl": 18.646934509277344, + "mean_kl": 6.918522834777832, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 8.638422012329102, + "code_tokens": 111, + "cvar95_kl": 18.534950256347656, + "delta_nll": 7.91851806640625, + "max_kl": 22.2772159576416, + "mean_kl": 7.960037708282471, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.07207207381725311 + }, + { + "candidate_nll": 5.9749860763549805, + "code_tokens": 62, + "cvar95_kl": 12.889986991882324, + "delta_nll": 5.187089920043945, + "max_kl": 13.866026878356934, + "mean_kl": 5.244415760040283, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.14516128599643707 + }, + { + "candidate_nll": 5.4741010665893555, + "code_tokens": 52, + "cvar95_kl": 13.840248107910156, + "delta_nll": 4.863521099090576, + "max_kl": 14.667696952819824, + "mean_kl": 4.974940776824951, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 6.160619735717773, + "code_tokens": 24, + "cvar95_kl": 11.49755859375, + "delta_nll": 4.265393257141113, + "max_kl": 13.409760475158691, + "mean_kl": 4.634641170501709, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.269798278808594, + "code_tokens": 48, + "cvar95_kl": 14.31173038482666, + "delta_nll": 5.378028392791748, + "max_kl": 15.96700382232666, + "mean_kl": 5.495912551879883, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.645552635192871, + "code_tokens": 98, + "cvar95_kl": 18.09518051147461, + "delta_nll": 6.595259666442871, + "max_kl": 21.234630584716797, + "mean_kl": 6.967278957366943, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.13265305757522583 + }, + { + "candidate_nll": 6.972604751586914, + "code_tokens": 48, + "cvar95_kl": 15.651473045349121, + "delta_nll": 5.979918956756592, + "max_kl": 19.74375343322754, + "mean_kl": 5.978766918182373, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 6.9195051193237305, + "code_tokens": 56, + "cvar95_kl": 15.460597038269043, + "delta_nll": 5.973509788513184, + "max_kl": 20.798690795898438, + "mean_kl": 6.291633605957031, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.223229885101318, + "code_tokens": 88, + "cvar95_kl": 16.296972274780273, + "delta_nll": 6.621953010559082, + "max_kl": 17.63336944580078, + "mean_kl": 6.471543788909912, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.1931818127632141 + }, + { + "candidate_nll": 8.255773544311523, + "code_tokens": 50, + "cvar95_kl": 15.693114280700684, + "delta_nll": 6.622243404388428, + "max_kl": 17.674583435058594, + "mean_kl": 6.613385200500488, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 6.722866058349609, + "code_tokens": 85, + "cvar95_kl": 16.85113525390625, + "delta_nll": 6.285879135131836, + "max_kl": 19.461755752563477, + "mean_kl": 6.088037967681885, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.1411764770746231 + }, + { + "candidate_nll": 5.845196723937988, + "code_tokens": 27, + "cvar95_kl": 10.406210899353027, + "delta_nll": 4.770983695983887, + "max_kl": 11.043562889099121, + "mean_kl": 4.897475719451904, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 8.470585823059082, + "code_tokens": 48, + "cvar95_kl": 20.714691162109375, + "delta_nll": 7.352967262268066, + "max_kl": 22.000654220581055, + "mean_kl": 7.630359649658203, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 5.989881992340088, + "code_tokens": 29, + "cvar95_kl": 10.567008018493652, + "delta_nll": 4.342034816741943, + "max_kl": 10.696300506591797, + "mean_kl": 5.351672649383545, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.834328651428223, + "code_tokens": 63, + "cvar95_kl": 19.762996673583984, + "delta_nll": 6.896113395690918, + "max_kl": 21.685874938964844, + "mean_kl": 7.577517986297607, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.341114044189453, + "code_tokens": 89, + "cvar95_kl": 15.837076187133789, + "delta_nll": 5.666055202484131, + "max_kl": 18.775287628173828, + "mean_kl": 5.839078426361084, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.17977528274059296 + }, + { + "candidate_nll": 10.101509094238281, + "code_tokens": 133, + "cvar95_kl": 20.08378791809082, + "delta_nll": 8.757230758666992, + "max_kl": 22.48053550720215, + "mean_kl": 8.891549110412598, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.12030075490474701 + }, + { + "candidate_nll": 5.2312726974487305, + "code_tokens": 25, + "cvar95_kl": 10.81533432006836, + "delta_nll": 4.282735824584961, + "max_kl": 12.395101547241211, + "mean_kl": 4.42136287689209, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 5.4075164794921875, + "code_tokens": 33, + "cvar95_kl": 11.795366287231445, + "delta_nll": 4.370216369628906, + "max_kl": 12.826933860778809, + "mean_kl": 4.9432806968688965, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 7.112778663635254, + "code_tokens": 93, + "cvar95_kl": 18.55746841430664, + "delta_nll": 6.356003761291504, + "max_kl": 22.2000675201416, + "mean_kl": 6.266803741455078, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.12903225421905518 + }, + { + "candidate_nll": 8.141539573669434, + "code_tokens": 101, + "cvar95_kl": 15.278285026550293, + "delta_nll": 7.173140048980713, + "max_kl": 17.03759002685547, + "mean_kl": 7.380816459655762, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.09900990128517151 + }, + { + "candidate_nll": 7.6642632484436035, + "code_tokens": 152, + "cvar95_kl": 16.56952476501465, + "delta_nll": 7.0995001792907715, + "max_kl": 20.019712448120117, + "mean_kl": 7.221053123474121, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.1184210553765297 + }, + { + "candidate_nll": 5.997351169586182, + "code_tokens": 33, + "cvar95_kl": 14.177846908569336, + "delta_nll": 4.596479415893555, + "max_kl": 16.443218231201172, + "mean_kl": 4.917084217071533, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 6.500733852386475, + "code_tokens": 29, + "cvar95_kl": 13.59922981262207, + "delta_nll": 5.214271068572998, + "max_kl": 16.152273178100586, + "mean_kl": 5.244920253753662, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 7.848512649536133, + "code_tokens": 27, + "cvar95_kl": 14.177190780639648, + "delta_nll": 6.390442848205566, + "max_kl": 15.575974464416504, + "mean_kl": 6.45961332321167, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 4.651499271392822, + "code_tokens": 33, + "cvar95_kl": 10.991683006286621, + "delta_nll": 4.341459274291992, + "max_kl": 11.177067756652832, + "mean_kl": 4.122121334075928, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 5.902487277984619, + "code_tokens": 27, + "cvar95_kl": 12.155679702758789, + "delta_nll": 5.106089115142822, + "max_kl": 13.074275016784668, + "mean_kl": 5.162576198577881, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 6.65779972076416, + "code_tokens": 84, + "cvar95_kl": 16.971397399902344, + "delta_nll": 5.953906059265137, + "max_kl": 17.81419563293457, + "mean_kl": 5.992504119873047, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 8.420004844665527, + "code_tokens": 103, + "cvar95_kl": 15.815312385559082, + "delta_nll": 7.374535083770752, + "max_kl": 16.51258659362793, + "mean_kl": 7.1687421798706055, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.10679611563682556 + }, + { + "candidate_nll": 7.179447650909424, + "code_tokens": 88, + "cvar95_kl": 14.39306640625, + "delta_nll": 6.2631001472473145, + "max_kl": 16.493675231933594, + "mean_kl": 6.102416515350342, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 9.600324630737305, + "code_tokens": 62, + "cvar95_kl": 17.240436553955078, + "delta_nll": 8.291394233703613, + "max_kl": 18.426298141479492, + "mean_kl": 8.284754753112793, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.11290322244167328 + }, + { + "candidate_nll": 5.636438369750977, + "code_tokens": 24, + "cvar95_kl": 13.568007469177246, + "delta_nll": 4.099729537963867, + "max_kl": 18.066354751586914, + "mean_kl": 4.122101306915283, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 5.783626556396484, + "code_tokens": 34, + "cvar95_kl": 10.162670135498047, + "delta_nll": 4.445174217224121, + "max_kl": 11.31076717376709, + "mean_kl": 4.536136627197266, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.77593994140625, + "code_tokens": 43, + "cvar95_kl": 14.325434684753418, + "delta_nll": 6.056194305419922, + "max_kl": 14.879423141479492, + "mean_kl": 5.890496730804443, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.1627907007932663 + }, + { + "candidate_nll": 9.436283111572266, + "code_tokens": 54, + "cvar95_kl": 16.793264389038086, + "delta_nll": 7.946977615356445, + "max_kl": 17.366168975830078, + "mean_kl": 8.045686721801758, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.12962962687015533 + }, + { + "candidate_nll": 5.931309700012207, + "code_tokens": 50, + "cvar95_kl": 13.238760948181152, + "delta_nll": 5.2014241218566895, + "max_kl": 16.770490646362305, + "mean_kl": 5.471024036407471, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 9.288484573364258, + "code_tokens": 235, + "cvar95_kl": 16.942646026611328, + "delta_nll": 8.530974388122559, + "max_kl": 18.011537551879883, + "mean_kl": 8.571817398071289, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.08936170488595963 + }, + { + "candidate_nll": 7.067055702209473, + "code_tokens": 33, + "cvar95_kl": 12.855844497680664, + "delta_nll": 5.038400650024414, + "max_kl": 13.229280471801758, + "mean_kl": 5.433363437652588, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 7.554831504821777, + "code_tokens": 45, + "cvar95_kl": 16.928438186645508, + "delta_nll": 6.568328857421875, + "max_kl": 18.155658721923828, + "mean_kl": 6.616612434387207, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.15555556118488312 + }, + { + "candidate_nll": 6.833436012268066, + "code_tokens": 28, + "cvar95_kl": 14.410512924194336, + "delta_nll": 6.210475921630859, + "max_kl": 14.47724437713623, + "mean_kl": 6.04376745223999, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 6.389471054077148, + "code_tokens": 49, + "cvar95_kl": 13.250988960266113, + "delta_nll": 5.711206436157227, + "max_kl": 15.588364601135254, + "mean_kl": 5.653984069824219, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.20408163964748383 + }, + { + "candidate_nll": 10.30508041381836, + "code_tokens": 107, + "cvar95_kl": 20.59326934814453, + "delta_nll": 10.055665016174316, + "max_kl": 23.337066650390625, + "mean_kl": 9.875467300415039, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.08411215245723724 + }, + { + "candidate_nll": 7.208099842071533, + "code_tokens": 48, + "cvar95_kl": 13.789859771728516, + "delta_nll": 5.987189769744873, + "max_kl": 15.360906600952148, + "mean_kl": 6.028657913208008, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 5.584551811218262, + "code_tokens": 40, + "cvar95_kl": 11.53369140625, + "delta_nll": 3.7450108528137207, + "max_kl": 11.641191482543945, + "mean_kl": 4.58064079284668, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 7.4042205810546875, + "code_tokens": 46, + "cvar95_kl": 15.229316711425781, + "delta_nll": 6.934736728668213, + "max_kl": 15.817511558532715, + "mean_kl": 6.971096515655518, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.15217390656471252 + }, + { + "candidate_nll": 6.6265082359313965, + "code_tokens": 31, + "cvar95_kl": 13.878485679626465, + "delta_nll": 5.58405876159668, + "max_kl": 14.33198356628418, + "mean_kl": 5.4495978355407715, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 7.010890483856201, + "code_tokens": 58, + "cvar95_kl": 16.021787643432617, + "delta_nll": 6.5944061279296875, + "max_kl": 18.28314781188965, + "mean_kl": 6.412586688995361, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 7.2595534324646, + "code_tokens": 49, + "cvar95_kl": 16.343244552612305, + "delta_nll": 6.141783714294434, + "max_kl": 16.871828079223633, + "mean_kl": 6.362412929534912, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.18367347121238708 + }, + { + "candidate_nll": 3.4845666885375977, + "code_tokens": 18, + "cvar95_kl": 7.830827236175537, + "delta_nll": 2.537973165512085, + "max_kl": 7.830827236175537, + "mean_kl": 2.7935361862182617, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 4.6804680824279785, + "code_tokens": 28, + "cvar95_kl": 11.9856595993042, + "delta_nll": 3.521056652069092, + "max_kl": 15.131917953491211, + "mean_kl": 3.984785556793213, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 7.425696849822998, + "code_tokens": 57, + "cvar95_kl": 17.484952926635742, + "delta_nll": 6.683629989624023, + "max_kl": 19.1612548828125, + "mean_kl": 6.629397392272949, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.12280701845884323 + }, + { + "candidate_nll": 8.84069538116455, + "code_tokens": 88, + "cvar95_kl": 18.415348052978516, + "delta_nll": 8.140585899353027, + "max_kl": 19.938819885253906, + "mean_kl": 8.327432632446289, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.10227272659540176 + }, + { + "candidate_nll": 6.895727634429932, + "code_tokens": 23, + "cvar95_kl": 12.953763961791992, + "delta_nll": 5.696940898895264, + "max_kl": 14.853610038757324, + "mean_kl": 6.085006237030029, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 6.784426212310791, + "code_tokens": 40, + "cvar95_kl": 17.546199798583984, + "delta_nll": 5.524969100952148, + "max_kl": 17.64912986755371, + "mean_kl": 6.017966270446777, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 5.114195823669434, + "code_tokens": 20, + "cvar95_kl": 14.339384078979492, + "delta_nll": 3.9529807567596436, + "max_kl": 14.339384078979492, + "mean_kl": 4.19483757019043, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.510035991668701, + "code_tokens": 63, + "cvar95_kl": 12.5385160446167, + "delta_nll": 5.494250297546387, + "max_kl": 15.357608795166016, + "mean_kl": 5.51970911026001, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 4.625622272491455, + "code_tokens": 39, + "cvar95_kl": 8.252402305603027, + "delta_nll": 3.672940969467163, + "max_kl": 8.992707252502441, + "mean_kl": 3.7964513301849365, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 8.252378463745117, + "code_tokens": 50, + "cvar95_kl": 15.868512153625488, + "delta_nll": 6.840513706207275, + "max_kl": 19.080984115600586, + "mean_kl": 6.935285568237305, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.07999999821186066 + }, + { + "candidate_nll": 6.902853012084961, + "code_tokens": 63, + "cvar95_kl": 15.640195846557617, + "delta_nll": 5.935375690460205, + "max_kl": 17.747350692749023, + "mean_kl": 6.521917343139648, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.1587301641702652 + }, + { + "candidate_nll": 8.712340354919434, + "code_tokens": 46, + "cvar95_kl": 19.437395095825195, + "delta_nll": 7.739778518676758, + "max_kl": 21.833450317382812, + "mean_kl": 7.487475395202637, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 6.664908409118652, + "code_tokens": 25, + "cvar95_kl": 15.190550804138184, + "delta_nll": 5.986013889312744, + "max_kl": 15.906623840332031, + "mean_kl": 5.753461837768555, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 7.773139953613281, + "code_tokens": 111, + "cvar95_kl": 15.589542388916016, + "delta_nll": 6.958874225616455, + "max_kl": 17.7369441986084, + "mean_kl": 6.986261367797852, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.12612612545490265 + }, + { + "candidate_nll": 6.176669120788574, + "code_tokens": 71, + "cvar95_kl": 13.054933547973633, + "delta_nll": 5.448111534118652, + "max_kl": 14.224143028259277, + "mean_kl": 5.541344165802002, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.11267605423927307 + }, + { + "candidate_nll": 7.317685604095459, + "code_tokens": 28, + "cvar95_kl": 12.038721084594727, + "delta_nll": 5.946961879730225, + "max_kl": 12.948813438415527, + "mean_kl": 5.896607875823975, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 6.861156463623047, + "code_tokens": 25, + "cvar95_kl": 12.771480560302734, + "delta_nll": 5.660700798034668, + "max_kl": 13.81649112701416, + "mean_kl": 5.582521915435791, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.553812980651855, + "code_tokens": 40, + "cvar95_kl": 17.353443145751953, + "delta_nll": 7.251041412353516, + "max_kl": 18.664867401123047, + "mean_kl": 7.5082597732543945, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 8.326626777648926, + "code_tokens": 177, + "cvar95_kl": 18.021677017211914, + "delta_nll": 7.282182693481445, + "max_kl": 21.9379940032959, + "mean_kl": 8.042093276977539, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.09604519605636597 + }, + { + "candidate_nll": 8.20163345336914, + "code_tokens": 51, + "cvar95_kl": 15.66223430633545, + "delta_nll": 7.11045503616333, + "max_kl": 16.05200958251953, + "mean_kl": 7.332179069519043, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.19607843458652496 + }, + { + "candidate_nll": 7.99001407623291, + "code_tokens": 69, + "cvar95_kl": 15.523186683654785, + "delta_nll": 7.535553455352783, + "max_kl": 16.592897415161133, + "mean_kl": 7.527052879333496, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.14492753148078918 + }, + { + "candidate_nll": 4.069450378417969, + "code_tokens": 26, + "cvar95_kl": 10.756540298461914, + "delta_nll": 3.321190357208252, + "max_kl": 14.462248802185059, + "mean_kl": 3.5556089878082275, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.698151588439941, + "code_tokens": 24, + "cvar95_kl": 10.00900650024414, + "delta_nll": 3.179812431335449, + "max_kl": 10.093698501586914, + "mean_kl": 3.5282251834869385, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 6.930079936981201, + "code_tokens": 15, + "cvar95_kl": 17.760242462158203, + "delta_nll": 5.779603958129883, + "max_kl": 17.760242462158203, + "mean_kl": 5.682565212249756, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 8.110429763793945, + "code_tokens": 43, + "cvar95_kl": 23.17325782775879, + "delta_nll": 6.961080551147461, + "max_kl": 30.46355438232422, + "mean_kl": 7.841249465942383, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.20930232107639313 + }, + { + "candidate_nll": 5.24425745010376, + "code_tokens": 18, + "cvar95_kl": 9.56009578704834, + "delta_nll": 4.1519317626953125, + "max_kl": 9.56009578704834, + "mean_kl": 4.027898788452148, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.7430949211120605, + "code_tokens": 49, + "cvar95_kl": 15.115547180175781, + "delta_nll": 6.015101432800293, + "max_kl": 15.668261528015137, + "mean_kl": 5.6433820724487305, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.22448979318141937 + }, + { + "candidate_nll": 7.56740140914917, + "code_tokens": 59, + "cvar95_kl": 17.451147079467773, + "delta_nll": 6.699645519256592, + "max_kl": 17.839183807373047, + "mean_kl": 6.900061130523682, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.1355932205915451 + }, + { + "candidate_nll": 7.481951713562012, + "code_tokens": 221, + "cvar95_kl": 15.159939765930176, + "delta_nll": 6.896685600280762, + "max_kl": 18.21139907836914, + "mean_kl": 7.0761637687683105, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.11312217265367508 + }, + { + "candidate_nll": 5.686654567718506, + "code_tokens": 25, + "cvar95_kl": 13.942217826843262, + "delta_nll": 4.484781265258789, + "max_kl": 14.421857833862305, + "mean_kl": 4.332006454467773, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 6.63933801651001, + "code_tokens": 36, + "cvar95_kl": 12.748212814331055, + "delta_nll": 5.915417671203613, + "max_kl": 12.952657699584961, + "mean_kl": 5.935183048248291, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 9.95650577545166, + "code_tokens": 158, + "cvar95_kl": 19.79081153869629, + "delta_nll": 9.312360763549805, + "max_kl": 20.667810440063477, + "mean_kl": 9.390172004699707, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.037974681705236435 + }, + { + "candidate_nll": 9.14405632019043, + "code_tokens": 92, + "cvar95_kl": 17.9633846282959, + "delta_nll": 7.828521251678467, + "max_kl": 19.339508056640625, + "mean_kl": 7.683929443359375, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.10869564861059189 + }, + { + "candidate_nll": 10.466630935668945, + "code_tokens": 102, + "cvar95_kl": 17.726058959960938, + "delta_nll": 9.55258846282959, + "max_kl": 19.31338119506836, + "mean_kl": 9.568127632141113, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.03921568766236305 + }, + { + "candidate_nll": 8.757177352905273, + "code_tokens": 171, + "cvar95_kl": 17.93651008605957, + "delta_nll": 8.147207260131836, + "max_kl": 19.839874267578125, + "mean_kl": 8.214503288269043, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.07017543911933899 + }, + { + "candidate_nll": 7.587628364562988, + "code_tokens": 40, + "cvar95_kl": 15.178832054138184, + "delta_nll": 6.3827643394470215, + "max_kl": 15.480307579040527, + "mean_kl": 7.1449103355407715, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.496194362640381, + "code_tokens": 48, + "cvar95_kl": 14.976641654968262, + "delta_nll": 6.260180473327637, + "max_kl": 16.195972442626953, + "mean_kl": 6.523040771484375, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 6.995415210723877, + "code_tokens": 24, + "cvar95_kl": 14.816816329956055, + "delta_nll": 5.3904948234558105, + "max_kl": 16.230127334594727, + "mean_kl": 5.908309459686279, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.7836995124816895, + "code_tokens": 48, + "cvar95_kl": 18.971445083618164, + "delta_nll": 6.796595573425293, + "max_kl": 22.84503173828125, + "mean_kl": 6.716524600982666, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 7.17112922668457, + "code_tokens": 25, + "cvar95_kl": 14.506488800048828, + "delta_nll": 5.78424596786499, + "max_kl": 17.094669342041016, + "mean_kl": 5.70901346206665, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.1092376708984375, + "code_tokens": 30, + "cvar95_kl": 11.596831321716309, + "delta_nll": 4.610492706298828, + "max_kl": 12.27322006225586, + "mean_kl": 4.368741035461426, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.501333713531494, + "code_tokens": 19, + "cvar95_kl": 12.461047172546387, + "delta_nll": 4.0745158195495605, + "max_kl": 12.461047172546387, + "mean_kl": 4.3469414710998535, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 3.480036973953247, + "code_tokens": 12, + "cvar95_kl": 7.6913065910339355, + "delta_nll": 2.4009644985198975, + "max_kl": 7.6913065910339355, + "mean_kl": 2.997282028198242, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.813887119293213, + "code_tokens": 37, + "cvar95_kl": 18.456462860107422, + "delta_nll": 6.473577499389648, + "max_kl": 19.49114418029785, + "mean_kl": 6.574635982513428, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 4.882472038269043, + "code_tokens": 33, + "cvar95_kl": 14.748077392578125, + "delta_nll": 3.677042245864868, + "max_kl": 16.225311279296875, + "mean_kl": 4.470540523529053, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 8.419113159179688, + "code_tokens": 94, + "cvar95_kl": 17.22762107849121, + "delta_nll": 7.56700325012207, + "max_kl": 21.906606674194336, + "mean_kl": 7.6225996017456055, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.13829787075519562 + }, + { + "candidate_nll": 7.990223407745361, + "code_tokens": 90, + "cvar95_kl": 16.826717376708984, + "delta_nll": 7.3654584884643555, + "max_kl": 20.929412841796875, + "mean_kl": 7.391547203063965, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 7.876465320587158, + "code_tokens": 47, + "cvar95_kl": 13.798367500305176, + "delta_nll": 6.428701400756836, + "max_kl": 15.129559516906738, + "mean_kl": 6.51858377456665, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.10638298094272614 + }, + { + "candidate_nll": 7.643665790557861, + "code_tokens": 61, + "cvar95_kl": 18.177139282226562, + "delta_nll": 6.719831466674805, + "max_kl": 20.046655654907227, + "mean_kl": 6.645212650299072, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.1803278625011444 + }, + { + "candidate_nll": 6.486293792724609, + "code_tokens": 39, + "cvar95_kl": 14.483570098876953, + "delta_nll": 5.262779235839844, + "max_kl": 17.509138107299805, + "mean_kl": 5.783594131469727, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 9.185606956481934, + "code_tokens": 152, + "cvar95_kl": 17.58617401123047, + "delta_nll": 8.25344467163086, + "max_kl": 20.396841049194336, + "mean_kl": 8.840598106384277, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.06578947603702545 + }, + { + "candidate_nll": 7.158775806427002, + "code_tokens": 100, + "cvar95_kl": 16.203519821166992, + "delta_nll": 6.189793586730957, + "max_kl": 18.155447006225586, + "mean_kl": 6.452398300170898, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 8.200955390930176, + "code_tokens": 89, + "cvar95_kl": 16.609237670898438, + "delta_nll": 7.5800275802612305, + "max_kl": 17.537399291992188, + "mean_kl": 7.611789703369141, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.08988764137029648 + }, + { + "candidate_nll": 5.1730194091796875, + "code_tokens": 40, + "cvar95_kl": 13.053426742553711, + "delta_nll": 3.86201810836792, + "max_kl": 14.964747428894043, + "mean_kl": 4.816222190856934, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 7.41151762008667, + "code_tokens": 76, + "cvar95_kl": 17.91535758972168, + "delta_nll": 6.7099504470825195, + "max_kl": 20.632925033569336, + "mean_kl": 6.79542875289917, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.17105263471603394 + }, + { + "candidate_nll": 6.539393901824951, + "code_tokens": 56, + "cvar95_kl": 12.984715461730957, + "delta_nll": 5.290406703948975, + "max_kl": 13.745094299316406, + "mean_kl": 5.91682767868042, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.930926322937012, + "code_tokens": 34, + "cvar95_kl": 17.372638702392578, + "delta_nll": 6.441495895385742, + "max_kl": 19.970806121826172, + "mean_kl": 6.441230773925781, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.0882352963089943 + }, + { + "candidate_nll": 9.357779502868652, + "code_tokens": 96, + "cvar95_kl": 17.9891300201416, + "delta_nll": 8.679668426513672, + "max_kl": 20.138734817504883, + "mean_kl": 8.660279273986816, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.1354166716337204 + }, + { + "candidate_nll": 8.94335651397705, + "code_tokens": 451, + "cvar95_kl": 17.83321189880371, + "delta_nll": 8.244661331176758, + "max_kl": 21.028715133666992, + "mean_kl": 8.2435941696167, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.05321507900953293 + }, + { + "candidate_nll": 6.6350202560424805, + "code_tokens": 37, + "cvar95_kl": 15.073501586914062, + "delta_nll": 5.022710800170898, + "max_kl": 15.331327438354492, + "mean_kl": 5.757560729980469, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 7.39089822769165, + "code_tokens": 42, + "cvar95_kl": 18.2423152923584, + "delta_nll": 6.149300575256348, + "max_kl": 19.003337860107422, + "mean_kl": 6.573614120483398, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 7.678212642669678, + "code_tokens": 33, + "cvar95_kl": 14.978504180908203, + "delta_nll": 6.66145658493042, + "max_kl": 15.095561027526855, + "mean_kl": 6.280180931091309, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.12121212482452393 + }, + { + "candidate_nll": 6.938271999359131, + "code_tokens": 43, + "cvar95_kl": 13.439679145812988, + "delta_nll": 6.01222038269043, + "max_kl": 15.314666748046875, + "mean_kl": 5.903386116027832, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.1627907007932663 + }, + { + "candidate_nll": 6.161199569702148, + "code_tokens": 73, + "cvar95_kl": 12.154244422912598, + "delta_nll": 5.424763202667236, + "max_kl": 13.866451263427734, + "mean_kl": 5.4989752769470215, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.19178082048892975 + }, + { + "candidate_nll": 5.053415775299072, + "code_tokens": 15, + "cvar95_kl": 14.599608421325684, + "delta_nll": 4.140939712524414, + "max_kl": 14.599608421325684, + "mean_kl": 3.8959193229675293, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.838046073913574, + "code_tokens": 34, + "cvar95_kl": 14.13016128540039, + "delta_nll": 5.57896614074707, + "max_kl": 14.645975112915039, + "mean_kl": 6.326904296875, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 8.403180122375488, + "code_tokens": 136, + "cvar95_kl": 16.426469802856445, + "delta_nll": 7.294696807861328, + "max_kl": 19.009437561035156, + "mean_kl": 7.204148292541504, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 4.546454429626465, + "code_tokens": 17, + "cvar95_kl": 11.092239379882812, + "delta_nll": 3.3002982139587402, + "max_kl": 11.092239379882812, + "mean_kl": 3.6824207305908203, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.187948226928711, + "code_tokens": 33, + "cvar95_kl": 12.988594055175781, + "delta_nll": 5.394893646240234, + "max_kl": 13.13505744934082, + "mean_kl": 5.616434574127197, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 5.974043846130371, + "code_tokens": 38, + "cvar95_kl": 12.639822959899902, + "delta_nll": 5.13159704208374, + "max_kl": 12.886316299438477, + "mean_kl": 4.98399543762207, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 5.908222198486328, + "code_tokens": 61, + "cvar95_kl": 13.441299438476562, + "delta_nll": 5.510697364807129, + "max_kl": 16.85961151123047, + "mean_kl": 5.527454376220703, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.21311475336551666 + }, + { + "candidate_nll": 7.937803745269775, + "code_tokens": 143, + "cvar95_kl": 15.887185096740723, + "delta_nll": 7.202084064483643, + "max_kl": 18.940948486328125, + "mean_kl": 7.381783962249756, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.1258741319179535 + }, + { + "candidate_nll": 7.074193477630615, + "code_tokens": 41, + "cvar95_kl": 17.310338973999023, + "delta_nll": 5.9851274490356445, + "max_kl": 18.499446868896484, + "mean_kl": 6.175234794616699, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.24390244483947754 + }, + { + "candidate_nll": 7.947854042053223, + "code_tokens": 67, + "cvar95_kl": 16.46123695373535, + "delta_nll": 7.58317232131958, + "max_kl": 19.312028884887695, + "mean_kl": 7.6421098709106445, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.13432836532592773 + }, + { + "candidate_nll": 6.954115867614746, + "code_tokens": 76, + "cvar95_kl": 13.979516983032227, + "delta_nll": 5.56385612487793, + "max_kl": 15.037158012390137, + "mean_kl": 5.9574294090271, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.1184210553765297 + }, + { + "candidate_nll": 7.178423881530762, + "code_tokens": 137, + "cvar95_kl": 15.9537992477417, + "delta_nll": 6.542571067810059, + "max_kl": 19.115190505981445, + "mean_kl": 6.491316318511963, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.14598539471626282 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2340", + "rounding": "stochastic", + "seed": 2340, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 10.314386367797852, + "median": 5.91074275970459, + "minimum": 2.144712209701538, + "q25": 5.055020570755005, + "q75": 6.795321702957153 + }, + "task_macro": { + "candidate_nll": 6.944392992019654, + "delta_nll": 5.930346412181854, + "mean_kl": 6.040665551662445, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.17865012399852276 + }, + "token_weighted": { + "candidate_nll": 7.576940059661865, + "cvar95_kl": 17.038928985595703, + "delta_nll": 6.660102367401123, + "max_kl": 30.46355438232422, + "mean_kl": 6.748311996459961, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.14538420736789703 + } + }, + "read_risk_l0_stochastic_seed_2341": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.378480144500733, + "macro_mean_kl": 4.5017442617416386, + "macro_top1_agreement": 0.2913162990808487, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 5.373108943939209, + "macro_mean_kl": 5.517089412689209, + "macro_top1_agreement": 0.2143056882619858, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 6.061836971282959, + "macro_mean_kl": 6.1226287422180175, + "macro_top1_agreement": 0.1670765461921692, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 7.34224199295044, + "macro_mean_kl": 7.375692317962646, + "macro_top1_agreement": 0.11220430266857147, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 7.8644280433654785, + "code_tokens": 98, + "cvar95_kl": 15.953160285949707, + "delta_nll": 7.137572288513184, + "max_kl": 18.069507598876953, + "mean_kl": 7.425797939300537, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 4.274013519287109, + "code_tokens": 19, + "cvar95_kl": 11.231156349182129, + "delta_nll": 3.114600419998169, + "max_kl": 11.231156349182129, + "mean_kl": 3.2550506591796875, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.5263158082962036 + }, + { + "candidate_nll": 6.096091270446777, + "code_tokens": 39, + "cvar95_kl": 15.214167594909668, + "delta_nll": 5.080741882324219, + "max_kl": 18.170093536376953, + "mean_kl": 5.456188678741455, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.25641027092933655 + }, + { + "candidate_nll": 5.585590362548828, + "code_tokens": 24, + "cvar95_kl": 10.113508224487305, + "delta_nll": 4.13930082321167, + "max_kl": 11.652104377746582, + "mean_kl": 4.367478847503662, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.387155055999756, + "code_tokens": 25, + "cvar95_kl": 12.178691864013672, + "delta_nll": 3.9793968200683594, + "max_kl": 12.725420951843262, + "mean_kl": 4.249690055847168, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.252253532409668, + "code_tokens": 55, + "cvar95_kl": 16.400548934936523, + "delta_nll": 6.35664176940918, + "max_kl": 19.468353271484375, + "mean_kl": 6.300553798675537, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.4256181716918945, + "code_tokens": 16, + "cvar95_kl": 16.867860794067383, + "delta_nll": 4.596177577972412, + "max_kl": 16.867860794067383, + "mean_kl": 4.574938774108887, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 10.97270679473877, + "code_tokens": 199, + "cvar95_kl": 20.201446533203125, + "delta_nll": 9.750813484191895, + "max_kl": 26.386524200439453, + "mean_kl": 9.614314079284668, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.06532663106918335 + }, + { + "candidate_nll": 6.6656999588012695, + "code_tokens": 28, + "cvar95_kl": 12.010416984558105, + "delta_nll": 5.099621772766113, + "max_kl": 13.462864875793457, + "mean_kl": 4.946407318115234, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.965374946594238, + "code_tokens": 87, + "cvar95_kl": 13.850422859191895, + "delta_nll": 6.092392444610596, + "max_kl": 15.158967018127441, + "mean_kl": 6.100453853607178, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.1149425283074379 + }, + { + "candidate_nll": 7.003138065338135, + "code_tokens": 35, + "cvar95_kl": 14.049370765686035, + "delta_nll": 5.770174026489258, + "max_kl": 14.529891967773438, + "mean_kl": 5.890683174133301, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.536870956420898, + "code_tokens": 62, + "cvar95_kl": 15.328444480895996, + "delta_nll": 6.897579193115234, + "max_kl": 15.90411376953125, + "mean_kl": 6.988208293914795, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.20967741310596466 + }, + { + "candidate_nll": 7.243923664093018, + "code_tokens": 58, + "cvar95_kl": 15.08755111694336, + "delta_nll": 6.138865947723389, + "max_kl": 19.70987892150879, + "mean_kl": 5.744325637817383, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 5.541332721710205, + "code_tokens": 79, + "cvar95_kl": 13.674612998962402, + "delta_nll": 4.382284641265869, + "max_kl": 17.146333694458008, + "mean_kl": 4.61063289642334, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.2278480976819992 + }, + { + "candidate_nll": 6.630873680114746, + "code_tokens": 58, + "cvar95_kl": 13.509181022644043, + "delta_nll": 5.537754058837891, + "max_kl": 14.617079734802246, + "mean_kl": 5.504669189453125, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 6.7472310066223145, + "code_tokens": 48, + "cvar95_kl": 14.083956718444824, + "delta_nll": 5.585033416748047, + "max_kl": 14.439610481262207, + "mean_kl": 5.92909049987793, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.141430854797363, + "code_tokens": 37, + "cvar95_kl": 16.117151260375977, + "delta_nll": 6.015998363494873, + "max_kl": 19.647573471069336, + "mean_kl": 5.995073318481445, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 6.503205299377441, + "code_tokens": 61, + "cvar95_kl": 16.37367820739746, + "delta_nll": 5.795798301696777, + "max_kl": 22.7978572845459, + "mean_kl": 5.601651191711426, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.16393442451953888 + }, + { + "candidate_nll": 6.3771467208862305, + "code_tokens": 77, + "cvar95_kl": 13.344128608703613, + "delta_nll": 5.60742712020874, + "max_kl": 16.207061767578125, + "mean_kl": 5.514313697814941, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.16883116960525513 + }, + { + "candidate_nll": 8.555076599121094, + "code_tokens": 98, + "cvar95_kl": 17.558916091918945, + "delta_nll": 7.5333733558654785, + "max_kl": 21.017940521240234, + "mean_kl": 7.714203357696533, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.0714285746216774 + }, + { + "candidate_nll": 8.435455322265625, + "code_tokens": 162, + "cvar95_kl": 16.722932815551758, + "delta_nll": 7.850304126739502, + "max_kl": 19.27519416809082, + "mean_kl": 7.969330787658691, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.06790123134851456 + }, + { + "candidate_nll": 8.113425254821777, + "code_tokens": 103, + "cvar95_kl": 17.210054397583008, + "delta_nll": 7.368959426879883, + "max_kl": 19.923656463623047, + "mean_kl": 7.465916156768799, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.09708737581968307 + }, + { + "candidate_nll": 7.0468430519104, + "code_tokens": 73, + "cvar95_kl": 12.586736679077148, + "delta_nll": 5.727872848510742, + "max_kl": 14.675745964050293, + "mean_kl": 5.529871463775635, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.1780821979045868 + }, + { + "candidate_nll": 7.033238887786865, + "code_tokens": 119, + "cvar95_kl": 16.314786911010742, + "delta_nll": 6.050265312194824, + "max_kl": 19.86404800415039, + "mean_kl": 6.184088706970215, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.1260504275560379 + }, + { + "candidate_nll": 4.109518527984619, + "code_tokens": 17, + "cvar95_kl": 13.282827377319336, + "delta_nll": 2.9199953079223633, + "max_kl": 13.282827377319336, + "mean_kl": 3.1197054386138916, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.529411792755127 + }, + { + "candidate_nll": 7.320526123046875, + "code_tokens": 56, + "cvar95_kl": 14.135458946228027, + "delta_nll": 5.894824504852295, + "max_kl": 17.100109100341797, + "mean_kl": 5.934932231903076, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.1607142835855484 + }, + { + "candidate_nll": 7.084149360656738, + "code_tokens": 57, + "cvar95_kl": 16.72643280029297, + "delta_nll": 6.616788864135742, + "max_kl": 17.65268898010254, + "mean_kl": 6.53778600692749, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.12280701845884323 + }, + { + "candidate_nll": 6.916807651519775, + "code_tokens": 63, + "cvar95_kl": 13.240188598632812, + "delta_nll": 6.284062385559082, + "max_kl": 14.535679817199707, + "mean_kl": 6.288628578186035, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 10.202853202819824, + "code_tokens": 200, + "cvar95_kl": 20.66899299621582, + "delta_nll": 9.364917755126953, + "max_kl": 25.875585556030273, + "mean_kl": 9.234315872192383, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.10499999672174454 + }, + { + "candidate_nll": 5.854452610015869, + "code_tokens": 30, + "cvar95_kl": 11.023900032043457, + "delta_nll": 4.770249843597412, + "max_kl": 11.917362213134766, + "mean_kl": 4.4611711502075195, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 6.839295864105225, + "code_tokens": 28, + "cvar95_kl": 16.429723739624023, + "delta_nll": 6.222944736480713, + "max_kl": 19.957422256469727, + "mean_kl": 6.012977123260498, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 7.188386917114258, + "code_tokens": 26, + "cvar95_kl": 14.227447509765625, + "delta_nll": 6.417430400848389, + "max_kl": 16.198415756225586, + "mean_kl": 6.213310718536377, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 8.271114349365234, + "code_tokens": 54, + "cvar95_kl": 15.638733863830566, + "delta_nll": 7.438172817230225, + "max_kl": 16.496484756469727, + "mean_kl": 7.371747016906738, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.654916286468506, + "code_tokens": 47, + "cvar95_kl": 15.911253929138184, + "delta_nll": 6.698180198669434, + "max_kl": 17.240036010742188, + "mean_kl": 6.887961387634277, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.21276596188545227 + }, + { + "candidate_nll": 8.106271743774414, + "code_tokens": 93, + "cvar95_kl": 14.8720064163208, + "delta_nll": 7.371742248535156, + "max_kl": 16.8441162109375, + "mean_kl": 7.2129740715026855, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.10752688348293304 + }, + { + "candidate_nll": 5.8537917137146, + "code_tokens": 30, + "cvar95_kl": 12.373466491699219, + "delta_nll": 4.994417190551758, + "max_kl": 12.80834674835205, + "mean_kl": 4.979757785797119, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.363938331604004, + "code_tokens": 89, + "cvar95_kl": 12.982147216796875, + "delta_nll": 5.295434951782227, + "max_kl": 14.226325035095215, + "mean_kl": 5.279111385345459, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.12359550595283508 + }, + { + "candidate_nll": 6.425664901733398, + "code_tokens": 70, + "cvar95_kl": 14.558073997497559, + "delta_nll": 5.461415767669678, + "max_kl": 15.498983383178711, + "mean_kl": 5.714221954345703, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.24285714328289032 + }, + { + "candidate_nll": 6.041818618774414, + "code_tokens": 26, + "cvar95_kl": 16.296367645263672, + "delta_nll": 4.428591728210449, + "max_kl": 19.668779373168945, + "mean_kl": 5.1706695556640625, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 7.081886291503906, + "code_tokens": 48, + "cvar95_kl": 14.058484077453613, + "delta_nll": 6.124179363250732, + "max_kl": 14.471619606018066, + "mean_kl": 6.052972793579102, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.565234184265137, + "code_tokens": 29, + "cvar95_kl": 10.162246704101562, + "delta_nll": 3.6960861682891846, + "max_kl": 11.350055694580078, + "mean_kl": 4.008163928985596, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 6.2376861572265625, + "code_tokens": 17, + "cvar95_kl": 10.65038776397705, + "delta_nll": 5.124523162841797, + "max_kl": 10.65038776397705, + "mean_kl": 4.995710372924805, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 7.385968208312988, + "code_tokens": 37, + "cvar95_kl": 17.172996520996094, + "delta_nll": 6.465752124786377, + "max_kl": 17.725629806518555, + "mean_kl": 6.3121657371521, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 6.263501167297363, + "code_tokens": 121, + "cvar95_kl": 15.543364524841309, + "delta_nll": 5.752099514007568, + "max_kl": 18.96817970275879, + "mean_kl": 5.742403984069824, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 6.111554145812988, + "code_tokens": 30, + "cvar95_kl": 12.871377944946289, + "delta_nll": 4.712710380554199, + "max_kl": 13.390666007995605, + "mean_kl": 5.129148483276367, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.477039337158203, + "code_tokens": 74, + "cvar95_kl": 16.516813278198242, + "delta_nll": 6.5785675048828125, + "max_kl": 20.331707000732422, + "mean_kl": 6.460587978363037, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 6.69320011138916, + "code_tokens": 53, + "cvar95_kl": 13.63248062133789, + "delta_nll": 5.858776092529297, + "max_kl": 14.5950288772583, + "mean_kl": 5.5935893058776855, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.2075471729040146 + }, + { + "candidate_nll": 6.079195976257324, + "code_tokens": 21, + "cvar95_kl": 11.001964569091797, + "delta_nll": 4.219799995422363, + "max_kl": 11.184830665588379, + "mean_kl": 4.589574337005615, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.935141086578369, + "code_tokens": 20, + "cvar95_kl": 10.93572998046875, + "delta_nll": 2.5727274417877197, + "max_kl": 10.93572998046875, + "mean_kl": 2.8914132118225098, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 7.817383766174316, + "code_tokens": 122, + "cvar95_kl": 17.109661102294922, + "delta_nll": 6.998385429382324, + "max_kl": 20.835508346557617, + "mean_kl": 7.376047134399414, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.0901639312505722 + }, + { + "candidate_nll": 7.218478679656982, + "code_tokens": 94, + "cvar95_kl": 13.852163314819336, + "delta_nll": 6.378338813781738, + "max_kl": 16.117517471313477, + "mean_kl": 6.408645153045654, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.13829787075519562 + }, + { + "candidate_nll": 3.5800654888153076, + "code_tokens": 12, + "cvar95_kl": 6.690055847167969, + "delta_nll": 2.514763832092285, + "max_kl": 6.690055847167969, + "mean_kl": 2.3396780490875244, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 6.498111724853516, + "code_tokens": 38, + "cvar95_kl": 12.092196464538574, + "delta_nll": 4.991711616516113, + "max_kl": 12.156574249267578, + "mean_kl": 5.469985485076904, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.349135398864746, + "code_tokens": 58, + "cvar95_kl": 17.343698501586914, + "delta_nll": 6.444348335266113, + "max_kl": 21.04039192199707, + "mean_kl": 6.16832971572876, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 8.01515007019043, + "code_tokens": 54, + "cvar95_kl": 15.302926063537598, + "delta_nll": 7.418886184692383, + "max_kl": 15.831031799316406, + "mean_kl": 7.160953998565674, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 5.425070762634277, + "code_tokens": 40, + "cvar95_kl": 12.442071914672852, + "delta_nll": 4.493530750274658, + "max_kl": 13.083504676818848, + "mean_kl": 4.531851768493652, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 7.239525318145752, + "code_tokens": 115, + "cvar95_kl": 13.78244686126709, + "delta_nll": 6.7269792556762695, + "max_kl": 14.664430618286133, + "mean_kl": 6.535130977630615, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.13913042843341827 + }, + { + "candidate_nll": 6.694847583770752, + "code_tokens": 60, + "cvar95_kl": 16.191387176513672, + "delta_nll": 6.170233726501465, + "max_kl": 17.378711700439453, + "mean_kl": 6.239512920379639, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.796583652496338, + "code_tokens": 124, + "cvar95_kl": 15.381441116333008, + "delta_nll": 7.026596546173096, + "max_kl": 18.948348999023438, + "mean_kl": 7.151911735534668, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.15322580933570862 + }, + { + "candidate_nll": 8.560880661010742, + "code_tokens": 91, + "cvar95_kl": 18.588516235351562, + "delta_nll": 7.79013204574585, + "max_kl": 22.888593673706055, + "mean_kl": 7.728568077087402, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.10989011079072952 + }, + { + "candidate_nll": 8.482603073120117, + "code_tokens": 125, + "cvar95_kl": 17.583194732666016, + "delta_nll": 7.662685871124268, + "max_kl": 21.20150375366211, + "mean_kl": 7.7856035232543945, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.10400000214576721 + }, + { + "candidate_nll": 5.090405464172363, + "code_tokens": 26, + "cvar95_kl": 10.123775482177734, + "delta_nll": 3.95452880859375, + "max_kl": 11.018282890319824, + "mean_kl": 3.9128506183624268, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 7.489429950714111, + "code_tokens": 24, + "cvar95_kl": 10.939863204956055, + "delta_nll": 5.24900484085083, + "max_kl": 12.361918449401855, + "mean_kl": 5.219531059265137, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 9.556541442871094, + "code_tokens": 180, + "cvar95_kl": 20.001583099365234, + "delta_nll": 8.702359199523926, + "max_kl": 23.799936294555664, + "mean_kl": 8.72722053527832, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 7.220541477203369, + "code_tokens": 40, + "cvar95_kl": 13.291269302368164, + "delta_nll": 5.845732688903809, + "max_kl": 13.928694725036621, + "mean_kl": 6.335854530334473, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.444513320922852, + "code_tokens": 72, + "cvar95_kl": 15.094672203063965, + "delta_nll": 5.469516277313232, + "max_kl": 18.09993553161621, + "mean_kl": 5.767693519592285, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 5.0570149421691895, + "code_tokens": 20, + "cvar95_kl": 13.389511108398438, + "delta_nll": 3.4794437885284424, + "max_kl": 13.389511108398438, + "mean_kl": 4.018343448638916, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 6.429698944091797, + "code_tokens": 76, + "cvar95_kl": 13.080979347229004, + "delta_nll": 5.439054489135742, + "max_kl": 13.76192569732666, + "mean_kl": 5.5224127769470215, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 7.370669364929199, + "code_tokens": 42, + "cvar95_kl": 13.97656536102295, + "delta_nll": 5.998928070068359, + "max_kl": 15.270158767700195, + "mean_kl": 5.677011013031006, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.67180871963501, + "code_tokens": 28, + "cvar95_kl": 10.566402435302734, + "delta_nll": 4.088585376739502, + "max_kl": 11.296002388000488, + "mean_kl": 3.9744484424591064, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.873215198516846, + "code_tokens": 58, + "cvar95_kl": 15.908611297607422, + "delta_nll": 5.839738368988037, + "max_kl": 18.225610733032227, + "mean_kl": 5.913068771362305, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 6.293164253234863, + "code_tokens": 25, + "cvar95_kl": 15.54391098022461, + "delta_nll": 5.4245147705078125, + "max_kl": 17.37750244140625, + "mean_kl": 5.586015701293945, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 7.519137859344482, + "code_tokens": 88, + "cvar95_kl": 15.40185832977295, + "delta_nll": 6.5541300773620605, + "max_kl": 16.30663299560547, + "mean_kl": 6.55354118347168, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.14772726595401764 + }, + { + "candidate_nll": 6.585968017578125, + "code_tokens": 42, + "cvar95_kl": 12.987994194030762, + "delta_nll": 5.458450794219971, + "max_kl": 14.249271392822266, + "mean_kl": 5.271246433258057, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 7.06173849105835, + "code_tokens": 24, + "cvar95_kl": 16.676136016845703, + "delta_nll": 6.2421875, + "max_kl": 17.505163192749023, + "mean_kl": 5.836732864379883, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.472192287445068, + "code_tokens": 24, + "cvar95_kl": 11.490671157836914, + "delta_nll": 3.787320852279663, + "max_kl": 14.159164428710938, + "mean_kl": 3.784616708755493, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.480797290802002, + "code_tokens": 43, + "cvar95_kl": 17.40677833557129, + "delta_nll": 6.413618087768555, + "max_kl": 18.804691314697266, + "mean_kl": 6.3386335372924805, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.1860465109348297 + }, + { + "candidate_nll": 3.990063428878784, + "code_tokens": 24, + "cvar95_kl": 8.735013961791992, + "delta_nll": 3.1038436889648438, + "max_kl": 8.822525978088379, + "mean_kl": 3.18257212638855, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.88037371635437, + "code_tokens": 14, + "cvar95_kl": 6.159151077270508, + "delta_nll": 1.5677480697631836, + "max_kl": 6.159151077270508, + "mean_kl": 1.9987298250198364, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 5.808995723724365, + "code_tokens": 41, + "cvar95_kl": 11.444302558898926, + "delta_nll": 5.317952632904053, + "max_kl": 12.223978042602539, + "mean_kl": 4.957873821258545, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.26829269528388977 + }, + { + "candidate_nll": 6.194431304931641, + "code_tokens": 32, + "cvar95_kl": 12.707841873168945, + "delta_nll": 5.206091403961182, + "max_kl": 13.002233505249023, + "mean_kl": 5.5733962059021, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 6.906732559204102, + "code_tokens": 53, + "cvar95_kl": 16.041379928588867, + "delta_nll": 6.145037651062012, + "max_kl": 19.166996002197266, + "mean_kl": 6.141238212585449, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.18867924809455872 + }, + { + "candidate_nll": 7.090978145599365, + "code_tokens": 51, + "cvar95_kl": 16.1820125579834, + "delta_nll": 6.712687015533447, + "max_kl": 17.295230865478516, + "mean_kl": 6.439662456512451, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.1568627506494522 + }, + { + "candidate_nll": 7.000752925872803, + "code_tokens": 36, + "cvar95_kl": 15.90015697479248, + "delta_nll": 5.308597564697266, + "max_kl": 16.999170303344727, + "mean_kl": 5.891681671142578, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 5.45490026473999, + "code_tokens": 24, + "cvar95_kl": 12.410028457641602, + "delta_nll": 3.8941478729248047, + "max_kl": 13.518098831176758, + "mean_kl": 4.56447696685791, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.748479843139648, + "code_tokens": 39, + "cvar95_kl": 14.270819664001465, + "delta_nll": 4.434145450592041, + "max_kl": 14.959564208984375, + "mean_kl": 5.166928291320801, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 6.974781036376953, + "code_tokens": 84, + "cvar95_kl": 15.49882984161377, + "delta_nll": 6.276368618011475, + "max_kl": 18.49595832824707, + "mean_kl": 6.424752235412598, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.1547619104385376 + }, + { + "candidate_nll": 6.238772869110107, + "code_tokens": 32, + "cvar95_kl": 14.095817565917969, + "delta_nll": 5.316004753112793, + "max_kl": 16.545793533325195, + "mean_kl": 5.089946269989014, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 5.367755889892578, + "code_tokens": 19, + "cvar95_kl": 11.556982040405273, + "delta_nll": 4.057592391967773, + "max_kl": 11.556982040405273, + "mean_kl": 3.834777355194092, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 6.601963043212891, + "code_tokens": 50, + "cvar95_kl": 15.808484077453613, + "delta_nll": 5.7839860916137695, + "max_kl": 16.73843002319336, + "mean_kl": 5.549462795257568, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 7.7392802238464355, + "code_tokens": 71, + "cvar95_kl": 16.66897964477539, + "delta_nll": 7.239466667175293, + "max_kl": 18.784809112548828, + "mean_kl": 7.1841721534729, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.1690140813589096 + }, + { + "candidate_nll": 7.335849761962891, + "code_tokens": 28, + "cvar95_kl": 16.71482276916504, + "delta_nll": 5.94788122177124, + "max_kl": 19.774669647216797, + "mean_kl": 6.239905834197998, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.3819780349731445, + "code_tokens": 76, + "cvar95_kl": 16.354673385620117, + "delta_nll": 6.512716770172119, + "max_kl": 17.176729202270508, + "mean_kl": 6.529090404510498, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.17105263471603394 + }, + { + "candidate_nll": 6.2171783447265625, + "code_tokens": 34, + "cvar95_kl": 12.605292320251465, + "delta_nll": 5.2545928955078125, + "max_kl": 12.724678993225098, + "mean_kl": 5.219692230224609, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 5.157953262329102, + "code_tokens": 11, + "cvar95_kl": 10.40308666229248, + "delta_nll": 3.010561466217041, + "max_kl": 10.40308666229248, + "mean_kl": 2.552518606185913, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 6.566460609436035, + "code_tokens": 31, + "cvar95_kl": 13.333516120910645, + "delta_nll": 5.150193214416504, + "max_kl": 14.911808013916016, + "mean_kl": 5.795379161834717, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 7.51524543762207, + "code_tokens": 118, + "cvar95_kl": 16.4262638092041, + "delta_nll": 6.668469429016113, + "max_kl": 19.919025421142578, + "mean_kl": 6.643702030181885, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.11016949266195297 + }, + { + "candidate_nll": 7.411120414733887, + "code_tokens": 60, + "cvar95_kl": 17.482749938964844, + "delta_nll": 6.635424613952637, + "max_kl": 18.830303192138672, + "mean_kl": 6.605514049530029, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.689809799194336, + "code_tokens": 47, + "cvar95_kl": 15.037007331848145, + "delta_nll": 6.7943596839904785, + "max_kl": 16.791967391967773, + "mean_kl": 6.514269828796387, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.1489361673593521 + }, + { + "candidate_nll": 8.108209609985352, + "code_tokens": 84, + "cvar95_kl": 15.557047843933105, + "delta_nll": 6.95619535446167, + "max_kl": 16.61133575439453, + "mean_kl": 7.369405269622803, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.152880668640137, + "code_tokens": 27, + "cvar95_kl": 13.301837921142578, + "delta_nll": 5.1588544845581055, + "max_kl": 13.33131217956543, + "mean_kl": 5.216986179351807, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 6.2657999992370605, + "code_tokens": 19, + "cvar95_kl": 13.965258598327637, + "delta_nll": 4.653892993927002, + "max_kl": 13.965258598327637, + "mean_kl": 4.48818826675415, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 7.472015380859375, + "code_tokens": 114, + "cvar95_kl": 15.634907722473145, + "delta_nll": 6.699002742767334, + "max_kl": 17.625791549682617, + "mean_kl": 7.060536861419678, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.07017543911933899 + }, + { + "candidate_nll": 7.291451454162598, + "code_tokens": 40, + "cvar95_kl": 16.4527645111084, + "delta_nll": 5.563558101654053, + "max_kl": 17.498659133911133, + "mean_kl": 5.795880317687988, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.304757118225098, + "code_tokens": 25, + "cvar95_kl": 12.450100898742676, + "delta_nll": 4.8190789222717285, + "max_kl": 12.757699966430664, + "mean_kl": 4.561700820922852, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 5.683879852294922, + "code_tokens": 23, + "cvar95_kl": 11.693649291992188, + "delta_nll": 4.913190841674805, + "max_kl": 12.011481285095215, + "mean_kl": 4.577056407928467, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 8.14244556427002, + "code_tokens": 77, + "cvar95_kl": 15.768040657043457, + "delta_nll": 7.126095771789551, + "max_kl": 16.3115234375, + "mean_kl": 7.066882610321045, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.985586643218994, + "code_tokens": 19, + "cvar95_kl": 16.313758850097656, + "delta_nll": 3.8779211044311523, + "max_kl": 16.313758850097656, + "mean_kl": 3.906806468963623, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 5.566734790802002, + "code_tokens": 37, + "cvar95_kl": 13.347123146057129, + "delta_nll": 4.58349084854126, + "max_kl": 13.659524917602539, + "mean_kl": 5.207223892211914, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 6.576502323150635, + "code_tokens": 33, + "cvar95_kl": 13.932290077209473, + "delta_nll": 5.193337440490723, + "max_kl": 15.83558177947998, + "mean_kl": 5.326735973358154, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 6.04339075088501, + "code_tokens": 95, + "cvar95_kl": 13.408607482910156, + "delta_nll": 5.383549213409424, + "max_kl": 15.259807586669922, + "mean_kl": 5.361382007598877, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 8.007200241088867, + "code_tokens": 125, + "cvar95_kl": 16.219858169555664, + "delta_nll": 6.994040012359619, + "max_kl": 18.094528198242188, + "mean_kl": 6.968021869659424, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.08799999952316284 + }, + { + "candidate_nll": 9.015948295593262, + "code_tokens": 168, + "cvar95_kl": 18.23637580871582, + "delta_nll": 8.302900314331055, + "max_kl": 19.29694366455078, + "mean_kl": 8.227910041809082, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.1071428582072258 + }, + { + "candidate_nll": 4.992420673370361, + "code_tokens": 31, + "cvar95_kl": 13.856060028076172, + "delta_nll": 3.677018642425537, + "max_kl": 14.184587478637695, + "mean_kl": 4.19223690032959, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 7.7954607009887695, + "code_tokens": 83, + "cvar95_kl": 16.0338134765625, + "delta_nll": 7.033753871917725, + "max_kl": 19.241971969604492, + "mean_kl": 6.9072442054748535, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.1204819306731224 + }, + { + "candidate_nll": 6.175511360168457, + "code_tokens": 50, + "cvar95_kl": 13.670931816101074, + "delta_nll": 5.475568771362305, + "max_kl": 14.630927085876465, + "mean_kl": 5.246484279632568, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 7.318321228027344, + "code_tokens": 69, + "cvar95_kl": 16.758785247802734, + "delta_nll": 6.951139450073242, + "max_kl": 26.44484519958496, + "mean_kl": 6.804355144500732, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.14492753148078918 + }, + { + "candidate_nll": 6.875690937042236, + "code_tokens": 50, + "cvar95_kl": 14.65756607055664, + "delta_nll": 5.790794849395752, + "max_kl": 14.927480697631836, + "mean_kl": 5.740048885345459, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 9.561309814453125, + "code_tokens": 161, + "cvar95_kl": 17.540437698364258, + "delta_nll": 8.557528495788574, + "max_kl": 20.393178939819336, + "mean_kl": 8.389383316040039, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.10559006035327911 + }, + { + "candidate_nll": 5.939314365386963, + "code_tokens": 53, + "cvar95_kl": 13.296576499938965, + "delta_nll": 5.283538341522217, + "max_kl": 13.416056632995605, + "mean_kl": 5.443920612335205, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.2641509473323822 + }, + { + "candidate_nll": 9.555769920349121, + "code_tokens": 97, + "cvar95_kl": 16.475971221923828, + "delta_nll": 9.110241889953613, + "max_kl": 17.687469482421875, + "mean_kl": 8.806982040405273, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.09278350323438644 + }, + { + "candidate_nll": 4.601040840148926, + "code_tokens": 22, + "cvar95_kl": 11.264168739318848, + "delta_nll": 3.2872278690338135, + "max_kl": 11.812883377075195, + "mean_kl": 3.6806833744049072, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 5.857928276062012, + "code_tokens": 34, + "cvar95_kl": 15.23739242553711, + "delta_nll": 4.9890241622924805, + "max_kl": 16.64924430847168, + "mean_kl": 5.1001715660095215, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 7.230011463165283, + "code_tokens": 81, + "cvar95_kl": 14.642265319824219, + "delta_nll": 6.371368408203125, + "max_kl": 15.221118927001953, + "mean_kl": 6.6563825607299805, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.07407407462596893 + }, + { + "candidate_nll": 4.130829811096191, + "code_tokens": 17, + "cvar95_kl": 10.851322174072266, + "delta_nll": 2.8687264919281006, + "max_kl": 10.851322174072266, + "mean_kl": 2.7048473358154297, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.529411792755127 + }, + { + "candidate_nll": 7.437968730926514, + "code_tokens": 181, + "cvar95_kl": 17.449142456054688, + "delta_nll": 6.612711429595947, + "max_kl": 19.18889617919922, + "mean_kl": 6.775224685668945, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.09944751113653183 + }, + { + "candidate_nll": 5.9007062911987305, + "code_tokens": 63, + "cvar95_kl": 12.721214294433594, + "delta_nll": 4.920297622680664, + "max_kl": 14.946910858154297, + "mean_kl": 4.999602794647217, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 5.675720691680908, + "code_tokens": 35, + "cvar95_kl": 14.580004692077637, + "delta_nll": 4.9821977615356445, + "max_kl": 16.63846206665039, + "mean_kl": 4.958596706390381, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 4.242765426635742, + "code_tokens": 25, + "cvar95_kl": 8.090002059936523, + "delta_nll": 3.4379780292510986, + "max_kl": 10.231537818908691, + "mean_kl": 3.1810669898986816, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 7.505487442016602, + "code_tokens": 60, + "cvar95_kl": 16.092893600463867, + "delta_nll": 6.417121887207031, + "max_kl": 16.518695831298828, + "mean_kl": 6.642117500305176, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 8.162610054016113, + "code_tokens": 86, + "cvar95_kl": 16.48114013671875, + "delta_nll": 7.25096321105957, + "max_kl": 17.605976104736328, + "mean_kl": 6.971296310424805, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.19767442345619202 + }, + { + "candidate_nll": 6.6496124267578125, + "code_tokens": 40, + "cvar95_kl": 12.463279724121094, + "delta_nll": 5.723446846008301, + "max_kl": 14.400857925415039, + "mean_kl": 5.772263526916504, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 7.157586574554443, + "code_tokens": 27, + "cvar95_kl": 11.944979667663574, + "delta_nll": 5.594026565551758, + "max_kl": 12.672025680541992, + "mean_kl": 5.506706714630127, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 6.907175064086914, + "code_tokens": 52, + "cvar95_kl": 16.678848266601562, + "delta_nll": 5.817592144012451, + "max_kl": 18.517784118652344, + "mean_kl": 6.411139488220215, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 8.168252944946289, + "code_tokens": 71, + "cvar95_kl": 17.9066162109375, + "delta_nll": 7.4263787269592285, + "max_kl": 19.887624740600586, + "mean_kl": 7.487473011016846, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.11267605423927307 + }, + { + "candidate_nll": 5.317846775054932, + "code_tokens": 25, + "cvar95_kl": 8.979981422424316, + "delta_nll": 4.215994834899902, + "max_kl": 9.13831615447998, + "mean_kl": 4.486873149871826, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.254608154296875, + "code_tokens": 103, + "cvar95_kl": 15.945569038391113, + "delta_nll": 7.660112380981445, + "max_kl": 17.464189529418945, + "mean_kl": 7.493223190307617, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.0776699036359787 + }, + { + "candidate_nll": 8.07870864868164, + "code_tokens": 116, + "cvar95_kl": 16.1131534576416, + "delta_nll": 7.021170616149902, + "max_kl": 17.246156692504883, + "mean_kl": 7.088337421417236, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.11206896603107452 + }, + { + "candidate_nll": 8.837485313415527, + "code_tokens": 107, + "cvar95_kl": 16.710126876831055, + "delta_nll": 8.189139366149902, + "max_kl": 19.911754608154297, + "mean_kl": 8.277382850646973, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.06542056053876877 + }, + { + "candidate_nll": 5.732837677001953, + "code_tokens": 54, + "cvar95_kl": 13.183466911315918, + "delta_nll": 4.789799690246582, + "max_kl": 14.024900436401367, + "mean_kl": 4.791962146759033, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.676779747009277, + "code_tokens": 46, + "cvar95_kl": 15.829436302185059, + "delta_nll": 6.043387413024902, + "max_kl": 19.962158203125, + "mean_kl": 5.987496852874756, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.19565217196941376 + }, + { + "candidate_nll": 7.8536906242370605, + "code_tokens": 170, + "cvar95_kl": 17.331565856933594, + "delta_nll": 7.524543762207031, + "max_kl": 19.361743927001953, + "mean_kl": 7.504598617553711, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.10588235408067703 + }, + { + "candidate_nll": 7.592378616333008, + "code_tokens": 45, + "cvar95_kl": 14.329140663146973, + "delta_nll": 6.16568660736084, + "max_kl": 14.882908821105957, + "mean_kl": 6.112443923950195, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 4.956096649169922, + "code_tokens": 24, + "cvar95_kl": 7.603468894958496, + "delta_nll": 3.547793388366699, + "max_kl": 7.68065071105957, + "mean_kl": 3.8222007751464844, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.030374526977539, + "code_tokens": 78, + "cvar95_kl": 13.840007781982422, + "delta_nll": 5.868414878845215, + "max_kl": 14.361696243286133, + "mean_kl": 6.151705741882324, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.08974359184503555 + }, + { + "candidate_nll": 6.7280354499816895, + "code_tokens": 33, + "cvar95_kl": 15.28109359741211, + "delta_nll": 6.09747838973999, + "max_kl": 15.39272403717041, + "mean_kl": 5.770846366882324, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 6.994520664215088, + "code_tokens": 32, + "cvar95_kl": 15.581897735595703, + "delta_nll": 5.878850936889648, + "max_kl": 18.694246292114258, + "mean_kl": 5.2856245040893555, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.15625 + }, + { + "candidate_nll": 8.661295890808105, + "code_tokens": 77, + "cvar95_kl": 17.53172492980957, + "delta_nll": 7.672545909881592, + "max_kl": 19.736604690551758, + "mean_kl": 7.563970565795898, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.11688311398029327 + }, + { + "candidate_nll": 8.538979530334473, + "code_tokens": 222, + "cvar95_kl": 18.461143493652344, + "delta_nll": 8.116728782653809, + "max_kl": 21.576078414916992, + "mean_kl": 8.012288093566895, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.09009008854627609 + }, + { + "candidate_nll": 7.096502780914307, + "code_tokens": 86, + "cvar95_kl": 15.380635261535645, + "delta_nll": 6.047907829284668, + "max_kl": 16.840879440307617, + "mean_kl": 5.985501289367676, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.11627907305955887 + }, + { + "candidate_nll": 4.862823486328125, + "code_tokens": 31, + "cvar95_kl": 13.165590286254883, + "delta_nll": 4.528773784637451, + "max_kl": 15.811705589294434, + "mean_kl": 4.482339859008789, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 4.748581409454346, + "code_tokens": 34, + "cvar95_kl": 13.526323318481445, + "delta_nll": 4.236981391906738, + "max_kl": 15.168425559997559, + "mean_kl": 4.013738632202148, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 7.250950813293457, + "code_tokens": 38, + "cvar95_kl": 15.481290817260742, + "delta_nll": 5.863490581512451, + "max_kl": 15.629331588745117, + "mean_kl": 5.990288734436035, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.1315789520740509 + }, + { + "candidate_nll": 8.86225700378418, + "code_tokens": 93, + "cvar95_kl": 18.029342651367188, + "delta_nll": 7.89921236038208, + "max_kl": 21.181095123291016, + "mean_kl": 8.138976097106934, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.09677419066429138 + }, + { + "candidate_nll": 8.14401912689209, + "code_tokens": 70, + "cvar95_kl": 17.755401611328125, + "delta_nll": 7.365413665771484, + "max_kl": 20.31374168395996, + "mean_kl": 7.41357946395874, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.12857143580913544 + }, + { + "candidate_nll": 7.073780059814453, + "code_tokens": 67, + "cvar95_kl": 13.525055885314941, + "delta_nll": 6.2632880210876465, + "max_kl": 17.171621322631836, + "mean_kl": 6.331653118133545, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.16417910158634186 + }, + { + "candidate_nll": 5.910323619842529, + "code_tokens": 66, + "cvar95_kl": 12.111337661743164, + "delta_nll": 4.6241960525512695, + "max_kl": 14.840818405151367, + "mean_kl": 4.64076566696167, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 5.6869659423828125, + "code_tokens": 34, + "cvar95_kl": 12.653406143188477, + "delta_nll": 4.674671649932861, + "max_kl": 13.091263771057129, + "mean_kl": 4.46985387802124, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 6.831941604614258, + "code_tokens": 64, + "cvar95_kl": 12.5966796875, + "delta_nll": 5.9219160079956055, + "max_kl": 14.673209190368652, + "mean_kl": 5.675182342529297, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.140625 + }, + { + "candidate_nll": 6.932868957519531, + "code_tokens": 56, + "cvar95_kl": 12.607760429382324, + "delta_nll": 5.622513294219971, + "max_kl": 13.272445678710938, + "mean_kl": 5.905806541442871, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.1607142835855484 + }, + { + "candidate_nll": 4.25448751449585, + "code_tokens": 21, + "cvar95_kl": 10.194217681884766, + "delta_nll": 3.1294050216674805, + "max_kl": 11.244791030883789, + "mean_kl": 3.0091724395751953, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 6.603165626525879, + "code_tokens": 60, + "cvar95_kl": 15.453797340393066, + "delta_nll": 5.9378252029418945, + "max_kl": 16.70812225341797, + "mean_kl": 6.391079902648926, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.21666666865348816 + }, + { + "candidate_nll": 6.74594259262085, + "code_tokens": 29, + "cvar95_kl": 15.253816604614258, + "delta_nll": 5.392166614532471, + "max_kl": 15.962396621704102, + "mean_kl": 5.6083984375, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.3448275923728943 + }, + { + "candidate_nll": 7.076218605041504, + "code_tokens": 34, + "cvar95_kl": 18.344314575195312, + "delta_nll": 6.35417366027832, + "max_kl": 19.154054641723633, + "mean_kl": 6.246561527252197, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 8.889676094055176, + "code_tokens": 88, + "cvar95_kl": 18.498573303222656, + "delta_nll": 7.970568656921387, + "max_kl": 20.741443634033203, + "mean_kl": 8.075152397155762, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.06818182021379471 + }, + { + "candidate_nll": 4.929675102233887, + "code_tokens": 17, + "cvar95_kl": 11.613389015197754, + "delta_nll": 4.386451721191406, + "max_kl": 11.613389015197754, + "mean_kl": 3.8493499755859375, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 5.867949485778809, + "code_tokens": 35, + "cvar95_kl": 8.915464401245117, + "delta_nll": 3.9774022102355957, + "max_kl": 8.943449020385742, + "mean_kl": 4.399409294128418, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.010980129241943, + "code_tokens": 48, + "cvar95_kl": 14.888352394104004, + "delta_nll": 6.121203422546387, + "max_kl": 16.067365646362305, + "mean_kl": 6.582242488861084, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 8.03459644317627, + "code_tokens": 130, + "cvar95_kl": 16.285917282104492, + "delta_nll": 7.044323921203613, + "max_kl": 19.32326316833496, + "mean_kl": 7.093155860900879, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.13846154510974884 + }, + { + "candidate_nll": 8.921618461608887, + "code_tokens": 65, + "cvar95_kl": 16.265438079833984, + "delta_nll": 7.826679229736328, + "max_kl": 17.736364364624023, + "mean_kl": 7.943497180938721, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.0615384615957737 + }, + { + "candidate_nll": 9.373717308044434, + "code_tokens": 145, + "cvar95_kl": 17.269569396972656, + "delta_nll": 8.498702049255371, + "max_kl": 18.75420379638672, + "mean_kl": 8.763126373291016, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.08275862038135529 + }, + { + "candidate_nll": 10.002456665039062, + "code_tokens": 80, + "cvar95_kl": 19.722442626953125, + "delta_nll": 9.286757469177246, + "max_kl": 21.2563533782959, + "mean_kl": 8.857890129089355, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.07500000298023224 + }, + { + "candidate_nll": 6.560429096221924, + "code_tokens": 70, + "cvar95_kl": 15.065215110778809, + "delta_nll": 5.978292465209961, + "max_kl": 16.25652503967285, + "mean_kl": 6.2518768310546875, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.18571428954601288 + }, + { + "candidate_nll": 6.765726089477539, + "code_tokens": 25, + "cvar95_kl": 12.082816123962402, + "delta_nll": 5.277437210083008, + "max_kl": 12.920289993286133, + "mean_kl": 5.305974960327148, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 6.625311851501465, + "code_tokens": 50, + "cvar95_kl": 13.585978507995605, + "delta_nll": 4.970928192138672, + "max_kl": 14.932779312133789, + "mean_kl": 5.558022975921631, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 6.1400299072265625, + "code_tokens": 48, + "cvar95_kl": 12.239749908447266, + "delta_nll": 5.341987133026123, + "max_kl": 13.429872512817383, + "mean_kl": 5.407861709594727, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.180146217346191, + "code_tokens": 104, + "cvar95_kl": 15.623039245605469, + "delta_nll": 7.836073875427246, + "max_kl": 16.429468154907227, + "mean_kl": 7.638809680938721, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.09615384787321091 + }, + { + "candidate_nll": 6.601535797119141, + "code_tokens": 68, + "cvar95_kl": 13.665502548217773, + "delta_nll": 5.611541748046875, + "max_kl": 16.160844802856445, + "mean_kl": 5.704036712646484, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.19117647409439087 + }, + { + "candidate_nll": 7.864756107330322, + "code_tokens": 145, + "cvar95_kl": 15.475691795349121, + "delta_nll": 7.109844207763672, + "max_kl": 21.250667572021484, + "mean_kl": 7.229648590087891, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 5.6053900718688965, + "code_tokens": 41, + "cvar95_kl": 11.581901550292969, + "delta_nll": 4.760502338409424, + "max_kl": 13.98574161529541, + "mean_kl": 5.141038417816162, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.17073170840740204 + }, + { + "candidate_nll": 7.336277008056641, + "code_tokens": 53, + "cvar95_kl": 13.678977012634277, + "delta_nll": 6.575154781341553, + "max_kl": 14.12326717376709, + "mean_kl": 6.558290481567383, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.16981132328510284 + }, + { + "candidate_nll": 6.937140941619873, + "code_tokens": 58, + "cvar95_kl": 13.125587463378906, + "delta_nll": 6.144911289215088, + "max_kl": 14.403704643249512, + "mean_kl": 6.079861164093018, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 4.674365997314453, + "code_tokens": 24, + "cvar95_kl": 10.918342590332031, + "delta_nll": 2.9423060417175293, + "max_kl": 11.824268341064453, + "mean_kl": 3.835158586502075, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.767027378082275, + "code_tokens": 89, + "cvar95_kl": 16.000537872314453, + "delta_nll": 6.775173187255859, + "max_kl": 18.881813049316406, + "mean_kl": 7.136115074157715, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.13483145833015442 + }, + { + "candidate_nll": 8.01286506652832, + "code_tokens": 101, + "cvar95_kl": 17.07183837890625, + "delta_nll": 7.527789115905762, + "max_kl": 19.1136417388916, + "mean_kl": 7.4051513671875, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.06930693238973618 + }, + { + "candidate_nll": 6.670533657073975, + "code_tokens": 33, + "cvar95_kl": 13.115750312805176, + "delta_nll": 5.212975978851318, + "max_kl": 13.23792839050293, + "mean_kl": 5.74259090423584, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 7.3623809814453125, + "code_tokens": 50, + "cvar95_kl": 15.0514497756958, + "delta_nll": 6.315715312957764, + "max_kl": 16.282506942749023, + "mean_kl": 6.484164237976074, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 7.155370712280273, + "code_tokens": 58, + "cvar95_kl": 14.4005708694458, + "delta_nll": 5.746617794036865, + "max_kl": 14.698195457458496, + "mean_kl": 6.024205207824707, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 5.803891181945801, + "code_tokens": 62, + "cvar95_kl": 11.223570823669434, + "delta_nll": 4.824732303619385, + "max_kl": 14.200459480285645, + "mean_kl": 4.647563934326172, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 6.863863945007324, + "code_tokens": 42, + "cvar95_kl": 14.859352111816406, + "delta_nll": 6.172173976898193, + "max_kl": 16.32931137084961, + "mean_kl": 6.07257604598999, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.1190476194024086 + }, + { + "candidate_nll": 5.0553765296936035, + "code_tokens": 15, + "cvar95_kl": 10.641762733459473, + "delta_nll": 4.130635738372803, + "max_kl": 10.641762733459473, + "mean_kl": 4.104698657989502, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.1616530418396, + "code_tokens": 57, + "cvar95_kl": 12.161917686462402, + "delta_nll": 4.422870635986328, + "max_kl": 13.042291641235352, + "mean_kl": 4.558621406555176, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.24561403691768646 + }, + { + "candidate_nll": 7.268365383148193, + "code_tokens": 58, + "cvar95_kl": 15.152759552001953, + "delta_nll": 6.22482967376709, + "max_kl": 16.553865432739258, + "mean_kl": 6.097057819366455, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 5.443023204803467, + "code_tokens": 45, + "cvar95_kl": 12.854846000671387, + "delta_nll": 4.571052074432373, + "max_kl": 14.18737506866455, + "mean_kl": 4.885447978973389, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.150914669036865, + "code_tokens": 32, + "cvar95_kl": 14.962427139282227, + "delta_nll": 5.1216840744018555, + "max_kl": 16.536697387695312, + "mean_kl": 5.717253684997559, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 7.402857780456543, + "code_tokens": 39, + "cvar95_kl": 15.752144813537598, + "delta_nll": 6.082191467285156, + "max_kl": 16.275108337402344, + "mean_kl": 6.55966329574585, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 9.528717041015625, + "code_tokens": 136, + "cvar95_kl": 18.39187240600586, + "delta_nll": 9.117956161499023, + "max_kl": 20.918760299682617, + "mean_kl": 8.914448738098145, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.08088235557079315 + }, + { + "candidate_nll": 8.954302787780762, + "code_tokens": 50, + "cvar95_kl": 18.377702713012695, + "delta_nll": 7.733546257019043, + "max_kl": 18.641143798828125, + "mean_kl": 7.928536891937256, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 7.482668399810791, + "code_tokens": 34, + "cvar95_kl": 14.168060302734375, + "delta_nll": 6.616752624511719, + "max_kl": 14.278877258300781, + "mean_kl": 6.57956075668335, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.815806865692139, + "code_tokens": 44, + "cvar95_kl": 10.71746826171875, + "delta_nll": 4.637016773223877, + "max_kl": 11.252595901489258, + "mean_kl": 4.638314723968506, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 5.261447906494141, + "code_tokens": 36, + "cvar95_kl": 9.468873023986816, + "delta_nll": 3.828026056289673, + "max_kl": 10.403212547302246, + "mean_kl": 3.4954795837402344, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.1944444477558136 + }, + { + "candidate_nll": 6.440370559692383, + "code_tokens": 46, + "cvar95_kl": 12.395153999328613, + "delta_nll": 5.637213230133057, + "max_kl": 12.881540298461914, + "mean_kl": 5.216826438903809, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.28260868787765503 + }, + { + "candidate_nll": 8.637107849121094, + "code_tokens": 48, + "cvar95_kl": 15.940424919128418, + "delta_nll": 7.456768035888672, + "max_kl": 16.22764778137207, + "mean_kl": 8.034126281738281, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 6.139268398284912, + "code_tokens": 25, + "cvar95_kl": 16.609691619873047, + "delta_nll": 5.243857383728027, + "max_kl": 20.637540817260742, + "mean_kl": 4.8430705070495605, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 6.6564507484436035, + "code_tokens": 69, + "cvar95_kl": 11.775941848754883, + "delta_nll": 5.152171611785889, + "max_kl": 13.567802429199219, + "mean_kl": 5.559670925140381, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.18840579688549042 + }, + { + "candidate_nll": 8.276742935180664, + "code_tokens": 60, + "cvar95_kl": 17.736289978027344, + "delta_nll": 7.311285018920898, + "max_kl": 19.186098098754883, + "mean_kl": 7.649018287658691, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.11666666716337204 + }, + { + "candidate_nll": 7.657903671264648, + "code_tokens": 49, + "cvar95_kl": 14.732333183288574, + "delta_nll": 6.743426322937012, + "max_kl": 15.574737548828125, + "mean_kl": 6.367096424102783, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.10204081982374191 + }, + { + "candidate_nll": 8.131625175476074, + "code_tokens": 44, + "cvar95_kl": 12.432707786560059, + "delta_nll": 6.431097030639648, + "max_kl": 12.838434219360352, + "mean_kl": 6.3055195808410645, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.15909090638160706 + }, + { + "candidate_nll": 8.652636528015137, + "code_tokens": 81, + "cvar95_kl": 16.683162689208984, + "delta_nll": 7.5072221755981445, + "max_kl": 18.447301864624023, + "mean_kl": 7.586498260498047, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.17283950746059418 + }, + { + "candidate_nll": 6.009152412414551, + "code_tokens": 26, + "cvar95_kl": 10.709059715270996, + "delta_nll": 4.572417259216309, + "max_kl": 12.490154266357422, + "mean_kl": 4.840586185455322, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 7.008281230926514, + "code_tokens": 32, + "cvar95_kl": 14.390373229980469, + "delta_nll": 6.128360271453857, + "max_kl": 15.372965812683105, + "mean_kl": 6.283890724182129, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 6.746063709259033, + "code_tokens": 48, + "cvar95_kl": 15.042130470275879, + "delta_nll": 5.798321723937988, + "max_kl": 17.160863876342773, + "mean_kl": 5.730899810791016, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.2291666716337204 + }, + { + "candidate_nll": 9.581347465515137, + "code_tokens": 180, + "cvar95_kl": 18.001745223999023, + "delta_nll": 8.882026672363281, + "max_kl": 19.566600799560547, + "mean_kl": 8.83419132232666, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.04444444552063942 + }, + { + "candidate_nll": 4.574972152709961, + "code_tokens": 38, + "cvar95_kl": 9.343717575073242, + "delta_nll": 3.85321044921875, + "max_kl": 9.368440628051758, + "mean_kl": 3.823573112487793, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.3947368562221527 + }, + { + "candidate_nll": 8.128355026245117, + "code_tokens": 90, + "cvar95_kl": 16.40103530883789, + "delta_nll": 7.433293342590332, + "max_kl": 19.555612564086914, + "mean_kl": 7.555657386779785, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.149336338043213, + "code_tokens": 44, + "cvar95_kl": 13.288162231445312, + "delta_nll": 5.574706077575684, + "max_kl": 15.933117866516113, + "mean_kl": 5.554112434387207, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.065471649169922, + "code_tokens": 60, + "cvar95_kl": 16.632822036743164, + "delta_nll": 7.321571350097656, + "max_kl": 18.0528507232666, + "mean_kl": 7.495049953460693, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 6.941237449645996, + "code_tokens": 69, + "cvar95_kl": 14.310531616210938, + "delta_nll": 5.93843936920166, + "max_kl": 15.038946151733398, + "mean_kl": 6.24050235748291, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 6.401218414306641, + "code_tokens": 71, + "cvar95_kl": 15.762722969055176, + "delta_nll": 5.8836565017700195, + "max_kl": 19.09183692932129, + "mean_kl": 6.189350128173828, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.1690140813589096 + }, + { + "candidate_nll": 4.747765064239502, + "code_tokens": 22, + "cvar95_kl": 10.623150825500488, + "delta_nll": 3.505091905593872, + "max_kl": 11.153043746948242, + "mean_kl": 3.410327672958374, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 8.805912971496582, + "code_tokens": 215, + "cvar95_kl": 17.774799346923828, + "delta_nll": 8.2521333694458, + "max_kl": 21.88459014892578, + "mean_kl": 8.2288179397583, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.07906977087259293 + }, + { + "candidate_nll": 6.948088645935059, + "code_tokens": 26, + "cvar95_kl": 14.78968334197998, + "delta_nll": 5.845298767089844, + "max_kl": 16.01020050048828, + "mean_kl": 5.54416561126709, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 6.240787029266357, + "code_tokens": 33, + "cvar95_kl": 14.69863224029541, + "delta_nll": 5.372540473937988, + "max_kl": 18.065963745117188, + "mean_kl": 5.5807929039001465, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 4.552809715270996, + "code_tokens": 18, + "cvar95_kl": 10.610799789428711, + "delta_nll": 3.616661548614502, + "max_kl": 10.610799789428711, + "mean_kl": 4.044672012329102, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 6.845539093017578, + "code_tokens": 71, + "cvar95_kl": 12.276687622070312, + "delta_nll": 5.56547737121582, + "max_kl": 13.411232948303223, + "mean_kl": 5.69692325592041, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.1690140813589096 + }, + { + "candidate_nll": 6.721257209777832, + "code_tokens": 107, + "cvar95_kl": 14.950776100158691, + "delta_nll": 5.939855575561523, + "max_kl": 18.693668365478516, + "mean_kl": 5.979617118835449, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.15887850522994995 + }, + { + "candidate_nll": 7.189251899719238, + "code_tokens": 40, + "cvar95_kl": 16.1181640625, + "delta_nll": 5.134227275848389, + "max_kl": 16.813282012939453, + "mean_kl": 6.176969051361084, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.671682357788086, + "code_tokens": 32, + "cvar95_kl": 15.142990112304688, + "delta_nll": 4.307385444641113, + "max_kl": 17.02056121826172, + "mean_kl": 4.64546012878418, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 7.861637592315674, + "code_tokens": 130, + "cvar95_kl": 16.09879493713379, + "delta_nll": 7.162384033203125, + "max_kl": 20.018747329711914, + "mean_kl": 7.050595283508301, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.08461538702249527 + }, + { + "candidate_nll": 6.233307838439941, + "code_tokens": 27, + "cvar95_kl": 12.055421829223633, + "delta_nll": 4.761886119842529, + "max_kl": 12.453105926513672, + "mean_kl": 4.435704708099365, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.750950336456299, + "code_tokens": 42, + "cvar95_kl": 13.813040733337402, + "delta_nll": 6.77800989151001, + "max_kl": 15.968344688415527, + "mean_kl": 6.508464813232422, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 4.616468906402588, + "code_tokens": 30, + "cvar95_kl": 7.964151859283447, + "delta_nll": 3.8709557056427, + "max_kl": 8.016855239868164, + "mean_kl": 3.627166748046875, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 7.477138042449951, + "code_tokens": 58, + "cvar95_kl": 13.912186622619629, + "delta_nll": 6.057905197143555, + "max_kl": 14.889266014099121, + "mean_kl": 5.976393222808838, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 6.161607265472412, + "code_tokens": 34, + "cvar95_kl": 12.004609107971191, + "delta_nll": 4.931380271911621, + "max_kl": 12.514575958251953, + "mean_kl": 4.959087371826172, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 10.352421760559082, + "code_tokens": 208, + "cvar95_kl": 19.9382381439209, + "delta_nll": 9.650959014892578, + "max_kl": 23.48748779296875, + "mean_kl": 9.743081092834473, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.0625 + }, + { + "candidate_nll": 5.988702774047852, + "code_tokens": 83, + "cvar95_kl": 11.63129711151123, + "delta_nll": 5.089797019958496, + "max_kl": 13.366227149963379, + "mean_kl": 4.8277387619018555, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.1927710771560669 + }, + { + "candidate_nll": 8.426682472229004, + "code_tokens": 170, + "cvar95_kl": 18.09656524658203, + "delta_nll": 7.9041266441345215, + "max_kl": 21.513734817504883, + "mean_kl": 7.7331461906433105, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.06470588594675064 + }, + { + "candidate_nll": 5.810843467712402, + "code_tokens": 40, + "cvar95_kl": 16.507694244384766, + "delta_nll": 5.029422760009766, + "max_kl": 16.698774337768555, + "mean_kl": 5.621943473815918, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 7.083827018737793, + "code_tokens": 31, + "cvar95_kl": 14.075886726379395, + "delta_nll": 5.8371171951293945, + "max_kl": 14.859519958496094, + "mean_kl": 5.509047031402588, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 6.1083855628967285, + "code_tokens": 42, + "cvar95_kl": 11.536599159240723, + "delta_nll": 5.313615798950195, + "max_kl": 13.939332962036133, + "mean_kl": 5.4429545402526855, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.3547210693359375, + "code_tokens": 27, + "cvar95_kl": 11.172109603881836, + "delta_nll": 4.585655212402344, + "max_kl": 11.232344627380371, + "mean_kl": 4.934433937072754, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 7.091073989868164, + "code_tokens": 25, + "cvar95_kl": 12.00991439819336, + "delta_nll": 4.891263484954834, + "max_kl": 13.73182201385498, + "mean_kl": 5.152096748352051, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 5.378137111663818, + "code_tokens": 40, + "cvar95_kl": 14.882997512817383, + "delta_nll": 4.809750556945801, + "max_kl": 16.84212875366211, + "mean_kl": 4.871915340423584, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 5.638514041900635, + "code_tokens": 24, + "cvar95_kl": 9.487178802490234, + "delta_nll": 4.254159450531006, + "max_kl": 9.497962951660156, + "mean_kl": 4.760129928588867, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.323188304901123, + "code_tokens": 28, + "cvar95_kl": 9.434906959533691, + "delta_nll": 4.440443515777588, + "max_kl": 10.654858589172363, + "mean_kl": 3.9921438694000244, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 5.294888973236084, + "code_tokens": 71, + "cvar95_kl": 13.394630432128906, + "delta_nll": 4.816024303436279, + "max_kl": 16.83076286315918, + "mean_kl": 4.745501518249512, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.23943662643432617 + }, + { + "candidate_nll": 4.646359920501709, + "code_tokens": 24, + "cvar95_kl": 9.202442169189453, + "delta_nll": 4.11699914932251, + "max_kl": 9.680320739746094, + "mean_kl": 4.070908069610596, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.326730728149414, + "code_tokens": 34, + "cvar95_kl": 13.668283462524414, + "delta_nll": 5.055149555206299, + "max_kl": 13.786046028137207, + "mean_kl": 5.62324857711792, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 7.982239246368408, + "code_tokens": 64, + "cvar95_kl": 17.927507400512695, + "delta_nll": 6.862945556640625, + "max_kl": 21.214948654174805, + "mean_kl": 7.442513465881348, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.09375 + }, + { + "candidate_nll": 5.120667457580566, + "code_tokens": 45, + "cvar95_kl": 11.6151704788208, + "delta_nll": 4.414076328277588, + "max_kl": 12.629439353942871, + "mean_kl": 4.496488571166992, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.424917221069336, + "code_tokens": 48, + "cvar95_kl": 13.185219764709473, + "delta_nll": 5.226212978363037, + "max_kl": 14.066389083862305, + "mean_kl": 5.966269016265869, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 5.422270774841309, + "code_tokens": 21, + "cvar95_kl": 10.968990325927734, + "delta_nll": 4.72206974029541, + "max_kl": 11.498151779174805, + "mean_kl": 4.575015544891357, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 6.008509159088135, + "code_tokens": 24, + "cvar95_kl": 12.63286304473877, + "delta_nll": 5.254095077514648, + "max_kl": 13.93513011932373, + "mean_kl": 4.89384126663208, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.463641166687012, + "code_tokens": 63, + "cvar95_kl": 15.343563079833984, + "delta_nll": 5.891714096069336, + "max_kl": 17.358592987060547, + "mean_kl": 5.993402481079102, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 5.927897930145264, + "code_tokens": 22, + "cvar95_kl": 12.65832805633545, + "delta_nll": 4.649138927459717, + "max_kl": 13.20510196685791, + "mean_kl": 4.319443702697754, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 4.194713592529297, + "code_tokens": 25, + "cvar95_kl": 10.626363754272461, + "delta_nll": 3.0660512447357178, + "max_kl": 10.802716255187988, + "mean_kl": 3.8965773582458496, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.102769374847412, + "code_tokens": 21, + "cvar95_kl": 12.759050369262695, + "delta_nll": 3.1424436569213867, + "max_kl": 16.666309356689453, + "mean_kl": 4.013519287109375, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 4.609527587890625, + "code_tokens": 24, + "cvar95_kl": 7.333052635192871, + "delta_nll": 3.143195629119873, + "max_kl": 7.636097431182861, + "mean_kl": 3.3157129287719727, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 5.390458583831787, + "code_tokens": 15, + "cvar95_kl": 9.88888168334961, + "delta_nll": 3.596970796585083, + "max_kl": 9.88888168334961, + "mean_kl": 3.8136448860168457, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.301196098327637, + "code_tokens": 59, + "cvar95_kl": 12.429542541503906, + "delta_nll": 4.520339488983154, + "max_kl": 13.191821098327637, + "mean_kl": 4.777937889099121, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.20338982343673706 + }, + { + "candidate_nll": 6.76007604598999, + "code_tokens": 54, + "cvar95_kl": 12.622645378112793, + "delta_nll": 5.825957775115967, + "max_kl": 12.976205825805664, + "mean_kl": 5.383543968200684, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.3248162269592285, + "code_tokens": 28, + "cvar95_kl": 12.335420608520508, + "delta_nll": 4.372957229614258, + "max_kl": 12.6639404296875, + "mean_kl": 5.086943626403809, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 6.510545253753662, + "code_tokens": 46, + "cvar95_kl": 13.404223442077637, + "delta_nll": 5.425819396972656, + "max_kl": 15.438034057617188, + "mean_kl": 5.868591785430908, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.15217390656471252 + }, + { + "candidate_nll": 5.649575710296631, + "code_tokens": 27, + "cvar95_kl": 13.355430603027344, + "delta_nll": 4.134753227233887, + "max_kl": 14.154590606689453, + "mean_kl": 4.929583549499512, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 8.299890518188477, + "code_tokens": 97, + "cvar95_kl": 16.92266082763672, + "delta_nll": 7.042459964752197, + "max_kl": 19.856731414794922, + "mean_kl": 7.25936222076416, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.10309278219938278 + }, + { + "candidate_nll": 4.886660099029541, + "code_tokens": 24, + "cvar95_kl": 11.247390747070312, + "delta_nll": 4.084427833557129, + "max_kl": 11.639456748962402, + "mean_kl": 4.392008304595947, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 6.47127103805542, + "code_tokens": 30, + "cvar95_kl": 13.621146202087402, + "delta_nll": 5.206122398376465, + "max_kl": 15.745635032653809, + "mean_kl": 5.875363826751709, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 6.455638885498047, + "code_tokens": 38, + "cvar95_kl": 15.182232856750488, + "delta_nll": 5.022569179534912, + "max_kl": 16.193653106689453, + "mean_kl": 5.673807144165039, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 4.961877822875977, + "code_tokens": 21, + "cvar95_kl": 9.806275367736816, + "delta_nll": 3.4740986824035645, + "max_kl": 11.643233299255371, + "mean_kl": 3.356788158416748, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 6.220954895019531, + "code_tokens": 65, + "cvar95_kl": 12.394546508789062, + "delta_nll": 5.683993339538574, + "max_kl": 13.102535247802734, + "mean_kl": 5.776814937591553, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.16923077404499054 + }, + { + "candidate_nll": 5.605522632598877, + "code_tokens": 27, + "cvar95_kl": 13.975765228271484, + "delta_nll": 4.820996284484863, + "max_kl": 14.270345687866211, + "mean_kl": 5.21040153503418, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.273707866668701, + "code_tokens": 35, + "cvar95_kl": 14.000242233276367, + "delta_nll": 5.549891948699951, + "max_kl": 15.778583526611328, + "mean_kl": 5.545926094055176, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 7.106863021850586, + "code_tokens": 75, + "cvar95_kl": 12.65846061706543, + "delta_nll": 6.182338714599609, + "max_kl": 13.174057006835938, + "mean_kl": 6.060174942016602, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.1066666692495346 + }, + { + "candidate_nll": 5.729144096374512, + "code_tokens": 25, + "cvar95_kl": 11.844375610351562, + "delta_nll": 4.087408542633057, + "max_kl": 13.5980224609375, + "mean_kl": 4.367155075073242, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 6.754776477813721, + "code_tokens": 52, + "cvar95_kl": 13.177912712097168, + "delta_nll": 5.830787658691406, + "max_kl": 13.729812622070312, + "mean_kl": 6.171891212463379, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.17307692766189575 + }, + { + "candidate_nll": 7.893949031829834, + "code_tokens": 110, + "cvar95_kl": 14.994452476501465, + "delta_nll": 7.283288955688477, + "max_kl": 17.78814697265625, + "mean_kl": 7.328400611877441, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.09090909361839294 + }, + { + "candidate_nll": 5.463952541351318, + "code_tokens": 25, + "cvar95_kl": 10.82664966583252, + "delta_nll": 4.231130599975586, + "max_kl": 12.234901428222656, + "mean_kl": 4.400898456573486, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.1224365234375, + "code_tokens": 56, + "cvar95_kl": 15.83352279663086, + "delta_nll": 7.105450630187988, + "max_kl": 17.212844848632812, + "mean_kl": 6.898735523223877, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.0892857164144516 + }, + { + "candidate_nll": 8.645548820495605, + "code_tokens": 108, + "cvar95_kl": 15.787726402282715, + "delta_nll": 7.091700553894043, + "max_kl": 17.14756965637207, + "mean_kl": 6.801246166229248, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.10185185074806213 + }, + { + "candidate_nll": 6.889382362365723, + "code_tokens": 47, + "cvar95_kl": 13.261077880859375, + "delta_nll": 5.517642498016357, + "max_kl": 14.350733757019043, + "mean_kl": 6.009490489959717, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.19148936867713928 + }, + { + "candidate_nll": 6.891030788421631, + "code_tokens": 102, + "cvar95_kl": 14.741992950439453, + "delta_nll": 5.892621994018555, + "max_kl": 17.795310974121094, + "mean_kl": 6.155881881713867, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 5.213870048522949, + "code_tokens": 18, + "cvar95_kl": 8.987027168273926, + "delta_nll": 3.5882444381713867, + "max_kl": 8.987027168273926, + "mean_kl": 4.337656497955322, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.162978172302246, + "code_tokens": 28, + "cvar95_kl": 14.838968276977539, + "delta_nll": 6.063313961029053, + "max_kl": 16.388870239257812, + "mean_kl": 5.90545129776001, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.321813583374023, + "code_tokens": 30, + "cvar95_kl": 19.186813354492188, + "delta_nll": 6.778412818908691, + "max_kl": 21.98862648010254, + "mean_kl": 6.6709418296813965, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.844112396240234, + "code_tokens": 48, + "cvar95_kl": 17.669654846191406, + "delta_nll": 7.368995189666748, + "max_kl": 18.193397521972656, + "mean_kl": 7.312493801116943, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.905741214752197, + "code_tokens": 59, + "cvar95_kl": 16.805429458618164, + "delta_nll": 7.3687262535095215, + "max_kl": 16.99029541015625, + "mean_kl": 7.215884685516357, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.16949152946472168 + }, + { + "candidate_nll": 8.322015762329102, + "code_tokens": 102, + "cvar95_kl": 16.96630859375, + "delta_nll": 7.279821872711182, + "max_kl": 18.161495208740234, + "mean_kl": 7.4429030418396, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.09803921729326248 + }, + { + "candidate_nll": 6.844533443450928, + "code_tokens": 36, + "cvar95_kl": 10.479378700256348, + "delta_nll": 6.104167938232422, + "max_kl": 10.710064888000488, + "mean_kl": 5.596273422241211, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.1944444477558136 + }, + { + "candidate_nll": 7.209078311920166, + "code_tokens": 51, + "cvar95_kl": 15.650996208190918, + "delta_nll": 6.320146560668945, + "max_kl": 16.769989013671875, + "mean_kl": 6.275389194488525, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.1568627506494522 + }, + { + "candidate_nll": 7.465526580810547, + "code_tokens": 62, + "cvar95_kl": 14.671765327453613, + "delta_nll": 6.219847679138184, + "max_kl": 16.4241886138916, + "mean_kl": 6.281627178192139, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 5.953440189361572, + "code_tokens": 39, + "cvar95_kl": 17.039403915405273, + "delta_nll": 5.377158164978027, + "max_kl": 19.235767364501953, + "mean_kl": 5.510340213775635, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 6.127708435058594, + "code_tokens": 76, + "cvar95_kl": 14.246227264404297, + "delta_nll": 5.38802433013916, + "max_kl": 15.734146118164062, + "mean_kl": 5.283856391906738, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 8.937034606933594, + "code_tokens": 58, + "cvar95_kl": 16.323528289794922, + "delta_nll": 7.4323835372924805, + "max_kl": 18.477062225341797, + "mean_kl": 7.104292392730713, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.1034482792019844 + }, + { + "candidate_nll": 9.239936828613281, + "code_tokens": 93, + "cvar95_kl": 16.55300521850586, + "delta_nll": 8.112541198730469, + "max_kl": 18.196252822875977, + "mean_kl": 8.126696586608887, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.10752688348293304 + }, + { + "candidate_nll": 7.194852828979492, + "code_tokens": 45, + "cvar95_kl": 12.338841438293457, + "delta_nll": 5.839607238769531, + "max_kl": 13.295415878295898, + "mean_kl": 6.076913833618164, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 7.944637298583984, + "code_tokens": 209, + "cvar95_kl": 16.479883193969727, + "delta_nll": 7.387661933898926, + "max_kl": 18.61865997314453, + "mean_kl": 7.487734794616699, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.08133970946073532 + }, + { + "candidate_nll": 7.596997261047363, + "code_tokens": 46, + "cvar95_kl": 14.74164867401123, + "delta_nll": 6.6285295486450195, + "max_kl": 15.748745918273926, + "mean_kl": 5.990118980407715, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 6.340638160705566, + "code_tokens": 39, + "cvar95_kl": 12.375508308410645, + "delta_nll": 5.619851112365723, + "max_kl": 12.491083145141602, + "mean_kl": 5.276240348815918, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 5.646460056304932, + "code_tokens": 25, + "cvar95_kl": 10.875345230102539, + "delta_nll": 4.7047648429870605, + "max_kl": 10.891363143920898, + "mean_kl": 4.620662212371826, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 5.809481143951416, + "code_tokens": 28, + "cvar95_kl": 13.468244552612305, + "delta_nll": 4.634284973144531, + "max_kl": 14.605151176452637, + "mean_kl": 5.087039947509766, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 6.863188743591309, + "code_tokens": 84, + "cvar95_kl": 14.54045295715332, + "delta_nll": 5.60926628112793, + "max_kl": 18.86033058166504, + "mean_kl": 6.326945781707764, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.1547619104385376 + }, + { + "candidate_nll": 5.785930633544922, + "code_tokens": 34, + "cvar95_kl": 14.011640548706055, + "delta_nll": 4.9736504554748535, + "max_kl": 16.368324279785156, + "mean_kl": 5.207851409912109, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 5.5699052810668945, + "code_tokens": 28, + "cvar95_kl": 9.67910385131836, + "delta_nll": 4.314858436584473, + "max_kl": 10.75238037109375, + "mean_kl": 4.168410778045654, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.790454387664795, + "code_tokens": 92, + "cvar95_kl": 15.548299789428711, + "delta_nll": 6.691608905792236, + "max_kl": 17.63614845275879, + "mean_kl": 7.014834403991699, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.10869564861059189 + }, + { + "candidate_nll": 8.52639389038086, + "code_tokens": 185, + "cvar95_kl": 18.054304122924805, + "delta_nll": 7.796279430389404, + "max_kl": 20.832555770874023, + "mean_kl": 7.740114212036133, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.1135135143995285 + }, + { + "candidate_nll": 7.389427185058594, + "code_tokens": 106, + "cvar95_kl": 16.87372398376465, + "delta_nll": 7.03643274307251, + "max_kl": 18.959171295166016, + "mean_kl": 7.04564094543457, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.1603773534297943 + }, + { + "candidate_nll": 8.345284461975098, + "code_tokens": 64, + "cvar95_kl": 15.539278030395508, + "delta_nll": 6.802730083465576, + "max_kl": 17.50446128845215, + "mean_kl": 6.569212913513184, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.109375 + }, + { + "candidate_nll": 6.755393981933594, + "code_tokens": 74, + "cvar95_kl": 16.84652328491211, + "delta_nll": 5.6446099281311035, + "max_kl": 18.43549156188965, + "mean_kl": 6.101490020751953, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.22972972691059113 + }, + { + "candidate_nll": 5.217386245727539, + "code_tokens": 27, + "cvar95_kl": 13.529723167419434, + "delta_nll": 4.229982376098633, + "max_kl": 16.178529739379883, + "mean_kl": 4.631825923919678, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 7.89124059677124, + "code_tokens": 72, + "cvar95_kl": 18.32428741455078, + "delta_nll": 7.267402648925781, + "max_kl": 24.288833618164062, + "mean_kl": 7.409251689910889, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 7.218481063842773, + "code_tokens": 62, + "cvar95_kl": 13.689577102661133, + "delta_nll": 6.2851786613464355, + "max_kl": 15.043797492980957, + "mean_kl": 5.971053123474121, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 8.198907852172852, + "code_tokens": 42, + "cvar95_kl": 17.811159133911133, + "delta_nll": 7.500741481781006, + "max_kl": 19.452529907226562, + "mean_kl": 7.2759785652160645, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 8.635335922241211, + "code_tokens": 238, + "cvar95_kl": 18.872467041015625, + "delta_nll": 7.748003005981445, + "max_kl": 27.844261169433594, + "mean_kl": 8.006643295288086, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.07563025504350662 + }, + { + "candidate_nll": 7.281637191772461, + "code_tokens": 68, + "cvar95_kl": 15.086183547973633, + "delta_nll": 6.367125511169434, + "max_kl": 15.507478713989258, + "mean_kl": 6.384670734405518, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.1617647111415863 + }, + { + "candidate_nll": 7.431295871734619, + "code_tokens": 83, + "cvar95_kl": 17.284242630004883, + "delta_nll": 6.447145938873291, + "max_kl": 23.644147872924805, + "mean_kl": 6.544534206390381, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.16867469251155853 + }, + { + "candidate_nll": 7.192830562591553, + "code_tokens": 64, + "cvar95_kl": 14.659782409667969, + "delta_nll": 5.929062843322754, + "max_kl": 15.761675834655762, + "mean_kl": 6.085310459136963, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.234375 + }, + { + "candidate_nll": 5.183047294616699, + "code_tokens": 37, + "cvar95_kl": 11.704694747924805, + "delta_nll": 4.526247501373291, + "max_kl": 14.042126655578613, + "mean_kl": 4.588278770446777, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 5.611789703369141, + "code_tokens": 27, + "cvar95_kl": 13.255016326904297, + "delta_nll": 4.10186243057251, + "max_kl": 16.33744239807129, + "mean_kl": 4.249753475189209, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.157260894775391, + "code_tokens": 40, + "cvar95_kl": 15.13749885559082, + "delta_nll": 5.265544414520264, + "max_kl": 15.690699577331543, + "mean_kl": 5.087503910064697, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 6.807032585144043, + "code_tokens": 28, + "cvar95_kl": 11.909867286682129, + "delta_nll": 5.80925989151001, + "max_kl": 11.914141654968262, + "mean_kl": 5.672232627868652, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 5.669111728668213, + "code_tokens": 59, + "cvar95_kl": 13.353999137878418, + "delta_nll": 4.81448221206665, + "max_kl": 14.258007049560547, + "mean_kl": 5.0509819984436035, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.1525423675775528 + }, + { + "candidate_nll": 6.431038856506348, + "code_tokens": 57, + "cvar95_kl": 16.21872901916504, + "delta_nll": 5.861038684844971, + "max_kl": 19.939592361450195, + "mean_kl": 5.7571635246276855, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.22807016968727112 + }, + { + "candidate_nll": 7.334634780883789, + "code_tokens": 25, + "cvar95_kl": 14.906620025634766, + "delta_nll": 5.950694561004639, + "max_kl": 17.518169403076172, + "mean_kl": 5.91068172454834, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.11999999731779099 + }, + { + "candidate_nll": 7.58611536026001, + "code_tokens": 48, + "cvar95_kl": 14.0753173828125, + "delta_nll": 6.838963031768799, + "max_kl": 14.659805297851562, + "mean_kl": 6.882654190063477, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.745389461517334, + "code_tokens": 35, + "cvar95_kl": 11.169576644897461, + "delta_nll": 5.883613586425781, + "max_kl": 11.311220169067383, + "mean_kl": 5.5847086906433105, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 5.864761829376221, + "code_tokens": 30, + "cvar95_kl": 12.862334251403809, + "delta_nll": 5.220054626464844, + "max_kl": 13.732975959777832, + "mean_kl": 4.92289924621582, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.506046295166016, + "code_tokens": 49, + "cvar95_kl": 13.736601829528809, + "delta_nll": 5.382968902587891, + "max_kl": 15.601652145385742, + "mean_kl": 5.516934871673584, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.26530611515045166 + }, + { + "candidate_nll": 9.361011505126953, + "code_tokens": 98, + "cvar95_kl": 18.028289794921875, + "delta_nll": 8.344794273376465, + "max_kl": 20.155498504638672, + "mean_kl": 8.485852241516113, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 5.594168186187744, + "code_tokens": 25, + "cvar95_kl": 9.186944961547852, + "delta_nll": 4.213361740112305, + "max_kl": 9.891919136047363, + "mean_kl": 4.324902534484863, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 5.956273555755615, + "code_tokens": 27, + "cvar95_kl": 13.341376304626465, + "delta_nll": 5.1785502433776855, + "max_kl": 15.048957824707031, + "mean_kl": 5.027408599853516, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.9331717491149902, + "code_tokens": 20, + "cvar95_kl": 10.761700630187988, + "delta_nll": 2.888087034225464, + "max_kl": 10.761700630187988, + "mean_kl": 3.7168827056884766, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 9.52890396118164, + "code_tokens": 253, + "cvar95_kl": 17.345897674560547, + "delta_nll": 8.836234092712402, + "max_kl": 20.65812110900879, + "mean_kl": 8.855895042419434, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.05928853899240494 + }, + { + "candidate_nll": 6.763467788696289, + "code_tokens": 55, + "cvar95_kl": 13.870368957519531, + "delta_nll": 5.934347629547119, + "max_kl": 15.019367218017578, + "mean_kl": 5.777322769165039, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.540221691131592, + "code_tokens": 31, + "cvar95_kl": 11.655901908874512, + "delta_nll": 4.333723068237305, + "max_kl": 13.312810897827148, + "mean_kl": 4.626158237457275, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 5.986238956451416, + "code_tokens": 35, + "cvar95_kl": 12.376958847045898, + "delta_nll": 5.2819929122924805, + "max_kl": 12.979392051696777, + "mean_kl": 5.077009201049805, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.22857142984867096 + }, + { + "candidate_nll": 5.726065158843994, + "code_tokens": 61, + "cvar95_kl": 12.358451843261719, + "delta_nll": 4.923196792602539, + "max_kl": 13.132713317871094, + "mean_kl": 5.022447109222412, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.13114753365516663 + }, + { + "candidate_nll": 5.875941276550293, + "code_tokens": 55, + "cvar95_kl": 11.501728057861328, + "delta_nll": 4.9212236404418945, + "max_kl": 12.508980751037598, + "mean_kl": 5.0808892250061035, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.16363635659217834 + }, + { + "candidate_nll": 8.843277931213379, + "code_tokens": 108, + "cvar95_kl": 15.979016304016113, + "delta_nll": 7.336310863494873, + "max_kl": 17.44266128540039, + "mean_kl": 7.405257225036621, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.740170001983643, + "code_tokens": 58, + "cvar95_kl": 12.83385181427002, + "delta_nll": 5.452863693237305, + "max_kl": 13.30158805847168, + "mean_kl": 5.273911476135254, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 8.268804550170898, + "code_tokens": 91, + "cvar95_kl": 17.039676666259766, + "delta_nll": 7.441349506378174, + "max_kl": 20.964881896972656, + "mean_kl": 7.283730506896973, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.09890110045671463 + }, + { + "candidate_nll": 7.975749969482422, + "code_tokens": 105, + "cvar95_kl": 17.5494327545166, + "delta_nll": 7.212010383605957, + "max_kl": 18.70838165283203, + "mean_kl": 7.458605766296387, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 6.393664836883545, + "code_tokens": 49, + "cvar95_kl": 17.222280502319336, + "delta_nll": 5.74641227722168, + "max_kl": 17.734054565429688, + "mean_kl": 5.646862030029297, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.2448979616165161 + }, + { + "candidate_nll": 7.179471969604492, + "code_tokens": 27, + "cvar95_kl": 12.422642707824707, + "delta_nll": 5.71730899810791, + "max_kl": 12.86152172088623, + "mean_kl": 5.785730361938477, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 5.4093337059021, + "code_tokens": 25, + "cvar95_kl": 11.26478385925293, + "delta_nll": 4.368957042694092, + "max_kl": 13.797957420349121, + "mean_kl": 4.523558139801025, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.851992130279541, + "code_tokens": 98, + "cvar95_kl": 15.982503890991211, + "delta_nll": 6.5569376945495605, + "max_kl": 17.528573989868164, + "mean_kl": 6.673429012298584, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.11224489659070969 + }, + { + "candidate_nll": 4.2427144050598145, + "code_tokens": 23, + "cvar95_kl": 9.709333419799805, + "delta_nll": 3.59293270111084, + "max_kl": 10.701260566711426, + "mean_kl": 3.6369292736053467, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 5.815527439117432, + "code_tokens": 31, + "cvar95_kl": 11.997713088989258, + "delta_nll": 3.9501538276672363, + "max_kl": 12.68079948425293, + "mean_kl": 4.975469589233398, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.22580644488334656 + }, + { + "candidate_nll": 5.300911903381348, + "code_tokens": 36, + "cvar95_kl": 12.232538223266602, + "delta_nll": 4.552818775177002, + "max_kl": 13.95749568939209, + "mean_kl": 4.598057270050049, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.641443252563477, + "code_tokens": 38, + "cvar95_kl": 12.979876518249512, + "delta_nll": 5.207765579223633, + "max_kl": 13.84743881225586, + "mean_kl": 5.224290370941162, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 5.650312900543213, + "code_tokens": 25, + "cvar95_kl": 10.434804916381836, + "delta_nll": 4.044222831726074, + "max_kl": 13.137804985046387, + "mean_kl": 3.616605520248413, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.59473991394043, + "code_tokens": 26, + "cvar95_kl": 9.448121070861816, + "delta_nll": 3.7152907848358154, + "max_kl": 10.656989097595215, + "mean_kl": 4.079739093780518, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 6.517138957977295, + "code_tokens": 62, + "cvar95_kl": 13.790609359741211, + "delta_nll": 5.7746968269348145, + "max_kl": 14.8629789352417, + "mean_kl": 5.662886142730713, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.14516128599643707 + }, + { + "candidate_nll": 6.830109119415283, + "code_tokens": 36, + "cvar95_kl": 15.686578750610352, + "delta_nll": 5.09158992767334, + "max_kl": 17.004783630371094, + "mean_kl": 5.654405117034912, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 9.64808464050293, + "code_tokens": 119, + "cvar95_kl": 18.2473201751709, + "delta_nll": 8.317495346069336, + "max_kl": 22.380565643310547, + "mean_kl": 8.509598731994629, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.05882352963089943 + }, + { + "candidate_nll": 4.463845729827881, + "code_tokens": 37, + "cvar95_kl": 11.630924224853516, + "delta_nll": 4.1511383056640625, + "max_kl": 12.028944969177246, + "mean_kl": 3.9618592262268066, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 7.67548131942749, + "code_tokens": 28, + "cvar95_kl": 19.358203887939453, + "delta_nll": 6.89274787902832, + "max_kl": 22.29008674621582, + "mean_kl": 7.200257301330566, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 9.86301326751709, + "code_tokens": 143, + "cvar95_kl": 17.8199462890625, + "delta_nll": 9.296666145324707, + "max_kl": 19.979907989501953, + "mean_kl": 9.52863597869873, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.04895104840397835 + }, + { + "candidate_nll": 6.136913299560547, + "code_tokens": 29, + "cvar95_kl": 11.802469253540039, + "delta_nll": 4.354607105255127, + "max_kl": 11.911277770996094, + "mean_kl": 5.410234451293945, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 5.574937343597412, + "code_tokens": 28, + "cvar95_kl": 10.20347785949707, + "delta_nll": 4.502449035644531, + "max_kl": 10.486940383911133, + "mean_kl": 4.167420387268066, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 7.215721607208252, + "code_tokens": 30, + "cvar95_kl": 14.068357467651367, + "delta_nll": 5.95168399810791, + "max_kl": 14.876054763793945, + "mean_kl": 6.293858528137207, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.23333333432674408 + }, + { + "candidate_nll": 7.681606769561768, + "code_tokens": 85, + "cvar95_kl": 20.53774642944336, + "delta_nll": 7.259796619415283, + "max_kl": 25.41874885559082, + "mean_kl": 7.095819473266602, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.12941177189350128 + }, + { + "candidate_nll": 7.065539836883545, + "code_tokens": 44, + "cvar95_kl": 13.298022270202637, + "delta_nll": 6.088296413421631, + "max_kl": 14.634627342224121, + "mean_kl": 5.977355480194092, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.15909090638160706 + }, + { + "candidate_nll": 5.580483913421631, + "code_tokens": 19, + "cvar95_kl": 10.963284492492676, + "delta_nll": 4.776381492614746, + "max_kl": 10.963284492492676, + "mean_kl": 5.139044761657715, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 8.326541900634766, + "code_tokens": 113, + "cvar95_kl": 16.82334327697754, + "delta_nll": 7.589118957519531, + "max_kl": 18.88829803466797, + "mean_kl": 7.50238561630249, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.09734513610601425 + }, + { + "candidate_nll": 6.257040977478027, + "code_tokens": 40, + "cvar95_kl": 12.870880126953125, + "delta_nll": 4.781846523284912, + "max_kl": 14.73540210723877, + "mean_kl": 5.40787410736084, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 7.842981815338135, + "code_tokens": 51, + "cvar95_kl": 14.988207817077637, + "delta_nll": 6.236321926116943, + "max_kl": 15.664250373840332, + "mean_kl": 6.923431873321533, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.1568627506494522 + }, + { + "candidate_nll": 6.6655402183532715, + "code_tokens": 55, + "cvar95_kl": 12.213672637939453, + "delta_nll": 5.084507942199707, + "max_kl": 12.300281524658203, + "mean_kl": 5.5255937576293945, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.09090909361839294 + }, + { + "candidate_nll": 7.3755669593811035, + "code_tokens": 27, + "cvar95_kl": 16.200557708740234, + "delta_nll": 6.2454986572265625, + "max_kl": 16.717273712158203, + "mean_kl": 6.169562816619873, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 5.885349750518799, + "code_tokens": 32, + "cvar95_kl": 11.134197235107422, + "delta_nll": 5.045543670654297, + "max_kl": 11.5238676071167, + "mean_kl": 5.535638809204102, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 7.755094528198242, + "code_tokens": 64, + "cvar95_kl": 16.210189819335938, + "delta_nll": 7.102556228637695, + "max_kl": 17.70528221130371, + "mean_kl": 6.812927722930908, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.990096092224121, + "code_tokens": 35, + "cvar95_kl": 13.693809509277344, + "delta_nll": 5.940442085266113, + "max_kl": 14.174893379211426, + "mean_kl": 5.941375255584717, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 8.767980575561523, + "code_tokens": 152, + "cvar95_kl": 18.950281143188477, + "delta_nll": 8.316132545471191, + "max_kl": 21.65281867980957, + "mean_kl": 8.18834114074707, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.09210526198148727 + }, + { + "candidate_nll": 7.62637996673584, + "code_tokens": 77, + "cvar95_kl": 14.927253723144531, + "delta_nll": 6.365045547485352, + "max_kl": 15.948203086853027, + "mean_kl": 6.461061000823975, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.16883116960525513 + }, + { + "candidate_nll": 8.463315963745117, + "code_tokens": 77, + "cvar95_kl": 18.352428436279297, + "delta_nll": 7.8950347900390625, + "max_kl": 19.948848724365234, + "mean_kl": 7.885838031768799, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.11688311398029327 + }, + { + "candidate_nll": 5.715290546417236, + "code_tokens": 71, + "cvar95_kl": 14.746050834655762, + "delta_nll": 5.153201580047607, + "max_kl": 17.03732681274414, + "mean_kl": 5.136462688446045, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.19718310236930847 + }, + { + "candidate_nll": 8.585774421691895, + "code_tokens": 112, + "cvar95_kl": 18.97043800354004, + "delta_nll": 7.707395076751709, + "max_kl": 21.311609268188477, + "mean_kl": 7.693817138671875, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.1160714253783226 + }, + { + "candidate_nll": 8.90445613861084, + "code_tokens": 111, + "cvar95_kl": 17.056930541992188, + "delta_nll": 8.184552192687988, + "max_kl": 19.2037353515625, + "mean_kl": 8.220453262329102, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.07207207381725311 + }, + { + "candidate_nll": 6.732752799987793, + "code_tokens": 62, + "cvar95_kl": 14.758539199829102, + "delta_nll": 5.944856643676758, + "max_kl": 17.599029541015625, + "mean_kl": 5.724658012390137, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.14516128599643707 + }, + { + "candidate_nll": 5.947854042053223, + "code_tokens": 52, + "cvar95_kl": 13.039204597473145, + "delta_nll": 5.337274074554443, + "max_kl": 13.961092948913574, + "mean_kl": 5.334310531616211, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 5.887746334075928, + "code_tokens": 24, + "cvar95_kl": 10.467778205871582, + "delta_nll": 3.9925198554992676, + "max_kl": 10.567131042480469, + "mean_kl": 4.383285045623779, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 6.577018737792969, + "code_tokens": 48, + "cvar95_kl": 12.322425842285156, + "delta_nll": 5.685248851776123, + "max_kl": 15.681009292602539, + "mean_kl": 5.794681549072266, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.1458333283662796 + }, + { + "candidate_nll": 7.987784385681152, + "code_tokens": 98, + "cvar95_kl": 18.226848602294922, + "delta_nll": 6.937491416931152, + "max_kl": 22.707590103149414, + "mean_kl": 7.091317176818848, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.15306122601032257 + }, + { + "candidate_nll": 7.675449371337891, + "code_tokens": 48, + "cvar95_kl": 16.424232482910156, + "delta_nll": 6.682763576507568, + "max_kl": 17.4176025390625, + "mean_kl": 6.591573715209961, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 6.500777244567871, + "code_tokens": 56, + "cvar95_kl": 13.773426055908203, + "delta_nll": 5.554781913757324, + "max_kl": 15.964561462402344, + "mean_kl": 5.888649940490723, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.2321428507566452 + }, + { + "candidate_nll": 6.767693042755127, + "code_tokens": 88, + "cvar95_kl": 16.976835250854492, + "delta_nll": 6.166416168212891, + "max_kl": 18.382083892822266, + "mean_kl": 6.173243999481201, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.21590909361839294 + }, + { + "candidate_nll": 8.219871520996094, + "code_tokens": 50, + "cvar95_kl": 12.589821815490723, + "delta_nll": 6.586341381072998, + "max_kl": 13.079410552978516, + "mean_kl": 6.737212657928467, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.05999999865889549 + }, + { + "candidate_nll": 6.649393081665039, + "code_tokens": 85, + "cvar95_kl": 14.358365058898926, + "delta_nll": 6.212406158447266, + "max_kl": 16.600786209106445, + "mean_kl": 6.27473783493042, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.11764705926179886 + }, + { + "candidate_nll": 6.417870998382568, + "code_tokens": 27, + "cvar95_kl": 14.772710800170898, + "delta_nll": 5.343657970428467, + "max_kl": 16.54167938232422, + "mean_kl": 5.870289325714111, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.18518517911434174 + }, + { + "candidate_nll": 6.760948657989502, + "code_tokens": 48, + "cvar95_kl": 13.504180908203125, + "delta_nll": 5.643330097198486, + "max_kl": 16.134336471557617, + "mean_kl": 5.995237827301025, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 6.584699630737305, + "code_tokens": 29, + "cvar95_kl": 13.199176788330078, + "delta_nll": 4.93685245513916, + "max_kl": 13.921878814697266, + "mean_kl": 5.426762580871582, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 7.163939952850342, + "code_tokens": 63, + "cvar95_kl": 15.430663108825684, + "delta_nll": 6.225724697113037, + "max_kl": 16.495941162109375, + "mean_kl": 6.511123180389404, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 7.261422634124756, + "code_tokens": 89, + "cvar95_kl": 15.66828441619873, + "delta_nll": 6.586363792419434, + "max_kl": 18.094545364379883, + "mean_kl": 6.66123628616333, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.10112359374761581 + }, + { + "candidate_nll": 9.470220565795898, + "code_tokens": 133, + "cvar95_kl": 17.33133316040039, + "delta_nll": 8.12594223022461, + "max_kl": 23.77682876586914, + "mean_kl": 8.248075485229492, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.06766917556524277 + }, + { + "candidate_nll": 5.355686664581299, + "code_tokens": 25, + "cvar95_kl": 10.257305145263672, + "delta_nll": 4.407149791717529, + "max_kl": 10.823506355285645, + "mean_kl": 4.440321922302246, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 5.037032604217529, + "code_tokens": 33, + "cvar95_kl": 11.216957092285156, + "delta_nll": 3.999732255935669, + "max_kl": 12.240660667419434, + "mean_kl": 4.654064655303955, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 7.377591609954834, + "code_tokens": 93, + "cvar95_kl": 17.501876831054688, + "delta_nll": 6.620816707611084, + "max_kl": 21.7191219329834, + "mean_kl": 6.574521064758301, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 7.940393924713135, + "code_tokens": 101, + "cvar95_kl": 14.832778930664062, + "delta_nll": 6.971994400024414, + "max_kl": 15.831155776977539, + "mean_kl": 6.775967121124268, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.1683168262243271 + }, + { + "candidate_nll": 8.542243957519531, + "code_tokens": 152, + "cvar95_kl": 18.61153793334961, + "delta_nll": 7.977480888366699, + "max_kl": 22.78667449951172, + "mean_kl": 8.01089096069336, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.675209045410156, + "code_tokens": 33, + "cvar95_kl": 14.410091400146484, + "delta_nll": 5.2743377685546875, + "max_kl": 15.10955810546875, + "mean_kl": 5.727102279663086, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 6.486159801483154, + "code_tokens": 29, + "cvar95_kl": 15.424844741821289, + "delta_nll": 5.199697017669678, + "max_kl": 16.021493911743164, + "mean_kl": 5.598285675048828, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.1034482792019844 + }, + { + "candidate_nll": 6.3913726806640625, + "code_tokens": 27, + "cvar95_kl": 11.755552291870117, + "delta_nll": 4.933302879333496, + "max_kl": 14.10982608795166, + "mean_kl": 4.972986221313477, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.14814814925193787 + }, + { + "candidate_nll": 4.949873924255371, + "code_tokens": 33, + "cvar95_kl": 13.388307571411133, + "delta_nll": 4.639833927154541, + "max_kl": 15.891862869262695, + "mean_kl": 4.666289329528809, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 5.350277423858643, + "code_tokens": 27, + "cvar95_kl": 10.70439338684082, + "delta_nll": 4.553879261016846, + "max_kl": 10.862907409667969, + "mean_kl": 4.623426914215088, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.126733779907227, + "code_tokens": 84, + "cvar95_kl": 16.93216896057129, + "delta_nll": 6.422840118408203, + "max_kl": 20.823768615722656, + "mean_kl": 6.572839736938477, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 8.959833145141602, + "code_tokens": 103, + "cvar95_kl": 17.11739730834961, + "delta_nll": 7.914363384246826, + "max_kl": 19.087596893310547, + "mean_kl": 7.801118850708008, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.11650485545396805 + }, + { + "candidate_nll": 8.378752708435059, + "code_tokens": 88, + "cvar95_kl": 18.154525756835938, + "delta_nll": 7.462405204772949, + "max_kl": 20.628372192382812, + "mean_kl": 7.298837661743164, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.14772726595401764 + }, + { + "candidate_nll": 8.187463760375977, + "code_tokens": 62, + "cvar95_kl": 15.356302261352539, + "delta_nll": 6.878532886505127, + "max_kl": 16.252742767333984, + "mean_kl": 6.7384772300720215, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 5.359361171722412, + "code_tokens": 24, + "cvar95_kl": 10.870662689208984, + "delta_nll": 3.8226523399353027, + "max_kl": 11.588395118713379, + "mean_kl": 3.7877323627471924, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 6.922032356262207, + "code_tokens": 34, + "cvar95_kl": 13.171253204345703, + "delta_nll": 5.583580017089844, + "max_kl": 13.286978721618652, + "mean_kl": 5.263422966003418, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.65228271484375, + "code_tokens": 43, + "cvar95_kl": 13.521621704101562, + "delta_nll": 4.932537078857422, + "max_kl": 14.849226951599121, + "mean_kl": 4.685118198394775, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.23255814611911774 + }, + { + "candidate_nll": 8.33398723602295, + "code_tokens": 54, + "cvar95_kl": 16.95148277282715, + "delta_nll": 6.844681739807129, + "max_kl": 19.035890579223633, + "mean_kl": 6.887607574462891, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.600183486938477, + "code_tokens": 50, + "cvar95_kl": 12.157979011535645, + "delta_nll": 4.870297908782959, + "max_kl": 13.114995956420898, + "mean_kl": 4.9782562255859375, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.2199999988079071 + }, + { + "candidate_nll": 9.638105392456055, + "code_tokens": 235, + "cvar95_kl": 19.159727096557617, + "delta_nll": 8.880595207214355, + "max_kl": 22.108299255371094, + "mean_kl": 8.834962844848633, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.05106382817029953 + }, + { + "candidate_nll": 7.260465145111084, + "code_tokens": 33, + "cvar95_kl": 14.864500045776367, + "delta_nll": 5.231810092926025, + "max_kl": 16.6160945892334, + "mean_kl": 5.697691917419434, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 6.845047950744629, + "code_tokens": 45, + "cvar95_kl": 15.629681587219238, + "delta_nll": 5.858545303344727, + "max_kl": 17.300352096557617, + "mean_kl": 5.758519172668457, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 6.062127590179443, + "code_tokens": 28, + "cvar95_kl": 14.11094856262207, + "delta_nll": 5.439167499542236, + "max_kl": 14.454726219177246, + "mean_kl": 5.20337438583374, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 6.982474327087402, + "code_tokens": 49, + "cvar95_kl": 14.196478843688965, + "delta_nll": 6.3042097091674805, + "max_kl": 15.295544624328613, + "mean_kl": 6.438956260681152, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.5619072914123535, + "code_tokens": 107, + "cvar95_kl": 16.049352645874023, + "delta_nll": 7.3124918937683105, + "max_kl": 18.93558692932129, + "mean_kl": 7.308789253234863, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.1214953288435936 + }, + { + "candidate_nll": 6.967126846313477, + "code_tokens": 48, + "cvar95_kl": 14.75008487701416, + "delta_nll": 5.746216773986816, + "max_kl": 16.551023483276367, + "mean_kl": 5.914834499359131, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 6.385186195373535, + "code_tokens": 40, + "cvar95_kl": 11.900189399719238, + "delta_nll": 4.545645236968994, + "max_kl": 12.158622741699219, + "mean_kl": 4.921051979064941, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.202678680419922, + "code_tokens": 46, + "cvar95_kl": 12.815601348876953, + "delta_nll": 5.733194828033447, + "max_kl": 15.393295288085938, + "mean_kl": 5.788159370422363, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.15217390656471252 + }, + { + "candidate_nll": 6.295970439910889, + "code_tokens": 31, + "cvar95_kl": 10.621480941772461, + "delta_nll": 5.253520965576172, + "max_kl": 10.858402252197266, + "mean_kl": 4.763294219970703, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 7.594814300537109, + "code_tokens": 58, + "cvar95_kl": 14.251850128173828, + "delta_nll": 7.178329944610596, + "max_kl": 15.499466896057129, + "mean_kl": 7.01655912399292, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 7.3945393562316895, + "code_tokens": 49, + "cvar95_kl": 15.890175819396973, + "delta_nll": 6.276769638061523, + "max_kl": 18.859460830688477, + "mean_kl": 6.344559669494629, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.18367347121238708 + }, + { + "candidate_nll": 3.232546806335449, + "code_tokens": 18, + "cvar95_kl": 7.709996223449707, + "delta_nll": 2.2859532833099365, + "max_kl": 7.709996223449707, + "mean_kl": 2.4263761043548584, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.894405841827393, + "code_tokens": 28, + "cvar95_kl": 10.375747680664062, + "delta_nll": 3.734994411468506, + "max_kl": 12.395173072814941, + "mean_kl": 3.6051509380340576, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 6.082608222961426, + "code_tokens": 57, + "cvar95_kl": 13.144176483154297, + "delta_nll": 5.340541362762451, + "max_kl": 13.725043296813965, + "mean_kl": 5.370726108551025, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 7.479808330535889, + "code_tokens": 88, + "cvar95_kl": 18.134130477905273, + "delta_nll": 6.779698371887207, + "max_kl": 23.937393188476562, + "mean_kl": 6.908042907714844, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.11363636702299118 + }, + { + "candidate_nll": 5.403602600097656, + "code_tokens": 23, + "cvar95_kl": 11.254439353942871, + "delta_nll": 4.204815864562988, + "max_kl": 11.783686637878418, + "mean_kl": 4.942874908447266, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 7.110675811767578, + "code_tokens": 40, + "cvar95_kl": 16.526363372802734, + "delta_nll": 5.851218223571777, + "max_kl": 17.400388717651367, + "mean_kl": 6.348052501678467, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 5.207161903381348, + "code_tokens": 20, + "cvar95_kl": 10.033302307128906, + "delta_nll": 4.045947074890137, + "max_kl": 10.033302307128906, + "mean_kl": 4.2980217933654785, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 5.5046305656433105, + "code_tokens": 63, + "cvar95_kl": 11.205473899841309, + "delta_nll": 4.488844871520996, + "max_kl": 12.309749603271484, + "mean_kl": 4.473923206329346, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 5.291966915130615, + "code_tokens": 39, + "cvar95_kl": 11.04848861694336, + "delta_nll": 4.339285850524902, + "max_kl": 11.63534164428711, + "mean_kl": 4.346642971038818, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 7.487890243530273, + "code_tokens": 50, + "cvar95_kl": 15.804692268371582, + "delta_nll": 6.076025485992432, + "max_kl": 16.323434829711914, + "mean_kl": 6.296799182891846, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 7.321791648864746, + "code_tokens": 63, + "cvar95_kl": 15.527398109436035, + "delta_nll": 6.35431432723999, + "max_kl": 19.276472091674805, + "mean_kl": 6.95456075668335, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.1269841343164444 + }, + { + "candidate_nll": 5.848536014556885, + "code_tokens": 46, + "cvar95_kl": 11.793532371520996, + "delta_nll": 4.875974178314209, + "max_kl": 12.781917572021484, + "mean_kl": 4.766500949859619, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.239130437374115 + }, + { + "candidate_nll": 5.620366096496582, + "code_tokens": 25, + "cvar95_kl": 13.87999153137207, + "delta_nll": 4.941471576690674, + "max_kl": 15.780950546264648, + "mean_kl": 4.812215805053711, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 6.726071834564209, + "code_tokens": 111, + "cvar95_kl": 13.29063892364502, + "delta_nll": 5.911806106567383, + "max_kl": 14.472166061401367, + "mean_kl": 5.981071949005127, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.14414414763450623 + }, + { + "candidate_nll": 7.416457653045654, + "code_tokens": 71, + "cvar95_kl": 13.286443710327148, + "delta_nll": 6.687899589538574, + "max_kl": 15.504067420959473, + "mean_kl": 6.398400783538818, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.1267605572938919 + }, + { + "candidate_nll": 6.637393474578857, + "code_tokens": 28, + "cvar95_kl": 13.453490257263184, + "delta_nll": 5.266669750213623, + "max_kl": 15.33100414276123, + "mean_kl": 5.876838207244873, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 5.9871649742126465, + "code_tokens": 25, + "cvar95_kl": 10.302449226379395, + "delta_nll": 4.786708831787109, + "max_kl": 10.99113655090332, + "mean_kl": 5.028310775756836, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 8.600775718688965, + "code_tokens": 40, + "cvar95_kl": 18.384765625, + "delta_nll": 7.298004150390625, + "max_kl": 18.594884872436523, + "mean_kl": 7.201474189758301, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 8.83250617980957, + "code_tokens": 177, + "cvar95_kl": 16.53539276123047, + "delta_nll": 7.78806209564209, + "max_kl": 18.444265365600586, + "mean_kl": 8.332423210144043, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.07909604161977768 + }, + { + "candidate_nll": 7.096303939819336, + "code_tokens": 51, + "cvar95_kl": 15.470919609069824, + "delta_nll": 6.005125522613525, + "max_kl": 18.425188064575195, + "mean_kl": 6.208795070648193, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.19607843458652496 + }, + { + "candidate_nll": 6.245669364929199, + "code_tokens": 69, + "cvar95_kl": 12.198820114135742, + "delta_nll": 5.791208744049072, + "max_kl": 14.21606159210205, + "mean_kl": 5.6666717529296875, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 4.369588375091553, + "code_tokens": 26, + "cvar95_kl": 10.224510192871094, + "delta_nll": 3.621328353881836, + "max_kl": 12.855220794677734, + "mean_kl": 3.466153144836426, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 5.7270941734313965, + "code_tokens": 24, + "cvar95_kl": 10.525779724121094, + "delta_nll": 4.208755016326904, + "max_kl": 11.109051704406738, + "mean_kl": 4.97491455078125, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 4.983551025390625, + "code_tokens": 15, + "cvar95_kl": 13.586735725402832, + "delta_nll": 3.8330750465393066, + "max_kl": 13.586735725402832, + "mean_kl": 3.6178321838378906, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.960725784301758, + "code_tokens": 43, + "cvar95_kl": 13.450359344482422, + "delta_nll": 4.811376571655273, + "max_kl": 14.528446197509766, + "mean_kl": 5.43659782409668, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 5.417092800140381, + "code_tokens": 18, + "cvar95_kl": 12.847880363464355, + "delta_nll": 4.324767112731934, + "max_kl": 12.847880363464355, + "mean_kl": 4.141488552093506, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 7.387372016906738, + "code_tokens": 49, + "cvar95_kl": 16.827024459838867, + "delta_nll": 6.659378528594971, + "max_kl": 18.11033821105957, + "mean_kl": 6.657153129577637, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.16326530277729034 + }, + { + "candidate_nll": 7.940781116485596, + "code_tokens": 59, + "cvar95_kl": 19.385892868041992, + "delta_nll": 7.073025226593018, + "max_kl": 19.61878204345703, + "mean_kl": 7.164468765258789, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.1355932205915451 + }, + { + "candidate_nll": 9.001656532287598, + "code_tokens": 221, + "cvar95_kl": 17.145370483398438, + "delta_nll": 8.416390419006348, + "max_kl": 20.044986724853516, + "mean_kl": 8.556669235229492, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.08144796639680862 + }, + { + "candidate_nll": 5.946462631225586, + "code_tokens": 25, + "cvar95_kl": 11.30242919921875, + "delta_nll": 4.744589805603027, + "max_kl": 13.05339241027832, + "mean_kl": 5.265052318572998, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.395029067993164, + "code_tokens": 36, + "cvar95_kl": 14.700695037841797, + "delta_nll": 5.671108245849609, + "max_kl": 16.22500228881836, + "mean_kl": 5.913853168487549, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.1944444477558136 + }, + { + "candidate_nll": 9.197209358215332, + "code_tokens": 158, + "cvar95_kl": 18.24317741394043, + "delta_nll": 8.553064346313477, + "max_kl": 20.82396125793457, + "mean_kl": 8.587793350219727, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.1265822798013687 + }, + { + "candidate_nll": 9.439599990844727, + "code_tokens": 92, + "cvar95_kl": 19.128944396972656, + "delta_nll": 8.124065399169922, + "max_kl": 23.061622619628906, + "mean_kl": 7.755072593688965, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.10869564861059189 + }, + { + "candidate_nll": 8.294965744018555, + "code_tokens": 102, + "cvar95_kl": 17.25802993774414, + "delta_nll": 7.380923271179199, + "max_kl": 21.93653678894043, + "mean_kl": 7.633340358734131, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.0784313753247261 + }, + { + "candidate_nll": 9.1343412399292, + "code_tokens": 171, + "cvar95_kl": 18.297454833984375, + "delta_nll": 8.524371147155762, + "max_kl": 21.73289680480957, + "mean_kl": 8.450557708740234, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.0760233923792839 + }, + { + "candidate_nll": 6.534271240234375, + "code_tokens": 40, + "cvar95_kl": 16.111255645751953, + "delta_nll": 5.329407215118408, + "max_kl": 17.58118438720703, + "mean_kl": 6.39892053604126, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 6.427910327911377, + "code_tokens": 48, + "cvar95_kl": 12.440707206726074, + "delta_nll": 5.191896438598633, + "max_kl": 12.565178871154785, + "mean_kl": 5.948092937469482, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.394636631011963, + "code_tokens": 24, + "cvar95_kl": 11.619314193725586, + "delta_nll": 5.7897162437438965, + "max_kl": 12.080916404724121, + "mean_kl": 5.9807515144348145, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 7.749431610107422, + "code_tokens": 48, + "cvar95_kl": 17.275714874267578, + "delta_nll": 6.762327194213867, + "max_kl": 21.255064010620117, + "mean_kl": 6.812342166900635, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 6.745427131652832, + "code_tokens": 25, + "cvar95_kl": 13.303813934326172, + "delta_nll": 5.358543872833252, + "max_kl": 14.131097793579102, + "mean_kl": 4.918761730194092, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 5.66212272644043, + "code_tokens": 30, + "cvar95_kl": 9.788607597351074, + "delta_nll": 4.16337776184082, + "max_kl": 10.453560829162598, + "mean_kl": 3.9204726219177246, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 5.399513244628906, + "code_tokens": 19, + "cvar95_kl": 10.992709159851074, + "delta_nll": 3.9726953506469727, + "max_kl": 10.992709159851074, + "mean_kl": 4.087630748748779, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.5959908962249756, + "code_tokens": 12, + "cvar95_kl": 7.146435260772705, + "delta_nll": 2.516918420791626, + "max_kl": 7.146435260772705, + "mean_kl": 2.9206676483154297, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 7.270519256591797, + "code_tokens": 37, + "cvar95_kl": 14.788522720336914, + "delta_nll": 5.930209636688232, + "max_kl": 15.391535758972168, + "mean_kl": 6.0127272605896, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 5.709114074707031, + "code_tokens": 33, + "cvar95_kl": 15.044073104858398, + "delta_nll": 4.503684043884277, + "max_kl": 17.768451690673828, + "mean_kl": 5.456212520599365, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 8.483305931091309, + "code_tokens": 94, + "cvar95_kl": 18.111263275146484, + "delta_nll": 7.631196022033691, + "max_kl": 19.372024536132812, + "mean_kl": 7.778371810913086, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.09574468433856964 + }, + { + "candidate_nll": 7.494216442108154, + "code_tokens": 90, + "cvar95_kl": 15.836697578430176, + "delta_nll": 6.869451522827148, + "max_kl": 17.483427047729492, + "mean_kl": 6.653045654296875, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.13333334028720856 + }, + { + "candidate_nll": 7.8163347244262695, + "code_tokens": 47, + "cvar95_kl": 14.708568572998047, + "delta_nll": 6.368570804595947, + "max_kl": 15.449639320373535, + "mean_kl": 6.443413257598877, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.1702127605676651 + }, + { + "candidate_nll": 6.816396236419678, + "code_tokens": 61, + "cvar95_kl": 14.21721076965332, + "delta_nll": 5.892561912536621, + "max_kl": 17.487207412719727, + "mean_kl": 6.104360103607178, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.16393442451953888 + }, + { + "candidate_nll": 6.296998977661133, + "code_tokens": 39, + "cvar95_kl": 14.180032730102539, + "delta_nll": 5.073484420776367, + "max_kl": 15.074982643127441, + "mean_kl": 5.804995536804199, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 8.610564231872559, + "code_tokens": 152, + "cvar95_kl": 16.556926727294922, + "delta_nll": 7.678402423858643, + "max_kl": 19.855178833007812, + "mean_kl": 7.9938249588012695, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.09210526198148727 + }, + { + "candidate_nll": 8.728375434875488, + "code_tokens": 100, + "cvar95_kl": 17.90008544921875, + "delta_nll": 7.759393215179443, + "max_kl": 21.89253807067871, + "mean_kl": 7.664051532745361, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.05999999865889549 + }, + { + "candidate_nll": 9.550278663635254, + "code_tokens": 89, + "cvar95_kl": 18.254901885986328, + "delta_nll": 8.929350852966309, + "max_kl": 19.850004196166992, + "mean_kl": 8.724377632141113, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.08988764137029648 + }, + { + "candidate_nll": 6.4178466796875, + "code_tokens": 40, + "cvar95_kl": 13.016363143920898, + "delta_nll": 5.106845378875732, + "max_kl": 14.093358039855957, + "mean_kl": 5.368229389190674, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.908441066741943, + "code_tokens": 76, + "cvar95_kl": 16.402854919433594, + "delta_nll": 7.206873893737793, + "max_kl": 17.818492889404297, + "mean_kl": 7.207874298095703, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.10526315867900848 + }, + { + "candidate_nll": 6.232840538024902, + "code_tokens": 56, + "cvar95_kl": 11.934481620788574, + "delta_nll": 4.983853340148926, + "max_kl": 12.800703048706055, + "mean_kl": 5.218331336975098, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 6.590324878692627, + "code_tokens": 34, + "cvar95_kl": 13.076437950134277, + "delta_nll": 5.100894451141357, + "max_kl": 13.498030662536621, + "mean_kl": 5.206934928894043, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.14705882966518402 + }, + { + "candidate_nll": 9.33968734741211, + "code_tokens": 96, + "cvar95_kl": 18.49220085144043, + "delta_nll": 8.661576271057129, + "max_kl": 19.822507858276367, + "mean_kl": 8.552289009094238, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.1041666641831398 + }, + { + "candidate_nll": 7.917817115783691, + "code_tokens": 451, + "cvar95_kl": 16.363418579101562, + "delta_nll": 7.219121932983398, + "max_kl": 20.659109115600586, + "mean_kl": 7.17897367477417, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.06430155038833618 + }, + { + "candidate_nll": 6.6869001388549805, + "code_tokens": 37, + "cvar95_kl": 13.637319564819336, + "delta_nll": 5.074590682983398, + "max_kl": 14.098847389221191, + "mean_kl": 5.874645233154297, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 6.401904106140137, + "code_tokens": 42, + "cvar95_kl": 14.770770072937012, + "delta_nll": 5.160305976867676, + "max_kl": 19.26398468017578, + "mean_kl": 6.109968662261963, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.345864295959473, + "code_tokens": 33, + "cvar95_kl": 13.185270309448242, + "delta_nll": 6.329108238220215, + "max_kl": 14.037763595581055, + "mean_kl": 6.042445659637451, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 6.525254249572754, + "code_tokens": 43, + "cvar95_kl": 13.34843921661377, + "delta_nll": 5.599202632904053, + "max_kl": 15.172074317932129, + "mean_kl": 5.6737542152404785, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.23255814611911774 + }, + { + "candidate_nll": 6.268006324768066, + "code_tokens": 73, + "cvar95_kl": 12.651838302612305, + "delta_nll": 5.531569957733154, + "max_kl": 12.988868713378906, + "mean_kl": 5.815316677093506, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.15068493783473969 + }, + { + "candidate_nll": 4.592410564422607, + "code_tokens": 15, + "cvar95_kl": 13.666496276855469, + "delta_nll": 3.679934501647949, + "max_kl": 13.666496276855469, + "mean_kl": 3.6762516498565674, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 6.2513885498046875, + "code_tokens": 34, + "cvar95_kl": 12.027372360229492, + "delta_nll": 4.992308616638184, + "max_kl": 13.35857105255127, + "mean_kl": 5.4397664070129395, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 8.790428161621094, + "code_tokens": 136, + "cvar95_kl": 17.49091148376465, + "delta_nll": 7.681944847106934, + "max_kl": 20.574209213256836, + "mean_kl": 7.79193115234375, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.0882352963089943 + }, + { + "candidate_nll": 4.0699462890625, + "code_tokens": 17, + "cvar95_kl": 7.8968000411987305, + "delta_nll": 2.8237900733947754, + "max_kl": 7.8968000411987305, + "mean_kl": 3.437905788421631, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 5.590200424194336, + "code_tokens": 33, + "cvar95_kl": 12.146036148071289, + "delta_nll": 4.797145843505859, + "max_kl": 12.5106201171875, + "mean_kl": 4.762210845947266, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 6.065141201019287, + "code_tokens": 38, + "cvar95_kl": 12.040993690490723, + "delta_nll": 5.222694396972656, + "max_kl": 12.334887504577637, + "mean_kl": 4.953282833099365, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 6.526878356933594, + "code_tokens": 61, + "cvar95_kl": 14.92432689666748, + "delta_nll": 6.1293535232543945, + "max_kl": 16.628849029541016, + "mean_kl": 6.1552300453186035, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.21311475336551666 + }, + { + "candidate_nll": 7.9373931884765625, + "code_tokens": 143, + "cvar95_kl": 14.740462303161621, + "delta_nll": 7.20167350769043, + "max_kl": 19.49589729309082, + "mean_kl": 7.167318344116211, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.07692307978868484 + }, + { + "candidate_nll": 7.8792619705200195, + "code_tokens": 41, + "cvar95_kl": 15.280052185058594, + "delta_nll": 6.790195941925049, + "max_kl": 16.26609230041504, + "mean_kl": 6.988038063049316, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.1463414579629898 + }, + { + "candidate_nll": 6.3456268310546875, + "code_tokens": 67, + "cvar95_kl": 13.915311813354492, + "delta_nll": 5.980945110321045, + "max_kl": 15.56528377532959, + "mean_kl": 5.991087913513184, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.16417910158634186 + }, + { + "candidate_nll": 7.517149448394775, + "code_tokens": 76, + "cvar95_kl": 16.199596405029297, + "delta_nll": 6.126889228820801, + "max_kl": 18.642292022705078, + "mean_kl": 6.213190078735352, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.17105263471603394 + }, + { + "candidate_nll": 8.672646522521973, + "code_tokens": 137, + "cvar95_kl": 17.744823455810547, + "delta_nll": 8.03679370880127, + "max_kl": 23.592477798461914, + "mean_kl": 8.029720306396484, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.08029197156429291 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2341", + "rounding": "stochastic", + "seed": 2341, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 9.750813484191895, + "median": 5.779341459274292, + "minimum": 1.5677480697631836, + "q25": 4.82051694393158, + "q75": 6.731091022491455 + }, + "task_macro": { + "candidate_nll": 6.802963601112365, + "delta_nll": 5.788917013168335, + "mean_kl": 5.879288683652878, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.1962257090508938 + }, + "token_weighted": { + "candidate_nll": 7.449468612670898, + "cvar95_kl": 16.641014099121094, + "delta_nll": 6.532630920410156, + "max_kl": 27.844261169433594, + "mean_kl": 6.598196029663086, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.1547745019197464 + } + }, + "uniform_int4_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.523714332580566, + "macro_mean_kl": 2.6566686334609986, + "macro_top1_agreement": 0.3876878571510315, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 2.9709696474075318, + "macro_mean_kl": 3.1206523551940917, + "macro_top1_agreement": 0.31672936415672304, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 3.0704434728622436, + "macro_mean_kl": 3.1326974391937257, + "macro_top1_agreement": 0.3173918128013611, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 3.233842721939087, + "macro_mean_kl": 3.2533009090423586, + "macro_top1_agreement": 0.3103079069256783, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 3.7664122581481934, + "code_tokens": 98, + "cvar95_kl": 8.38406753540039, + "delta_nll": 3.0395565032958984, + "max_kl": 10.258127212524414, + "mean_kl": 3.394542694091797, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.33673468232154846 + }, + { + "candidate_nll": 4.149313449859619, + "code_tokens": 19, + "cvar95_kl": 8.947985649108887, + "delta_nll": 2.9899003505706787, + "max_kl": 8.947985649108887, + "mean_kl": 3.2295119762420654, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 3.891326427459717, + "code_tokens": 39, + "cvar95_kl": 10.0741605758667, + "delta_nll": 2.875977039337158, + "max_kl": 12.670684814453125, + "mean_kl": 3.260664463043213, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 4.744483470916748, + "code_tokens": 24, + "cvar95_kl": 6.9792985916137695, + "delta_nll": 3.29819393157959, + "max_kl": 7.633026123046875, + "mean_kl": 3.202848434448242, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.8999876976013184, + "code_tokens": 25, + "cvar95_kl": 8.560966491699219, + "delta_nll": 2.492229461669922, + "max_kl": 10.639869689941406, + "mean_kl": 2.446340560913086, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.493342876434326, + "code_tokens": 55, + "cvar95_kl": 9.399588584899902, + "delta_nll": 3.597731351852417, + "max_kl": 12.972951889038086, + "mean_kl": 3.4826667308807373, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.30909091234207153 + }, + { + "candidate_nll": 3.192394256591797, + "code_tokens": 16, + "cvar95_kl": 5.85267448425293, + "delta_nll": 2.3629536628723145, + "max_kl": 5.85267448425293, + "mean_kl": 2.256340980529785, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 5.062644958496094, + "code_tokens": 199, + "cvar95_kl": 10.03557014465332, + "delta_nll": 3.8407516479492188, + "max_kl": 14.8927001953125, + "mean_kl": 3.8721749782562256, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.23115578293800354 + }, + { + "candidate_nll": 4.431500434875488, + "code_tokens": 28, + "cvar95_kl": 6.362828254699707, + "delta_nll": 2.865422248840332, + "max_kl": 6.479719161987305, + "mean_kl": 2.7046711444854736, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.7658004760742188, + "code_tokens": 87, + "cvar95_kl": 7.893912315368652, + "delta_nll": 2.892817974090576, + "max_kl": 9.843452453613281, + "mean_kl": 2.8645589351654053, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.29885056614875793 + }, + { + "candidate_nll": 3.8566553592681885, + "code_tokens": 35, + "cvar95_kl": 9.331412315368652, + "delta_nll": 2.6236910820007324, + "max_kl": 9.351311683654785, + "mean_kl": 3.5940160751342773, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.823774814605713, + "code_tokens": 62, + "cvar95_kl": 9.140983581542969, + "delta_nll": 3.184483289718628, + "max_kl": 10.529592514038086, + "mean_kl": 3.092163562774658, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 5.359539985656738, + "code_tokens": 58, + "cvar95_kl": 10.713452339172363, + "delta_nll": 4.254482269287109, + "max_kl": 11.073169708251953, + "mean_kl": 4.141488552093506, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 4.255224704742432, + "code_tokens": 79, + "cvar95_kl": 7.321687698364258, + "delta_nll": 3.0961766242980957, + "max_kl": 7.591400146484375, + "mean_kl": 3.3038218021392822, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.2531645596027374 + }, + { + "candidate_nll": 3.8157670497894287, + "code_tokens": 58, + "cvar95_kl": 6.812357425689697, + "delta_nll": 2.722647190093994, + "max_kl": 7.4202189445495605, + "mean_kl": 2.8616652488708496, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 4.844391345977783, + "code_tokens": 48, + "cvar95_kl": 8.299520492553711, + "delta_nll": 3.6821935176849365, + "max_kl": 9.358929634094238, + "mean_kl": 3.9137237071990967, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 4.425922393798828, + "code_tokens": 37, + "cvar95_kl": 9.267054557800293, + "delta_nll": 3.300489902496338, + "max_kl": 10.764337539672852, + "mean_kl": 3.1025164127349854, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.5322911739349365, + "code_tokens": 61, + "cvar95_kl": 7.575699806213379, + "delta_nll": 2.8248839378356934, + "max_kl": 11.057958602905273, + "mean_kl": 2.847332000732422, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.32786884903907776 + }, + { + "candidate_nll": 3.7819926738739014, + "code_tokens": 77, + "cvar95_kl": 6.878523826599121, + "delta_nll": 3.012273073196411, + "max_kl": 7.46014928817749, + "mean_kl": 2.8802757263183594, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.475198745727539, + "code_tokens": 98, + "cvar95_kl": 8.037261009216309, + "delta_nll": 3.453495502471924, + "max_kl": 10.399234771728516, + "mean_kl": 3.3554909229278564, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.30612245202064514 + }, + { + "candidate_nll": 5.357379913330078, + "code_tokens": 162, + "cvar95_kl": 10.442813873291016, + "delta_nll": 4.772228717803955, + "max_kl": 12.562193870544434, + "mean_kl": 4.820985317230225, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.12345679104328156 + }, + { + "candidate_nll": 4.62529182434082, + "code_tokens": 103, + "cvar95_kl": 9.557621955871582, + "delta_nll": 3.8808257579803467, + "max_kl": 10.936281204223633, + "mean_kl": 3.8761308193206787, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.24271844327449799 + }, + { + "candidate_nll": 5.040172576904297, + "code_tokens": 73, + "cvar95_kl": 7.74887228012085, + "delta_nll": 3.7212021350860596, + "max_kl": 8.244718551635742, + "mean_kl": 3.487346887588501, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.3287671208381653 + }, + { + "candidate_nll": 4.642014026641846, + "code_tokens": 119, + "cvar95_kl": 9.844925880432129, + "delta_nll": 3.6590404510498047, + "max_kl": 13.877107620239258, + "mean_kl": 3.783954620361328, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.19327731430530548 + }, + { + "candidate_nll": 2.379567861557007, + "code_tokens": 17, + "cvar95_kl": 6.902372360229492, + "delta_nll": 1.1900445222854614, + "max_kl": 6.902372360229492, + "mean_kl": 2.173980236053467, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.034422397613525, + "code_tokens": 56, + "cvar95_kl": 8.087773323059082, + "delta_nll": 2.6087207794189453, + "max_kl": 8.598217010498047, + "mean_kl": 2.706983804702759, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.3392857015132904 + }, + { + "candidate_nll": 3.627689838409424, + "code_tokens": 57, + "cvar95_kl": 8.029515266418457, + "delta_nll": 3.1603293418884277, + "max_kl": 8.393468856811523, + "mean_kl": 3.0816924571990967, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.35087719559669495 + }, + { + "candidate_nll": 3.672466516494751, + "code_tokens": 63, + "cvar95_kl": 8.205222129821777, + "delta_nll": 3.0397212505340576, + "max_kl": 8.763727188110352, + "mean_kl": 3.0795977115631104, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 0.4126984179019928 + }, + { + "candidate_nll": 5.310922622680664, + "code_tokens": 200, + "cvar95_kl": 12.095563888549805, + "delta_nll": 4.472987174987793, + "max_kl": 14.60595703125, + "mean_kl": 4.534121990203857, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.23000000417232513 + }, + { + "candidate_nll": 4.181536674499512, + "code_tokens": 30, + "cvar95_kl": 7.999033451080322, + "delta_nll": 3.0973339080810547, + "max_kl": 8.434910774230957, + "mean_kl": 3.207280397415161, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.7458109855651855, + "code_tokens": 28, + "cvar95_kl": 11.766042709350586, + "delta_nll": 3.129459857940674, + "max_kl": 12.460432052612305, + "mean_kl": 2.9581243991851807, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.6298487186431885, + "code_tokens": 26, + "cvar95_kl": 6.9225850105285645, + "delta_nll": 2.8588919639587402, + "max_kl": 7.256990909576416, + "mean_kl": 2.7480664253234863, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.3608033657073975, + "code_tokens": 54, + "cvar95_kl": 7.426509857177734, + "delta_nll": 2.527862071990967, + "max_kl": 9.804802894592285, + "mean_kl": 2.533320188522339, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 3.2992470264434814, + "code_tokens": 47, + "cvar95_kl": 5.38717794418335, + "delta_nll": 2.342510938644409, + "max_kl": 6.0475616455078125, + "mean_kl": 2.4189908504486084, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.3404255211353302 + }, + { + "candidate_nll": 4.257634162902832, + "code_tokens": 93, + "cvar95_kl": 8.440088272094727, + "delta_nll": 3.523104667663574, + "max_kl": 9.631627082824707, + "mean_kl": 3.2952053546905518, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.31182795763015747 + }, + { + "candidate_nll": 3.8380072116851807, + "code_tokens": 30, + "cvar95_kl": 8.992044448852539, + "delta_nll": 2.978632688522339, + "max_kl": 10.486231803894043, + "mean_kl": 2.982222557067871, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.268701553344727, + "code_tokens": 89, + "cvar95_kl": 7.669314384460449, + "delta_nll": 3.200198173522949, + "max_kl": 9.174586296081543, + "mean_kl": 3.2235395908355713, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.2247191071510315 + }, + { + "candidate_nll": 4.515193462371826, + "code_tokens": 70, + "cvar95_kl": 9.276607513427734, + "delta_nll": 3.5509440898895264, + "max_kl": 11.100701332092285, + "mean_kl": 3.7310128211975098, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.3287577629089355, + "code_tokens": 26, + "cvar95_kl": 6.706403732299805, + "delta_nll": 2.7155306339263916, + "max_kl": 6.838807582855225, + "mean_kl": 3.2019741535186768, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.596815824508667, + "code_tokens": 48, + "cvar95_kl": 9.1279878616333, + "delta_nll": 2.639108896255493, + "max_kl": 12.232829093933105, + "mean_kl": 2.8147497177124023, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.2708333432674408 + }, + { + "candidate_nll": 3.317744493484497, + "code_tokens": 29, + "cvar95_kl": 7.680596828460693, + "delta_nll": 2.448596477508545, + "max_kl": 8.038989067077637, + "mean_kl": 2.6000916957855225, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 0.4137931168079376 + }, + { + "candidate_nll": 4.116034984588623, + "code_tokens": 17, + "cvar95_kl": 7.639901161193848, + "delta_nll": 3.0028719902038574, + "max_kl": 7.639901161193848, + "mean_kl": 2.9017791748046875, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 3.0927469730377197, + "code_tokens": 37, + "cvar95_kl": 6.885171890258789, + "delta_nll": 2.1725308895111084, + "max_kl": 7.109859466552734, + "mean_kl": 2.2919492721557617, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.3513513505458832 + }, + { + "candidate_nll": 5.122945308685303, + "code_tokens": 121, + "cvar95_kl": 11.932059288024902, + "delta_nll": 4.611543655395508, + "max_kl": 16.249616622924805, + "mean_kl": 4.560413837432861, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.20661157369613647 + }, + { + "candidate_nll": 3.9537606239318848, + "code_tokens": 30, + "cvar95_kl": 9.024372100830078, + "delta_nll": 2.5549168586730957, + "max_kl": 11.036246299743652, + "mean_kl": 2.829641819000244, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.071406364440918, + "code_tokens": 74, + "cvar95_kl": 8.209741592407227, + "delta_nll": 3.1729345321655273, + "max_kl": 9.09450912475586, + "mean_kl": 3.2202224731445312, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.546560287475586, + "code_tokens": 53, + "cvar95_kl": 7.649753570556641, + "delta_nll": 2.7121365070343018, + "max_kl": 7.869818687438965, + "mean_kl": 2.625974178314209, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 0.37735849618911743 + }, + { + "candidate_nll": 3.9406607151031494, + "code_tokens": 21, + "cvar95_kl": 5.468418598175049, + "delta_nll": 2.0812649726867676, + "max_kl": 5.550998687744141, + "mean_kl": 2.4146623611450195, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.772265911102295, + "code_tokens": 20, + "cvar95_kl": 7.139959812164307, + "delta_nll": 1.4098522663116455, + "max_kl": 7.139959812164307, + "mean_kl": 1.5119413137435913, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 3.2215187549591064, + "code_tokens": 122, + "cvar95_kl": 5.963348865509033, + "delta_nll": 2.4025204181671143, + "max_kl": 7.232659816741943, + "mean_kl": 2.429250717163086, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.37704917788505554 + }, + { + "candidate_nll": 4.165090084075928, + "code_tokens": 94, + "cvar95_kl": 8.356420516967773, + "delta_nll": 3.3249502182006836, + "max_kl": 8.975976943969727, + "mean_kl": 3.382554054260254, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.28723403811454773 + }, + { + "candidate_nll": 2.9728457927703857, + "code_tokens": 12, + "cvar95_kl": 5.312210559844971, + "delta_nll": 1.9075442552566528, + "max_kl": 5.312210559844971, + "mean_kl": 1.75045645236969, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.991575717926025, + "code_tokens": 38, + "cvar95_kl": 8.959644317626953, + "delta_nll": 3.485175609588623, + "max_kl": 9.423612594604492, + "mean_kl": 3.9555130004882812, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 2.117633819580078, + "code_tokens": 58, + "cvar95_kl": 5.960376262664795, + "delta_nll": 1.2128467559814453, + "max_kl": 6.718116283416748, + "mean_kl": 1.3771212100982666, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 0.5517241358757019 + }, + { + "candidate_nll": 4.910107135772705, + "code_tokens": 54, + "cvar95_kl": 9.73076343536377, + "delta_nll": 4.313843250274658, + "max_kl": 11.136575698852539, + "mean_kl": 4.2481913566589355, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.66205096244812, + "code_tokens": 40, + "cvar95_kl": 7.012556076049805, + "delta_nll": 2.73051118850708, + "max_kl": 7.201176166534424, + "mean_kl": 3.032318592071533, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.4418506622314453, + "code_tokens": 115, + "cvar95_kl": 7.656381607055664, + "delta_nll": 2.929304599761963, + "max_kl": 10.029156684875488, + "mean_kl": 2.9149982929229736, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.3391304314136505 + }, + { + "candidate_nll": 3.907224178314209, + "code_tokens": 60, + "cvar95_kl": 9.032503128051758, + "delta_nll": 3.382610559463501, + "max_kl": 10.235370635986328, + "mean_kl": 3.451953411102295, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 3.930058240890503, + "code_tokens": 124, + "cvar95_kl": 8.1185884475708, + "delta_nll": 3.1600711345672607, + "max_kl": 9.828448295593262, + "mean_kl": 3.135624408721924, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.292812347412109, + "code_tokens": 91, + "cvar95_kl": 11.584701538085938, + "delta_nll": 4.522063732147217, + "max_kl": 12.021390914916992, + "mean_kl": 4.505959510803223, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.16483516991138458 + }, + { + "candidate_nll": 5.907611846923828, + "code_tokens": 125, + "cvar95_kl": 12.780604362487793, + "delta_nll": 5.0876946449279785, + "max_kl": 14.524517059326172, + "mean_kl": 5.146956920623779, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.18400000035762787 + }, + { + "candidate_nll": 2.5905098915100098, + "code_tokens": 26, + "cvar95_kl": 4.356642723083496, + "delta_nll": 1.454633355140686, + "max_kl": 4.392017364501953, + "mean_kl": 1.9260729551315308, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 3.9847633838653564, + "code_tokens": 24, + "cvar95_kl": 7.428703308105469, + "delta_nll": 1.7443382740020752, + "max_kl": 8.041558265686035, + "mean_kl": 2.1978509426116943, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.619900941848755, + "code_tokens": 180, + "cvar95_kl": 8.591968536376953, + "delta_nll": 2.765719175338745, + "max_kl": 11.978545188903809, + "mean_kl": 2.80114483833313, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.33888888359069824 + }, + { + "candidate_nll": 5.233104228973389, + "code_tokens": 40, + "cvar95_kl": 9.967483520507812, + "delta_nll": 3.8582956790924072, + "max_kl": 11.625901222229004, + "mean_kl": 3.966789960861206, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.098239421844482, + "code_tokens": 72, + "cvar95_kl": 8.667765617370605, + "delta_nll": 3.1232426166534424, + "max_kl": 9.659649848937988, + "mean_kl": 3.1194727420806885, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.2638888955116272 + }, + { + "candidate_nll": 2.9412615299224854, + "code_tokens": 20, + "cvar95_kl": 4.5153045654296875, + "delta_nll": 1.3636903762817383, + "max_kl": 4.5153045654296875, + "mean_kl": 1.8786125183105469, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.2834954261779785, + "code_tokens": 76, + "cvar95_kl": 7.880440711975098, + "delta_nll": 3.292850971221924, + "max_kl": 9.166062355041504, + "mean_kl": 3.370410203933716, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 4.528679847717285, + "code_tokens": 42, + "cvar95_kl": 9.591974258422852, + "delta_nll": 3.1569385528564453, + "max_kl": 10.1939697265625, + "mean_kl": 3.1520988941192627, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.261904776096344 + }, + { + "candidate_nll": 2.442310094833374, + "code_tokens": 28, + "cvar95_kl": 6.308202743530273, + "delta_nll": 1.8590869903564453, + "max_kl": 6.390542030334473, + "mean_kl": 1.7288532257080078, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 0.6428571343421936 + }, + { + "candidate_nll": 4.790449142456055, + "code_tokens": 58, + "cvar95_kl": 10.5787992477417, + "delta_nll": 3.756972312927246, + "max_kl": 10.616368293762207, + "mean_kl": 3.864464521408081, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 3.8505160808563232, + "code_tokens": 25, + "cvar95_kl": 7.605593681335449, + "delta_nll": 2.9818665981292725, + "max_kl": 8.119839668273926, + "mean_kl": 3.246250867843628, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.626301288604736, + "code_tokens": 88, + "cvar95_kl": 8.688606262207031, + "delta_nll": 3.6612935066223145, + "max_kl": 9.43653392791748, + "mean_kl": 3.6825876235961914, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.2613636255264282 + }, + { + "candidate_nll": 3.632288694381714, + "code_tokens": 42, + "cvar95_kl": 6.7347731590271, + "delta_nll": 2.5047714710235596, + "max_kl": 7.440560340881348, + "mean_kl": 2.343937397003174, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.4047619104385376 + }, + { + "candidate_nll": 3.8622360229492188, + "code_tokens": 24, + "cvar95_kl": 6.684353828430176, + "delta_nll": 3.0426852703094482, + "max_kl": 7.178956031799316, + "mean_kl": 3.0226192474365234, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.847337007522583, + "code_tokens": 24, + "cvar95_kl": 5.261401653289795, + "delta_nll": 2.1624655723571777, + "max_kl": 5.291695594787598, + "mean_kl": 2.2087502479553223, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.381443738937378, + "code_tokens": 43, + "cvar95_kl": 7.580766201019287, + "delta_nll": 2.3142647743225098, + "max_kl": 8.971184730529785, + "mean_kl": 2.256549835205078, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 0.5116279125213623 + }, + { + "candidate_nll": 3.467910051345825, + "code_tokens": 24, + "cvar95_kl": 9.238639831542969, + "delta_nll": 2.5816903114318848, + "max_kl": 9.809638023376465, + "mean_kl": 2.779672861099243, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.751603364944458, + "code_tokens": 14, + "cvar95_kl": 4.877512454986572, + "delta_nll": 2.4389777183532715, + "max_kl": 4.877512454986572, + "mean_kl": 2.4515538215637207, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 3.7782461643218994, + "code_tokens": 41, + "cvar95_kl": 7.268362522125244, + "delta_nll": 3.287202835083008, + "max_kl": 8.485128402709961, + "mean_kl": 2.9382009506225586, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.3658536672592163 + }, + { + "candidate_nll": 4.438844680786133, + "code_tokens": 32, + "cvar95_kl": 9.277170181274414, + "delta_nll": 3.450504779815674, + "max_kl": 10.04997730255127, + "mean_kl": 3.3922677040100098, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 2.7914841175079346, + "code_tokens": 53, + "cvar95_kl": 6.117984294891357, + "delta_nll": 2.0297892093658447, + "max_kl": 6.980735778808594, + "mean_kl": 1.8949875831604004, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 0.5094339847564697 + }, + { + "candidate_nll": 3.260664701461792, + "code_tokens": 51, + "cvar95_kl": 8.19973087310791, + "delta_nll": 2.882373571395874, + "max_kl": 8.525491714477539, + "mean_kl": 2.8704683780670166, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 4.405463218688965, + "code_tokens": 36, + "cvar95_kl": 7.061243057250977, + "delta_nll": 2.7133078575134277, + "max_kl": 7.1261749267578125, + "mean_kl": 2.9145424365997314, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 3.525358200073242, + "code_tokens": 24, + "cvar95_kl": 5.177626609802246, + "delta_nll": 1.9646059274673462, + "max_kl": 5.3324713706970215, + "mean_kl": 2.070081949234009, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.899878978729248, + "code_tokens": 39, + "cvar95_kl": 9.809728622436523, + "delta_nll": 3.5855445861816406, + "max_kl": 10.055256843566895, + "mean_kl": 4.037482738494873, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 5.758738994598389, + "code_tokens": 84, + "cvar95_kl": 10.550141334533691, + "delta_nll": 5.06032657623291, + "max_kl": 15.882719993591309, + "mean_kl": 5.104015350341797, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 3.2006826400756836, + "code_tokens": 32, + "cvar95_kl": 6.10108757019043, + "delta_nll": 2.277914524078369, + "max_kl": 6.253726005554199, + "mean_kl": 2.1592047214508057, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.505847454071045, + "code_tokens": 19, + "cvar95_kl": 10.207890510559082, + "delta_nll": 3.1956839561462402, + "max_kl": 10.207890510559082, + "mean_kl": 3.1247029304504395, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 0.5263158082962036 + }, + { + "candidate_nll": 3.897122859954834, + "code_tokens": 50, + "cvar95_kl": 8.90195369720459, + "delta_nll": 3.079146146774292, + "max_kl": 10.412129402160645, + "mean_kl": 3.0039680004119873, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.2848048210144043, + "code_tokens": 71, + "cvar95_kl": 7.633419990539551, + "delta_nll": 2.7849912643432617, + "max_kl": 11.858368873596191, + "mean_kl": 2.691610813140869, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.4225352108478546 + }, + { + "candidate_nll": 4.337283611297607, + "code_tokens": 28, + "cvar95_kl": 9.74201774597168, + "delta_nll": 2.949315071105957, + "max_kl": 9.982402801513672, + "mean_kl": 3.2217249870300293, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 2.9190003871917725, + "code_tokens": 76, + "cvar95_kl": 6.007754325866699, + "delta_nll": 2.049738883972168, + "max_kl": 7.286333084106445, + "mean_kl": 2.0321688652038574, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.43421053886413574 + }, + { + "candidate_nll": 3.8200221061706543, + "code_tokens": 34, + "cvar95_kl": 6.370345115661621, + "delta_nll": 2.8574368953704834, + "max_kl": 6.555943012237549, + "mean_kl": 2.9013257026672363, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 4.174374580383301, + "code_tokens": 11, + "cvar95_kl": 4.35053825378418, + "delta_nll": 2.0269827842712402, + "max_kl": 4.35053825378418, + "mean_kl": 1.997620701789856, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 3.623605251312256, + "code_tokens": 31, + "cvar95_kl": 7.949816703796387, + "delta_nll": 2.2073378562927246, + "max_kl": 8.425786972045898, + "mean_kl": 2.7997424602508545, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.32258063554763794 + }, + { + "candidate_nll": 3.273397445678711, + "code_tokens": 118, + "cvar95_kl": 6.517803192138672, + "delta_nll": 2.426621198654175, + "max_kl": 7.703571319580078, + "mean_kl": 2.4185972213745117, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.3644067943096161 + }, + { + "candidate_nll": 2.8282949924468994, + "code_tokens": 60, + "cvar95_kl": 9.896921157836914, + "delta_nll": 2.0525994300842285, + "max_kl": 12.209373474121094, + "mean_kl": 2.3485896587371826, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 4.1720685958862305, + "code_tokens": 47, + "cvar95_kl": 7.402275085449219, + "delta_nll": 3.276618719100952, + "max_kl": 8.12134838104248, + "mean_kl": 3.366013288497925, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.23404255509376526 + }, + { + "candidate_nll": 3.8756513595581055, + "code_tokens": 84, + "cvar95_kl": 7.992628574371338, + "delta_nll": 2.723637104034424, + "max_kl": 10.890752792358398, + "mean_kl": 2.8173635005950928, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.4047619104385376 + }, + { + "candidate_nll": 4.770659923553467, + "code_tokens": 27, + "cvar95_kl": 10.533048629760742, + "delta_nll": 3.7766337394714355, + "max_kl": 11.212759017944336, + "mean_kl": 3.6659109592437744, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.7506837844848633, + "code_tokens": 19, + "cvar95_kl": 5.9892659187316895, + "delta_nll": 2.1387767791748047, + "max_kl": 5.9892659187316895, + "mean_kl": 2.097929000854492, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 3.9867894649505615, + "code_tokens": 114, + "cvar95_kl": 9.246536254882812, + "delta_nll": 3.2137770652770996, + "max_kl": 11.366734504699707, + "mean_kl": 3.423133134841919, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.2982456088066101 + }, + { + "candidate_nll": 4.751593589782715, + "code_tokens": 40, + "cvar95_kl": 9.304534912109375, + "delta_nll": 3.02370023727417, + "max_kl": 9.527302742004395, + "mean_kl": 3.4249043464660645, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.802781343460083, + "code_tokens": 25, + "cvar95_kl": 6.212045669555664, + "delta_nll": 2.317103147506714, + "max_kl": 7.142171382904053, + "mean_kl": 2.297528028488159, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 3.81661057472229, + "code_tokens": 23, + "cvar95_kl": 9.430822372436523, + "delta_nll": 3.045921564102173, + "max_kl": 9.763358116149902, + "mean_kl": 2.817875385284424, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 4.694406509399414, + "code_tokens": 77, + "cvar95_kl": 9.551265716552734, + "delta_nll": 3.6780569553375244, + "max_kl": 11.779208183288574, + "mean_kl": 3.665038585662842, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.3246753215789795 + }, + { + "candidate_nll": 3.825714111328125, + "code_tokens": 19, + "cvar95_kl": 8.146946907043457, + "delta_nll": 2.718048572540283, + "max_kl": 8.146946907043457, + "mean_kl": 2.4111897945404053, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 4.164647102355957, + "code_tokens": 37, + "cvar95_kl": 7.200336456298828, + "delta_nll": 3.181403398513794, + "max_kl": 8.611533164978027, + "mean_kl": 3.1491568088531494, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 3.8620927333831787, + "code_tokens": 33, + "cvar95_kl": 6.434575080871582, + "delta_nll": 2.4789278507232666, + "max_kl": 7.300389289855957, + "mean_kl": 2.7227821350097656, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.038670063018799, + "code_tokens": 95, + "cvar95_kl": 7.567311763763428, + "delta_nll": 3.378828525543213, + "max_kl": 9.297948837280273, + "mean_kl": 3.2933812141418457, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.3052631616592407 + }, + { + "candidate_nll": 3.782163619995117, + "code_tokens": 125, + "cvar95_kl": 7.405381202697754, + "delta_nll": 2.769003391265869, + "max_kl": 7.868445873260498, + "mean_kl": 2.85356068611145, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.2720000147819519 + }, + { + "candidate_nll": 5.415889739990234, + "code_tokens": 168, + "cvar95_kl": 11.145367622375488, + "delta_nll": 4.702841281890869, + "max_kl": 12.293146133422852, + "mean_kl": 4.647309303283691, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 3.244804859161377, + "code_tokens": 31, + "cvar95_kl": 12.409086227416992, + "delta_nll": 1.9294027090072632, + "max_kl": 13.092423439025879, + "mean_kl": 2.710458755493164, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 0.4193548262119293 + }, + { + "candidate_nll": 3.8820576667785645, + "code_tokens": 83, + "cvar95_kl": 8.467916488647461, + "delta_nll": 3.1203508377075195, + "max_kl": 9.966136932373047, + "mean_kl": 3.08866024017334, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.3734939694404602 + }, + { + "candidate_nll": 4.539491176605225, + "code_tokens": 50, + "cvar95_kl": 8.004000663757324, + "delta_nll": 3.8395485877990723, + "max_kl": 10.434915542602539, + "mean_kl": 3.7447502613067627, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 3.1107804775238037, + "code_tokens": 69, + "cvar95_kl": 6.948183536529541, + "delta_nll": 2.743598461151123, + "max_kl": 7.529196739196777, + "mean_kl": 2.7359745502471924, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 4.259094715118408, + "code_tokens": 50, + "cvar95_kl": 8.697810173034668, + "delta_nll": 3.174198627471924, + "max_kl": 9.75216007232666, + "mean_kl": 3.4503355026245117, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 4.517528057098389, + "code_tokens": 161, + "cvar95_kl": 7.99152135848999, + "delta_nll": 3.5137462615966797, + "max_kl": 9.696879386901855, + "mean_kl": 3.475428342819214, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.2732919156551361 + }, + { + "candidate_nll": 5.422668933868408, + "code_tokens": 53, + "cvar95_kl": 12.107203483581543, + "delta_nll": 4.766892910003662, + "max_kl": 14.614930152893066, + "mean_kl": 4.683996200561523, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.24528302252292633 + }, + { + "candidate_nll": 4.259059429168701, + "code_tokens": 97, + "cvar95_kl": 9.390853881835938, + "delta_nll": 3.8135313987731934, + "max_kl": 10.085846900939941, + "mean_kl": 3.606504201889038, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.3711340129375458 + }, + { + "candidate_nll": 3.160634756088257, + "code_tokens": 22, + "cvar95_kl": 7.3593549728393555, + "delta_nll": 1.8468217849731445, + "max_kl": 9.356971740722656, + "mean_kl": 2.134869337081909, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 0.5454545617103577 + }, + { + "candidate_nll": 3.1339950561523438, + "code_tokens": 34, + "cvar95_kl": 12.453664779663086, + "delta_nll": 2.2650909423828125, + "max_kl": 12.9107027053833, + "mean_kl": 2.866389036178589, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 0.44117647409439087 + }, + { + "candidate_nll": 4.3677592277526855, + "code_tokens": 81, + "cvar95_kl": 9.114741325378418, + "delta_nll": 3.5091159343719482, + "max_kl": 9.948921203613281, + "mean_kl": 3.732825756072998, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 2.420463800430298, + "code_tokens": 17, + "cvar95_kl": 5.373634338378906, + "delta_nll": 1.158360481262207, + "max_kl": 5.373634338378906, + "mean_kl": 1.2727811336517334, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 4.430771827697754, + "code_tokens": 181, + "cvar95_kl": 8.188791275024414, + "delta_nll": 3.6055145263671875, + "max_kl": 9.45089340209961, + "mean_kl": 3.55151104927063, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.3093922734260559 + }, + { + "candidate_nll": 4.059077262878418, + "code_tokens": 63, + "cvar95_kl": 7.254202365875244, + "delta_nll": 3.0786688327789307, + "max_kl": 7.943193435668945, + "mean_kl": 3.1802234649658203, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 0.2698412835597992 + }, + { + "candidate_nll": 1.7975528240203857, + "code_tokens": 35, + "cvar95_kl": 4.385894775390625, + "delta_nll": 1.104029655456543, + "max_kl": 4.390296936035156, + "mean_kl": 1.3332635164260864, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 3.833508253097534, + "code_tokens": 25, + "cvar95_kl": 5.512701988220215, + "delta_nll": 3.0287208557128906, + "max_kl": 5.634686470031738, + "mean_kl": 2.7849013805389404, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 4.963274002075195, + "code_tokens": 60, + "cvar95_kl": 8.895182609558105, + "delta_nll": 3.874908447265625, + "max_kl": 9.771342277526855, + "mean_kl": 4.120390892028809, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.701871871948242, + "code_tokens": 86, + "cvar95_kl": 9.448307991027832, + "delta_nll": 3.790225028991699, + "max_kl": 11.64909839630127, + "mean_kl": 3.666611433029175, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 4.524605751037598, + "code_tokens": 40, + "cvar95_kl": 9.897785186767578, + "delta_nll": 3.598439931869507, + "max_kl": 11.273552894592285, + "mean_kl": 3.716430187225342, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 3.652282238006592, + "code_tokens": 27, + "cvar95_kl": 5.284392356872559, + "delta_nll": 2.088721990585327, + "max_kl": 5.7514543533325195, + "mean_kl": 2.1308515071868896, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 4.228484153747559, + "code_tokens": 52, + "cvar95_kl": 7.096895694732666, + "delta_nll": 3.1389012336730957, + "max_kl": 7.37968635559082, + "mean_kl": 3.4820752143859863, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 3.3323163986206055, + "code_tokens": 71, + "cvar95_kl": 7.901246070861816, + "delta_nll": 2.590442180633545, + "max_kl": 8.784496307373047, + "mean_kl": 2.6489477157592773, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.3380281627178192 + }, + { + "candidate_nll": 3.5935921669006348, + "code_tokens": 25, + "cvar95_kl": 7.632483005523682, + "delta_nll": 2.4917404651641846, + "max_kl": 8.027271270751953, + "mean_kl": 2.587705612182617, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 2.3177568912506104, + "code_tokens": 103, + "cvar95_kl": 5.17860746383667, + "delta_nll": 1.7232612371444702, + "max_kl": 6.501383304595947, + "mean_kl": 1.768396019935608, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.5339806079864502 + }, + { + "candidate_nll": 3.679250717163086, + "code_tokens": 116, + "cvar95_kl": 7.914159774780273, + "delta_nll": 2.6217126846313477, + "max_kl": 8.096830368041992, + "mean_kl": 2.7133145332336426, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.4051724076271057 + }, + { + "candidate_nll": 4.2293243408203125, + "code_tokens": 107, + "cvar95_kl": 7.864154815673828, + "delta_nll": 3.5809781551361084, + "max_kl": 9.929819107055664, + "mean_kl": 3.6327974796295166, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 0.25233644247055054 + }, + { + "candidate_nll": 3.671194314956665, + "code_tokens": 54, + "cvar95_kl": 6.248693943023682, + "delta_nll": 2.728156566619873, + "max_kl": 7.763869762420654, + "mean_kl": 2.647960901260376, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.24074074625968933 + }, + { + "candidate_nll": 4.393448829650879, + "code_tokens": 46, + "cvar95_kl": 9.488480567932129, + "delta_nll": 3.760056495666504, + "max_kl": 10.544024467468262, + "mean_kl": 3.587456703186035, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.222973823547363, + "code_tokens": 170, + "cvar95_kl": 9.782427787780762, + "delta_nll": 3.893826961517334, + "max_kl": 14.057965278625488, + "mean_kl": 3.8211896419525146, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 3.8314123153686523, + "code_tokens": 45, + "cvar95_kl": 6.6022210121154785, + "delta_nll": 2.4047200679779053, + "max_kl": 8.143301963806152, + "mean_kl": 2.6127963066101074, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.2888889014720917 + }, + { + "candidate_nll": 4.276156425476074, + "code_tokens": 24, + "cvar95_kl": 7.011848449707031, + "delta_nll": 2.8678531646728516, + "max_kl": 7.296291351318359, + "mean_kl": 3.132916212081909, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 4.1344170570373535, + "code_tokens": 78, + "cvar95_kl": 8.744482040405273, + "delta_nll": 2.97245717048645, + "max_kl": 10.33784294128418, + "mean_kl": 3.050956964492798, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 4.038511276245117, + "code_tokens": 33, + "cvar95_kl": 7.134281158447266, + "delta_nll": 3.407954216003418, + "max_kl": 7.149947166442871, + "mean_kl": 3.223501443862915, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.8294172286987305, + "code_tokens": 32, + "cvar95_kl": 9.122516632080078, + "delta_nll": 3.713747501373291, + "max_kl": 9.456341743469238, + "mean_kl": 3.5063488483428955, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 5.088923931121826, + "code_tokens": 77, + "cvar95_kl": 8.66441535949707, + "delta_nll": 4.1001739501953125, + "max_kl": 8.903407096862793, + "mean_kl": 3.934325695037842, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 2.485135555267334, + "code_tokens": 222, + "cvar95_kl": 7.412210941314697, + "delta_nll": 2.062885046005249, + "max_kl": 10.159187316894531, + "mean_kl": 2.1103405952453613, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.5315315127372742 + }, + { + "candidate_nll": 3.9113800525665283, + "code_tokens": 86, + "cvar95_kl": 8.0269136428833, + "delta_nll": 2.8627853393554688, + "max_kl": 9.563424110412598, + "mean_kl": 2.968512535095215, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 2.8098394870758057, + "code_tokens": 31, + "cvar95_kl": 7.216091156005859, + "delta_nll": 2.475790023803711, + "max_kl": 8.120597839355469, + "mean_kl": 2.4162392616271973, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 3.5639967918395996, + "code_tokens": 34, + "cvar95_kl": 7.343200206756592, + "delta_nll": 3.052396774291992, + "max_kl": 7.397664546966553, + "mean_kl": 3.0592055320739746, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.38235294818878174 + }, + { + "candidate_nll": 4.478091716766357, + "code_tokens": 38, + "cvar95_kl": 7.9981279373168945, + "delta_nll": 3.0906314849853516, + "max_kl": 8.164901733398438, + "mean_kl": 3.1678895950317383, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.28947368264198303 + }, + { + "candidate_nll": 4.0144124031066895, + "code_tokens": 93, + "cvar95_kl": 7.794415473937988, + "delta_nll": 3.05136775970459, + "max_kl": 8.795658111572266, + "mean_kl": 3.143176555633545, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.154757976531982, + "code_tokens": 70, + "cvar95_kl": 8.394577980041504, + "delta_nll": 3.376152276992798, + "max_kl": 9.205666542053223, + "mean_kl": 3.37726092338562, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.37142857909202576 + }, + { + "candidate_nll": 5.5489325523376465, + "code_tokens": 67, + "cvar95_kl": 10.0623779296875, + "delta_nll": 4.73844051361084, + "max_kl": 11.22844123840332, + "mean_kl": 4.708843231201172, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 0.1492537260055542 + }, + { + "candidate_nll": 4.281634330749512, + "code_tokens": 66, + "cvar95_kl": 7.786943435668945, + "delta_nll": 2.995507001876831, + "max_kl": 9.36795711517334, + "mean_kl": 3.045809030532837, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 3.6385183334350586, + "code_tokens": 34, + "cvar95_kl": 6.245553493499756, + "delta_nll": 2.6262240409851074, + "max_kl": 7.040338039398193, + "mean_kl": 2.724132537841797, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 3.314260959625244, + "code_tokens": 64, + "cvar95_kl": 7.029295921325684, + "delta_nll": 2.404235601425171, + "max_kl": 8.206830978393555, + "mean_kl": 2.467315196990967, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 3.6370015144348145, + "code_tokens": 56, + "cvar95_kl": 6.77085542678833, + "delta_nll": 2.326645851135254, + "max_kl": 7.624614238739014, + "mean_kl": 2.500717878341675, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.4107142984867096 + }, + { + "candidate_nll": 3.2628729343414307, + "code_tokens": 21, + "cvar95_kl": 6.059933185577393, + "delta_nll": 2.1377906799316406, + "max_kl": 6.4874396324157715, + "mean_kl": 2.3361728191375732, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.772418975830078, + "code_tokens": 60, + "cvar95_kl": 6.819569110870361, + "delta_nll": 3.1070785522460938, + "max_kl": 7.9312028884887695, + "mean_kl": 3.17063570022583, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.227932929992676, + "code_tokens": 29, + "cvar95_kl": 7.475390434265137, + "delta_nll": 2.874156951904297, + "max_kl": 7.663685321807861, + "mean_kl": 2.824254274368286, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 5.630523681640625, + "code_tokens": 34, + "cvar95_kl": 10.817977905273438, + "delta_nll": 4.908478736877441, + "max_kl": 11.113961219787598, + "mean_kl": 4.947022438049316, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.468457221984863, + "code_tokens": 88, + "cvar95_kl": 9.794919967651367, + "delta_nll": 4.549349784851074, + "max_kl": 12.077648162841797, + "mean_kl": 4.599836349487305, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.7930047512054443, + "code_tokens": 17, + "cvar95_kl": 5.7475128173828125, + "delta_nll": 2.2497811317443848, + "max_kl": 5.7475128173828125, + "mean_kl": 2.1518077850341797, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.248149871826172, + "code_tokens": 35, + "cvar95_kl": 6.580766201019287, + "delta_nll": 2.357602596282959, + "max_kl": 7.215487003326416, + "mean_kl": 2.54371976852417, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.1264255046844482, + "code_tokens": 48, + "cvar95_kl": 6.93898344039917, + "delta_nll": 2.2366490364074707, + "max_kl": 8.594228744506836, + "mean_kl": 2.3424367904663086, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 0.5208333134651184 + }, + { + "candidate_nll": 3.85402512550354, + "code_tokens": 130, + "cvar95_kl": 7.563445568084717, + "delta_nll": 2.863752841949463, + "max_kl": 10.12913703918457, + "mean_kl": 2.75119686126709, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.35384616255760193 + }, + { + "candidate_nll": 5.426913738250732, + "code_tokens": 65, + "cvar95_kl": 9.202330589294434, + "delta_nll": 4.331974029541016, + "max_kl": 9.970547676086426, + "mean_kl": 4.383175849914551, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 4.673245429992676, + "code_tokens": 145, + "cvar95_kl": 9.858207702636719, + "delta_nll": 3.7982304096221924, + "max_kl": 12.207542419433594, + "mean_kl": 3.859558582305908, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 2.8526241779327393, + "code_tokens": 80, + "cvar95_kl": 7.717275619506836, + "delta_nll": 2.1369247436523438, + "max_kl": 8.99539852142334, + "mean_kl": 2.1344175338745117, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.48750001192092896 + }, + { + "candidate_nll": 3.670846939086914, + "code_tokens": 70, + "cvar95_kl": 7.402796268463135, + "delta_nll": 3.088710069656372, + "max_kl": 9.310441970825195, + "mean_kl": 3.2762513160705566, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.3285714387893677 + }, + { + "candidate_nll": 4.1106953620910645, + "code_tokens": 25, + "cvar95_kl": 7.4535746574401855, + "delta_nll": 2.622406482696533, + "max_kl": 7.781219482421875, + "mean_kl": 2.744468927383423, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.5130693912506104, + "code_tokens": 50, + "cvar95_kl": 6.433637619018555, + "delta_nll": 1.8586859703063965, + "max_kl": 8.409744262695312, + "mean_kl": 2.36962890625, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 3.785855531692505, + "code_tokens": 48, + "cvar95_kl": 7.169675827026367, + "delta_nll": 2.9878125190734863, + "max_kl": 8.156593322753906, + "mean_kl": 3.1657965183258057, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.522458076477051, + "code_tokens": 104, + "cvar95_kl": 9.720070838928223, + "delta_nll": 3.1783857345581055, + "max_kl": 12.350061416625977, + "mean_kl": 2.9993667602539062, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 3.8676528930664062, + "code_tokens": 68, + "cvar95_kl": 7.127436637878418, + "delta_nll": 2.8776586055755615, + "max_kl": 8.637825012207031, + "mean_kl": 2.917590379714966, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 3.5744595527648926, + "code_tokens": 145, + "cvar95_kl": 6.802331447601318, + "delta_nll": 2.819547414779663, + "max_kl": 8.060219764709473, + "mean_kl": 2.7774813175201416, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.3655172288417816 + }, + { + "candidate_nll": 3.1793015003204346, + "code_tokens": 41, + "cvar95_kl": 6.643798351287842, + "delta_nll": 2.334413766860962, + "max_kl": 6.812431812286377, + "mean_kl": 2.5246379375457764, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 0.39024388790130615 + }, + { + "candidate_nll": 2.7646031379699707, + "code_tokens": 53, + "cvar95_kl": 6.995388031005859, + "delta_nll": 2.003480911254883, + "max_kl": 7.369541168212891, + "mean_kl": 1.8686801195144653, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.5660377144813538 + }, + { + "candidate_nll": 3.971863269805908, + "code_tokens": 58, + "cvar95_kl": 8.917978286743164, + "delta_nll": 3.179633617401123, + "max_kl": 13.145102500915527, + "mean_kl": 3.0552196502685547, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 4.383185863494873, + "code_tokens": 24, + "cvar95_kl": 10.76356315612793, + "delta_nll": 2.651125907897949, + "max_kl": 11.920948028564453, + "mean_kl": 3.05184268951416, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.1393251419067383, + "code_tokens": 89, + "cvar95_kl": 7.966253757476807, + "delta_nll": 2.1474711894989014, + "max_kl": 11.073119163513184, + "mean_kl": 2.394813060760498, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.47191011905670166 + }, + { + "candidate_nll": 4.176098346710205, + "code_tokens": 101, + "cvar95_kl": 9.635716438293457, + "delta_nll": 3.6910221576690674, + "max_kl": 10.653604507446289, + "mean_kl": 3.657911777496338, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 0.2574257552623749 + }, + { + "candidate_nll": 5.257620811462402, + "code_tokens": 33, + "cvar95_kl": 10.380867004394531, + "delta_nll": 3.800063133239746, + "max_kl": 12.692362785339355, + "mean_kl": 3.830845355987549, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.8801565170288086, + "code_tokens": 50, + "cvar95_kl": 9.313899993896484, + "delta_nll": 2.8334908485412598, + "max_kl": 9.843820571899414, + "mean_kl": 3.086979866027832, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.761054039001465, + "code_tokens": 58, + "cvar95_kl": 9.731385231018066, + "delta_nll": 3.3523011207580566, + "max_kl": 12.450210571289062, + "mean_kl": 3.576413154602051, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.22413793206214905 + }, + { + "candidate_nll": 4.070394515991211, + "code_tokens": 62, + "cvar95_kl": 8.055257797241211, + "delta_nll": 3.091235637664795, + "max_kl": 9.362038612365723, + "mean_kl": 3.257218360900879, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 3.690685510635376, + "code_tokens": 42, + "cvar95_kl": 7.719529628753662, + "delta_nll": 2.998995542526245, + "max_kl": 9.722907066345215, + "mean_kl": 3.0206799507141113, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 3.7232627868652344, + "code_tokens": 15, + "cvar95_kl": 7.71462869644165, + "delta_nll": 2.7985217571258545, + "max_kl": 7.71462869644165, + "mean_kl": 2.46602725982666, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.3326520919799805, + "code_tokens": 57, + "cvar95_kl": 7.719557285308838, + "delta_nll": 3.593869924545288, + "max_kl": 8.125247955322266, + "mean_kl": 3.676182985305786, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.21052631735801697 + }, + { + "candidate_nll": 3.893308639526367, + "code_tokens": 58, + "cvar95_kl": 6.625524044036865, + "delta_nll": 2.8497726917266846, + "max_kl": 6.830093860626221, + "mean_kl": 2.7018167972564697, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 4.0603437423706055, + "code_tokens": 45, + "cvar95_kl": 7.228200435638428, + "delta_nll": 3.1883726119995117, + "max_kl": 8.084911346435547, + "mean_kl": 3.2077291011810303, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.551851272583008, + "code_tokens": 32, + "cvar95_kl": 8.066949844360352, + "delta_nll": 2.522620916366577, + "max_kl": 8.070785522460938, + "mean_kl": 3.1027331352233887, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.34375 + }, + { + "candidate_nll": 4.6727399826049805, + "code_tokens": 39, + "cvar95_kl": 11.032657623291016, + "delta_nll": 3.3520736694335938, + "max_kl": 12.359375, + "mean_kl": 3.81355881690979, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 3.088874340057373, + "code_tokens": 136, + "cvar95_kl": 7.735730171203613, + "delta_nll": 2.6781129837036133, + "max_kl": 9.68453598022461, + "mean_kl": 2.6130034923553467, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.41911765933036804 + }, + { + "candidate_nll": 4.610614776611328, + "code_tokens": 50, + "cvar95_kl": 9.486408233642578, + "delta_nll": 3.3898582458496094, + "max_kl": 9.905241012573242, + "mean_kl": 3.3982303142547607, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 3.482875347137451, + "code_tokens": 34, + "cvar95_kl": 7.607154846191406, + "delta_nll": 2.616959571838379, + "max_kl": 8.12242603302002, + "mean_kl": 2.6963276863098145, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.48274564743042, + "code_tokens": 44, + "cvar95_kl": 10.421761512756348, + "delta_nll": 3.303955554962158, + "max_kl": 13.856709480285645, + "mean_kl": 3.6389098167419434, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.34090909361839294 + }, + { + "candidate_nll": 3.0367825031280518, + "code_tokens": 36, + "cvar95_kl": 5.89003849029541, + "delta_nll": 1.603360652923584, + "max_kl": 7.397391319274902, + "mean_kl": 1.8957871198654175, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.460653781890869, + "code_tokens": 46, + "cvar95_kl": 8.991292953491211, + "delta_nll": 3.657496452331543, + "max_kl": 9.452770233154297, + "mean_kl": 3.743454694747925, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 4.3318190574646, + "code_tokens": 48, + "cvar95_kl": 9.805851936340332, + "delta_nll": 3.1514792442321777, + "max_kl": 10.665495872497559, + "mean_kl": 3.447067975997925, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.8156206607818604, + "code_tokens": 25, + "cvar95_kl": 8.599863052368164, + "delta_nll": 2.9202096462249756, + "max_kl": 10.256918907165527, + "mean_kl": 2.701446771621704, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.599009037017822, + "code_tokens": 69, + "cvar95_kl": 10.220185279846191, + "delta_nll": 3.0947299003601074, + "max_kl": 11.083846092224121, + "mean_kl": 3.575578212738037, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 3.9488556385040283, + "code_tokens": 60, + "cvar95_kl": 7.416545867919922, + "delta_nll": 2.983397960662842, + "max_kl": 7.565510272979736, + "mean_kl": 3.2888567447662354, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 3.6424496173858643, + "code_tokens": 49, + "cvar95_kl": 8.90544605255127, + "delta_nll": 2.7279720306396484, + "max_kl": 13.41318130493164, + "mean_kl": 2.6898796558380127, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.3265306055545807 + }, + { + "candidate_nll": 4.287105083465576, + "code_tokens": 44, + "cvar95_kl": 9.587298393249512, + "delta_nll": 2.5865769386291504, + "max_kl": 10.898160934448242, + "mean_kl": 2.5177383422851562, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.0189738273620605, + "code_tokens": 81, + "cvar95_kl": 7.622276306152344, + "delta_nll": 2.8735594749450684, + "max_kl": 9.404717445373535, + "mean_kl": 2.8959436416625977, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 4.2365288734436035, + "code_tokens": 26, + "cvar95_kl": 8.119511604309082, + "delta_nll": 2.7997937202453613, + "max_kl": 9.624934196472168, + "mean_kl": 2.808807373046875, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.059637069702148, + "code_tokens": 32, + "cvar95_kl": 10.485386848449707, + "delta_nll": 3.179716110229492, + "max_kl": 10.829595565795898, + "mean_kl": 3.624828577041626, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 4.5890679359436035, + "code_tokens": 48, + "cvar95_kl": 7.982027530670166, + "delta_nll": 3.6413259506225586, + "max_kl": 8.70114517211914, + "mean_kl": 3.5416364669799805, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.647439956665039, + "code_tokens": 180, + "cvar95_kl": 7.571989059448242, + "delta_nll": 2.9481186866760254, + "max_kl": 9.20739459991455, + "mean_kl": 2.928934097290039, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.31111112236976624 + }, + { + "candidate_nll": 4.0465545654296875, + "code_tokens": 38, + "cvar95_kl": 8.67829704284668, + "delta_nll": 3.3247928619384766, + "max_kl": 8.822488784790039, + "mean_kl": 3.527998685836792, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 3.422511100769043, + "code_tokens": 90, + "cvar95_kl": 8.535375595092773, + "delta_nll": 2.727449417114258, + "max_kl": 9.374414443969727, + "mean_kl": 2.7709755897521973, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.3444444537162781 + }, + { + "candidate_nll": 4.414219379425049, + "code_tokens": 44, + "cvar95_kl": 8.50815486907959, + "delta_nll": 3.8395893573760986, + "max_kl": 9.0233736038208, + "mean_kl": 3.7499406337738037, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 2.9356234073638916, + "code_tokens": 60, + "cvar95_kl": 7.611464023590088, + "delta_nll": 2.191723346710205, + "max_kl": 8.198963165283203, + "mean_kl": 2.31815505027771, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 3.4897501468658447, + "code_tokens": 69, + "cvar95_kl": 6.17899227142334, + "delta_nll": 2.486952066421509, + "max_kl": 6.792649745941162, + "mean_kl": 2.366588592529297, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 4.089226245880127, + "code_tokens": 71, + "cvar95_kl": 7.7783355712890625, + "delta_nll": 3.571664333343506, + "max_kl": 8.54160213470459, + "mean_kl": 3.7117760181427, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.2535211145877838 + }, + { + "candidate_nll": 3.3825953006744385, + "code_tokens": 22, + "cvar95_kl": 7.3777666091918945, + "delta_nll": 2.1399221420288086, + "max_kl": 8.566722869873047, + "mean_kl": 1.9118329286575317, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 2.283029317855835, + "code_tokens": 215, + "cvar95_kl": 6.2007670402526855, + "delta_nll": 1.7292498350143433, + "max_kl": 9.285374641418457, + "mean_kl": 1.8153493404388428, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.5116279125213623 + }, + { + "candidate_nll": 4.797551155090332, + "code_tokens": 26, + "cvar95_kl": 9.301051139831543, + "delta_nll": 3.6947615146636963, + "max_kl": 10.256996154785156, + "mean_kl": 3.464982748031616, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 3.9066805839538574, + "code_tokens": 33, + "cvar95_kl": 8.091815948486328, + "delta_nll": 3.0384340286254883, + "max_kl": 9.878740310668945, + "mean_kl": 3.082937240600586, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 2.6877050399780273, + "code_tokens": 18, + "cvar95_kl": 6.767054557800293, + "delta_nll": 1.7515568733215332, + "max_kl": 6.767054557800293, + "mean_kl": 1.9960858821868896, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.788707256317139, + "code_tokens": 71, + "cvar95_kl": 9.93177604675293, + "delta_nll": 3.508645534515381, + "max_kl": 12.776721954345703, + "mean_kl": 3.728562116622925, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.3239436745643616 + }, + { + "candidate_nll": 3.651622772216797, + "code_tokens": 107, + "cvar95_kl": 6.133072376251221, + "delta_nll": 2.8702213764190674, + "max_kl": 6.4738030433654785, + "mean_kl": 2.7436163425445557, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.3644859790802002 + }, + { + "candidate_nll": 4.842979431152344, + "code_tokens": 40, + "cvar95_kl": 9.33761215209961, + "delta_nll": 2.787954807281494, + "max_kl": 10.308077812194824, + "mean_kl": 3.458218812942505, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.8805949687957764, + "code_tokens": 32, + "cvar95_kl": 6.741325378417969, + "delta_nll": 2.5162980556488037, + "max_kl": 8.123666763305664, + "mean_kl": 2.6641600131988525, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.4375 + }, + { + "candidate_nll": 3.641751766204834, + "code_tokens": 130, + "cvar95_kl": 7.146689414978027, + "delta_nll": 2.942497968673706, + "max_kl": 9.172150611877441, + "mean_kl": 3.0431630611419678, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.2846153974533081 + }, + { + "candidate_nll": 2.93428635597229, + "code_tokens": 27, + "cvar95_kl": 4.964244842529297, + "delta_nll": 1.462864637374878, + "max_kl": 5.057807445526123, + "mean_kl": 1.8194681406021118, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.329661846160889, + "code_tokens": 42, + "cvar95_kl": 9.33948802947998, + "delta_nll": 3.3567214012145996, + "max_kl": 10.242304801940918, + "mean_kl": 3.220517158508301, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.456398963928223, + "code_tokens": 30, + "cvar95_kl": 8.869171142578125, + "delta_nll": 3.710885763168335, + "max_kl": 10.413792610168457, + "mean_kl": 3.7548270225524902, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 5.436509132385254, + "code_tokens": 58, + "cvar95_kl": 9.90967845916748, + "delta_nll": 4.017276287078857, + "max_kl": 10.570554733276367, + "mean_kl": 3.8941075801849365, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 4.258218765258789, + "code_tokens": 34, + "cvar95_kl": 8.014579772949219, + "delta_nll": 3.027991533279419, + "max_kl": 9.365297317504883, + "mean_kl": 2.9604034423828125, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 2.5235466957092285, + "code_tokens": 208, + "cvar95_kl": 6.394019603729248, + "delta_nll": 1.8220841884613037, + "max_kl": 10.350736618041992, + "mean_kl": 1.874619722366333, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.45673078298568726 + }, + { + "candidate_nll": 3.601691484451294, + "code_tokens": 83, + "cvar95_kl": 6.50521183013916, + "delta_nll": 2.7027857303619385, + "max_kl": 7.5803375244140625, + "mean_kl": 2.8162975311279297, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.3253012001514435 + }, + { + "candidate_nll": 3.7155888080596924, + "code_tokens": 170, + "cvar95_kl": 9.092093467712402, + "delta_nll": 3.193033218383789, + "max_kl": 12.873892784118652, + "mean_kl": 3.188279151916504, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.3235294222831726 + }, + { + "candidate_nll": 3.514446973800659, + "code_tokens": 40, + "cvar95_kl": 8.405462265014648, + "delta_nll": 2.7330265045166016, + "max_kl": 9.33468246459961, + "mean_kl": 2.9366984367370605, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.4749999940395355 + }, + { + "candidate_nll": 3.665970802307129, + "code_tokens": 31, + "cvar95_kl": 7.7201457023620605, + "delta_nll": 2.4192612171173096, + "max_kl": 8.230094909667969, + "mean_kl": 2.5218679904937744, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 0.4516128897666931 + }, + { + "candidate_nll": 4.619250774383545, + "code_tokens": 42, + "cvar95_kl": 7.901052474975586, + "delta_nll": 3.8244807720184326, + "max_kl": 9.82599925994873, + "mean_kl": 4.089231491088867, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 4.071943759918213, + "code_tokens": 27, + "cvar95_kl": 8.017488479614258, + "delta_nll": 2.302877902984619, + "max_kl": 9.077926635742188, + "mean_kl": 3.1219301223754883, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 4.108912467956543, + "code_tokens": 25, + "cvar95_kl": 6.679482460021973, + "delta_nll": 1.909101963043213, + "max_kl": 7.279641628265381, + "mean_kl": 2.2553720474243164, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 4.183228969573975, + "code_tokens": 40, + "cvar95_kl": 8.07467269897461, + "delta_nll": 3.614842414855957, + "max_kl": 8.314855575561523, + "mean_kl": 3.6975150108337402, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 3.8422622680664062, + "code_tokens": 24, + "cvar95_kl": 6.197222709655762, + "delta_nll": 2.4579076766967773, + "max_kl": 6.342657566070557, + "mean_kl": 2.3194968700408936, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 3.380577802658081, + "code_tokens": 28, + "cvar95_kl": 5.826620101928711, + "delta_nll": 2.497832775115967, + "max_kl": 6.192239284515381, + "mean_kl": 2.4061121940612793, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 3.9301254749298096, + "code_tokens": 71, + "cvar95_kl": 8.732564926147461, + "delta_nll": 3.451261043548584, + "max_kl": 9.627111434936523, + "mean_kl": 3.293316125869751, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.3239436745643616 + }, + { + "candidate_nll": 3.327026605606079, + "code_tokens": 24, + "cvar95_kl": 7.790476322174072, + "delta_nll": 2.79766583442688, + "max_kl": 8.464487075805664, + "mean_kl": 2.6727190017700195, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.248255252838135, + "code_tokens": 34, + "cvar95_kl": 9.891931533813477, + "delta_nll": 2.9766740798950195, + "max_kl": 10.12065601348877, + "mean_kl": 3.4336791038513184, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.2647058963775635 + }, + { + "candidate_nll": 4.602066516876221, + "code_tokens": 64, + "cvar95_kl": 9.181890487670898, + "delta_nll": 3.4827725887298584, + "max_kl": 9.81509780883789, + "mean_kl": 3.8793296813964844, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.9755821228027344, + "code_tokens": 45, + "cvar95_kl": 6.011260986328125, + "delta_nll": 2.2689907550811768, + "max_kl": 6.351089000701904, + "mean_kl": 2.087141513824463, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.5111111402511597 + }, + { + "candidate_nll": 4.390018939971924, + "code_tokens": 48, + "cvar95_kl": 9.5652494430542, + "delta_nll": 3.191314697265625, + "max_kl": 10.328932762145996, + "mean_kl": 3.531367301940918, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.3641321659088135, + "code_tokens": 21, + "cvar95_kl": 7.467554569244385, + "delta_nll": 2.663930892944336, + "max_kl": 7.683157444000244, + "mean_kl": 2.754385471343994, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.050568103790283, + "code_tokens": 24, + "cvar95_kl": 10.474175453186035, + "delta_nll": 3.296154022216797, + "max_kl": 11.067089080810547, + "mean_kl": 3.648087501525879, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.391454219818115, + "code_tokens": 63, + "cvar95_kl": 7.367810249328613, + "delta_nll": 2.8195269107818604, + "max_kl": 8.00245475769043, + "mean_kl": 2.9554567337036133, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.354425430297852, + "code_tokens": 22, + "cvar95_kl": 8.19725227355957, + "delta_nll": 4.075666427612305, + "max_kl": 8.415875434875488, + "mean_kl": 3.7802796363830566, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 3.115346670150757, + "code_tokens": 25, + "cvar95_kl": 5.182452201843262, + "delta_nll": 1.9866843223571777, + "max_kl": 5.349338054656982, + "mean_kl": 2.1988472938537598, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.2102530002593994, + "code_tokens": 21, + "cvar95_kl": 5.9929351806640625, + "delta_nll": 1.249927282333374, + "max_kl": 6.068973541259766, + "mean_kl": 1.6142609119415283, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.5714285969734192 + }, + { + "candidate_nll": 2.7355949878692627, + "code_tokens": 24, + "cvar95_kl": 4.6976518630981445, + "delta_nll": 1.2692629098892212, + "max_kl": 5.538305282592773, + "mean_kl": 1.6080914735794067, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 5.61820650100708, + "code_tokens": 15, + "cvar95_kl": 9.832240104675293, + "delta_nll": 3.824718713760376, + "max_kl": 9.832240104675293, + "mean_kl": 4.017518997192383, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.066774845123291, + "code_tokens": 59, + "cvar95_kl": 8.200403213500977, + "delta_nll": 3.2859184741973877, + "max_kl": 8.9993896484375, + "mean_kl": 3.189260721206665, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.3050847351551056 + }, + { + "candidate_nll": 3.8294942378997803, + "code_tokens": 54, + "cvar95_kl": 7.813264846801758, + "delta_nll": 2.895375967025757, + "max_kl": 8.384539604187012, + "mean_kl": 2.930708646774292, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 4.782858371734619, + "code_tokens": 28, + "cvar95_kl": 7.395012855529785, + "delta_nll": 2.8309993743896484, + "max_kl": 7.885943412780762, + "mean_kl": 3.1850342750549316, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.4847824573516846, + "code_tokens": 46, + "cvar95_kl": 6.767747402191162, + "delta_nll": 2.4000563621520996, + "max_kl": 8.293885231018066, + "mean_kl": 2.60836124420166, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.3695652186870575 + }, + { + "candidate_nll": 2.499498128890991, + "code_tokens": 27, + "cvar95_kl": 4.570356369018555, + "delta_nll": 0.9846757650375366, + "max_kl": 4.688472747802734, + "mean_kl": 1.6208142042160034, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.48148149251937866 + }, + { + "candidate_nll": 4.408600330352783, + "code_tokens": 97, + "cvar95_kl": 6.757037162780762, + "delta_nll": 3.151169776916504, + "max_kl": 7.945012092590332, + "mean_kl": 3.2020773887634277, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.25773194432258606 + }, + { + "candidate_nll": 4.370863437652588, + "code_tokens": 24, + "cvar95_kl": 7.796977996826172, + "delta_nll": 3.5686309337615967, + "max_kl": 8.103171348571777, + "mean_kl": 3.330293893814087, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 5.055784225463867, + "code_tokens": 30, + "cvar95_kl": 9.202210426330566, + "delta_nll": 3.790635585784912, + "max_kl": 10.56954288482666, + "mean_kl": 4.145026683807373, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.413614273071289, + "code_tokens": 38, + "cvar95_kl": 11.37320327758789, + "delta_nll": 2.9805445671081543, + "max_kl": 12.380721092224121, + "mean_kl": 3.3064615726470947, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 2.5225093364715576, + "code_tokens": 21, + "cvar95_kl": 4.080326080322266, + "delta_nll": 1.034730315208435, + "max_kl": 4.3199615478515625, + "mean_kl": 1.3415125608444214, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.940645217895508, + "code_tokens": 65, + "cvar95_kl": 9.104715347290039, + "delta_nll": 4.403683662414551, + "max_kl": 10.236310005187988, + "mean_kl": 4.443620681762695, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 2.959641456604004, + "code_tokens": 27, + "cvar95_kl": 6.089597702026367, + "delta_nll": 2.1751151084899902, + "max_kl": 6.250226974487305, + "mean_kl": 2.2075634002685547, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.8383195400238037, + "code_tokens": 35, + "cvar95_kl": 10.372416496276855, + "delta_nll": 3.1145033836364746, + "max_kl": 13.251947402954102, + "mean_kl": 3.0151166915893555, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 3.705793857574463, + "code_tokens": 75, + "cvar95_kl": 7.00559139251709, + "delta_nll": 2.7812697887420654, + "max_kl": 8.51404857635498, + "mean_kl": 2.8707666397094727, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.2800000011920929 + }, + { + "candidate_nll": 4.125631332397461, + "code_tokens": 25, + "cvar95_kl": 6.916342735290527, + "delta_nll": 2.483895778656006, + "max_kl": 6.955812931060791, + "mean_kl": 3.127067804336548, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.63437557220459, + "code_tokens": 52, + "cvar95_kl": 7.310029983520508, + "delta_nll": 2.7103867530822754, + "max_kl": 8.377080917358398, + "mean_kl": 2.7261576652526855, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.42307692766189575 + }, + { + "candidate_nll": 3.252655267715454, + "code_tokens": 110, + "cvar95_kl": 9.365931510925293, + "delta_nll": 2.6419951915740967, + "max_kl": 11.886602401733398, + "mean_kl": 2.7463583946228027, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 3.1271047592163086, + "code_tokens": 25, + "cvar95_kl": 6.014121055603027, + "delta_nll": 1.8942828178405762, + "max_kl": 6.226978778839111, + "mean_kl": 2.0569372177124023, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.47999998927116394 + }, + { + "candidate_nll": 4.2334513664245605, + "code_tokens": 56, + "cvar95_kl": 5.697482585906982, + "delta_nll": 3.216465473175049, + "max_kl": 5.913536071777344, + "mean_kl": 3.0271382331848145, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.2321428507566452 + }, + { + "candidate_nll": 4.823466777801514, + "code_tokens": 108, + "cvar95_kl": 6.915374755859375, + "delta_nll": 3.269618511199951, + "max_kl": 7.6416239738464355, + "mean_kl": 3.3588006496429443, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.21296297013759613 + }, + { + "candidate_nll": 3.947444200515747, + "code_tokens": 47, + "cvar95_kl": 8.32578182220459, + "delta_nll": 2.575704336166382, + "max_kl": 9.44296646118164, + "mean_kl": 3.073638916015625, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.3191489279270172 + }, + { + "candidate_nll": 4.077665328979492, + "code_tokens": 102, + "cvar95_kl": 8.063263893127441, + "delta_nll": 3.079256772994995, + "max_kl": 9.046688079833984, + "mean_kl": 3.2492878437042236, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.30392158031463623 + }, + { + "candidate_nll": 3.092355966567993, + "code_tokens": 18, + "cvar95_kl": 5.006380081176758, + "delta_nll": 1.4667303562164307, + "max_kl": 5.006380081176758, + "mean_kl": 1.7450199127197266, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 4.9176225662231445, + "code_tokens": 28, + "cvar95_kl": 8.856426239013672, + "delta_nll": 3.817958354949951, + "max_kl": 10.231290817260742, + "mean_kl": 3.7431488037109375, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 4.403152942657471, + "code_tokens": 30, + "cvar95_kl": 7.159755706787109, + "delta_nll": 2.8597521781921387, + "max_kl": 7.397472858428955, + "mean_kl": 3.0528855323791504, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.492540121078491, + "code_tokens": 48, + "cvar95_kl": 8.284401893615723, + "delta_nll": 3.017423152923584, + "max_kl": 11.802966117858887, + "mean_kl": 3.028527021408081, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.3556177616119385, + "code_tokens": 59, + "cvar95_kl": 6.845510005950928, + "delta_nll": 2.8186028003692627, + "max_kl": 7.735580921173096, + "mean_kl": 2.9114139080047607, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 0.35593220591545105 + }, + { + "candidate_nll": 4.115127086639404, + "code_tokens": 102, + "cvar95_kl": 10.304710388183594, + "delta_nll": 3.0729331970214844, + "max_kl": 12.934366226196289, + "mean_kl": 3.139615535736084, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.36274510622024536 + }, + { + "candidate_nll": 3.591944694519043, + "code_tokens": 36, + "cvar95_kl": 7.16849946975708, + "delta_nll": 2.851579189300537, + "max_kl": 7.963933944702148, + "mean_kl": 2.8699254989624023, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 5.279996871948242, + "code_tokens": 51, + "cvar95_kl": 11.630035400390625, + "delta_nll": 4.39106559753418, + "max_kl": 13.342601776123047, + "mean_kl": 4.197364807128906, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.3137255012989044 + }, + { + "candidate_nll": 3.65476393699646, + "code_tokens": 62, + "cvar95_kl": 6.737730026245117, + "delta_nll": 2.409085273742676, + "max_kl": 6.838538646697998, + "mean_kl": 2.820497512817383, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.35483869910240173 + }, + { + "candidate_nll": 4.438981056213379, + "code_tokens": 39, + "cvar95_kl": 10.584783554077148, + "delta_nll": 3.862699031829834, + "max_kl": 11.654885292053223, + "mean_kl": 3.784137487411499, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 3.4360315799713135, + "code_tokens": 76, + "cvar95_kl": 6.867332458496094, + "delta_nll": 2.696347236633301, + "max_kl": 7.91359806060791, + "mean_kl": 2.755835771560669, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.3815789520740509 + }, + { + "candidate_nll": 5.322134017944336, + "code_tokens": 58, + "cvar95_kl": 9.567680358886719, + "delta_nll": 3.8174829483032227, + "max_kl": 10.004609107971191, + "mean_kl": 3.6917757987976074, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 3.9783711433410645, + "code_tokens": 93, + "cvar95_kl": 9.397175788879395, + "delta_nll": 2.85097599029541, + "max_kl": 14.383652687072754, + "mean_kl": 2.851611375808716, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.31182795763015747 + }, + { + "candidate_nll": 5.486507892608643, + "code_tokens": 45, + "cvar95_kl": 10.694029808044434, + "delta_nll": 4.131262302398682, + "max_kl": 11.472169876098633, + "mean_kl": 4.348259449005127, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.17777778208255768 + }, + { + "candidate_nll": 2.0917749404907227, + "code_tokens": 209, + "cvar95_kl": 6.000291347503662, + "delta_nll": 1.534799575805664, + "max_kl": 8.18809700012207, + "mean_kl": 1.6851645708084106, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.5119616985321045 + }, + { + "candidate_nll": 3.8180434703826904, + "code_tokens": 46, + "cvar95_kl": 7.256175994873047, + "delta_nll": 2.8495757579803467, + "max_kl": 8.863800048828125, + "mean_kl": 2.521113634109497, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.011497974395752, + "code_tokens": 39, + "cvar95_kl": 9.648407936096191, + "delta_nll": 3.290710926055908, + "max_kl": 10.380239486694336, + "mean_kl": 3.2123634815216064, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.28205129504203796 + }, + { + "candidate_nll": 2.8750500679016113, + "code_tokens": 25, + "cvar95_kl": 5.364563465118408, + "delta_nll": 1.9333546161651611, + "max_kl": 5.720942497253418, + "mean_kl": 2.029864549636841, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.1905388832092285, + "code_tokens": 28, + "cvar95_kl": 13.14030647277832, + "delta_nll": 3.0153427124023438, + "max_kl": 15.168493270874023, + "mean_kl": 3.1766040325164795, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 0.4642857015132904 + }, + { + "candidate_nll": 4.425013542175293, + "code_tokens": 84, + "cvar95_kl": 7.576647758483887, + "delta_nll": 3.171091079711914, + "max_kl": 8.247421264648438, + "mean_kl": 3.3318190574645996, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 3.5515494346618652, + "code_tokens": 34, + "cvar95_kl": 6.33892822265625, + "delta_nll": 2.739269256591797, + "max_kl": 6.783594608306885, + "mean_kl": 2.862056016921997, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 3.3771519660949707, + "code_tokens": 28, + "cvar95_kl": 5.649690628051758, + "delta_nll": 2.1221048831939697, + "max_kl": 5.81315279006958, + "mean_kl": 2.370088815689087, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.465479612350464, + "code_tokens": 92, + "cvar95_kl": 7.9074273109436035, + "delta_nll": 2.3666341304779053, + "max_kl": 8.312686920166016, + "mean_kl": 2.5616188049316406, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 4.818500995635986, + "code_tokens": 185, + "cvar95_kl": 10.350135803222656, + "delta_nll": 4.088386535644531, + "max_kl": 13.188481330871582, + "mean_kl": 4.161860466003418, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.716224431991577, + "code_tokens": 106, + "cvar95_kl": 9.02600383758545, + "delta_nll": 3.363229751586914, + "max_kl": 11.46351432800293, + "mean_kl": 3.4261088371276855, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.27358490228652954 + }, + { + "candidate_nll": 5.1325201988220215, + "code_tokens": 64, + "cvar95_kl": 8.975317001342773, + "delta_nll": 3.5899658203125, + "max_kl": 10.031129837036133, + "mean_kl": 3.6231119632720947, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.21875 + }, + { + "candidate_nll": 4.057122707366943, + "code_tokens": 74, + "cvar95_kl": 7.503694534301758, + "delta_nll": 2.946338653564453, + "max_kl": 8.414779663085938, + "mean_kl": 2.9383046627044678, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 0.3378378450870514 + }, + { + "candidate_nll": 3.145627737045288, + "code_tokens": 27, + "cvar95_kl": 8.492475509643555, + "delta_nll": 2.158223867416382, + "max_kl": 9.762248039245605, + "mean_kl": 2.4370038509368896, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 4.848402500152588, + "code_tokens": 72, + "cvar95_kl": 9.141937255859375, + "delta_nll": 4.224564552307129, + "max_kl": 9.60826587677002, + "mean_kl": 4.3385748863220215, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 3.669295310974121, + "code_tokens": 62, + "cvar95_kl": 5.774733543395996, + "delta_nll": 2.735992908477783, + "max_kl": 6.172547340393066, + "mean_kl": 2.620638608932495, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.4032258093357086 + }, + { + "candidate_nll": 3.5554704666137695, + "code_tokens": 42, + "cvar95_kl": 8.068933486938477, + "delta_nll": 2.857304096221924, + "max_kl": 11.021377563476562, + "mean_kl": 2.9032015800476074, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 0.3571428656578064 + }, + { + "candidate_nll": 4.9088358879089355, + "code_tokens": 238, + "cvar95_kl": 9.216582298278809, + "delta_nll": 4.02150297164917, + "max_kl": 12.40401554107666, + "mean_kl": 4.027645111083984, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.18487395346164703 + }, + { + "candidate_nll": 4.171257495880127, + "code_tokens": 68, + "cvar95_kl": 8.754104614257812, + "delta_nll": 3.2567458152770996, + "max_kl": 10.869786262512207, + "mean_kl": 3.2704176902770996, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.36764705181121826 + }, + { + "candidate_nll": 4.2320780754089355, + "code_tokens": 83, + "cvar95_kl": 10.099309921264648, + "delta_nll": 3.2479281425476074, + "max_kl": 13.063088417053223, + "mean_kl": 3.3663830757141113, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.28915661573410034 + }, + { + "candidate_nll": 6.075829982757568, + "code_tokens": 64, + "cvar95_kl": 11.277589797973633, + "delta_nll": 4.8120622634887695, + "max_kl": 12.879749298095703, + "mean_kl": 4.884391784667969, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 0.171875 + }, + { + "candidate_nll": 2.5860376358032227, + "code_tokens": 37, + "cvar95_kl": 4.242875099182129, + "delta_nll": 1.929237723350525, + "max_kl": 4.421916961669922, + "mean_kl": 1.9409809112548828, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 4.346824645996094, + "code_tokens": 27, + "cvar95_kl": 8.059272766113281, + "delta_nll": 2.836897373199463, + "max_kl": 8.221054077148438, + "mean_kl": 3.0882411003112793, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 3.851156711578369, + "code_tokens": 40, + "cvar95_kl": 7.429754257202148, + "delta_nll": 2.959440231323242, + "max_kl": 7.529832363128662, + "mean_kl": 3.211522340774536, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 3.9250240325927734, + "code_tokens": 28, + "cvar95_kl": 8.379828453063965, + "delta_nll": 2.9272513389587402, + "max_kl": 9.595967292785645, + "mean_kl": 3.0551083087921143, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.2202727794647217, + "code_tokens": 59, + "cvar95_kl": 7.447076797485352, + "delta_nll": 2.36564302444458, + "max_kl": 8.114598274230957, + "mean_kl": 2.590769052505493, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.37288135290145874 + }, + { + "candidate_nll": 4.350296974182129, + "code_tokens": 57, + "cvar95_kl": 8.328933715820312, + "delta_nll": 3.780296802520752, + "max_kl": 10.103645324707031, + "mean_kl": 3.5650980472564697, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.24561403691768646 + }, + { + "candidate_nll": 4.285502910614014, + "code_tokens": 25, + "cvar95_kl": 6.605354309082031, + "delta_nll": 2.9015626907348633, + "max_kl": 6.847584247589111, + "mean_kl": 3.0932226181030273, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 2.7095539569854736, + "code_tokens": 48, + "cvar95_kl": 5.971989154815674, + "delta_nll": 1.9624016284942627, + "max_kl": 6.698564529418945, + "mean_kl": 1.904178500175476, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.048312187194824, + "code_tokens": 35, + "cvar95_kl": 6.700927734375, + "delta_nll": 3.1865363121032715, + "max_kl": 6.768379211425781, + "mean_kl": 2.9965391159057617, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 3.0717077255249023, + "code_tokens": 30, + "cvar95_kl": 6.802938461303711, + "delta_nll": 2.4270005226135254, + "max_kl": 8.258716583251953, + "mean_kl": 2.3971261978149414, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.375793695449829, + "code_tokens": 49, + "cvar95_kl": 6.551450252532959, + "delta_nll": 2.252716064453125, + "max_kl": 8.711559295654297, + "mean_kl": 2.468888759613037, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 4.947697162628174, + "code_tokens": 98, + "cvar95_kl": 9.327775001525879, + "delta_nll": 3.9314804077148438, + "max_kl": 10.920431137084961, + "mean_kl": 3.8226168155670166, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.20408163964748383 + }, + { + "candidate_nll": 3.030388116836548, + "code_tokens": 25, + "cvar95_kl": 4.991889953613281, + "delta_nll": 1.649581789970398, + "max_kl": 5.070796966552734, + "mean_kl": 2.0051686763763428, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.191920042037964, + "code_tokens": 27, + "cvar95_kl": 8.155403137207031, + "delta_nll": 2.4141969680786133, + "max_kl": 9.085434913635254, + "mean_kl": 2.5190975666046143, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 0.5555555820465088 + }, + { + "candidate_nll": 2.6808035373687744, + "code_tokens": 20, + "cvar95_kl": 7.3460259437561035, + "delta_nll": 1.635718822479248, + "max_kl": 7.3460259437561035, + "mean_kl": 2.049699306488037, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 0.550000011920929 + }, + { + "candidate_nll": 4.6419758796691895, + "code_tokens": 253, + "cvar95_kl": 9.901167869567871, + "delta_nll": 3.949306011199951, + "max_kl": 12.195657730102539, + "mean_kl": 4.035452365875244, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.20948617160320282 + }, + { + "candidate_nll": 4.189707279205322, + "code_tokens": 55, + "cvar95_kl": 8.55445671081543, + "delta_nll": 3.3605871200561523, + "max_kl": 9.597867012023926, + "mean_kl": 3.4941139221191406, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.7045929431915283, + "code_tokens": 31, + "cvar95_kl": 6.002673625946045, + "delta_nll": 2.4980945587158203, + "max_kl": 6.063689231872559, + "mean_kl": 2.710921287536621, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.25806450843811035 + }, + { + "candidate_nll": 4.413511276245117, + "code_tokens": 35, + "cvar95_kl": 11.002779006958008, + "delta_nll": 3.7092652320861816, + "max_kl": 11.721814155578613, + "mean_kl": 3.683044910430908, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.1123881340026855, + "code_tokens": 61, + "cvar95_kl": 7.090149879455566, + "delta_nll": 2.3095197677612305, + "max_kl": 7.8957133293151855, + "mean_kl": 2.432605743408203, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.44262295961380005 + }, + { + "candidate_nll": 3.821178436279297, + "code_tokens": 55, + "cvar95_kl": 8.215091705322266, + "delta_nll": 2.8664608001708984, + "max_kl": 8.375269889831543, + "mean_kl": 2.870405673980713, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.2181818187236786 + }, + { + "candidate_nll": 4.998034477233887, + "code_tokens": 108, + "cvar95_kl": 9.250978469848633, + "delta_nll": 3.491067409515381, + "max_kl": 16.425533294677734, + "mean_kl": 3.717940092086792, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.17592592537403107 + }, + { + "candidate_nll": 3.704317331314087, + "code_tokens": 58, + "cvar95_kl": 7.26602029800415, + "delta_nll": 2.417011260986328, + "max_kl": 8.903836250305176, + "mean_kl": 2.564934015274048, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.30674409866333, + "code_tokens": 91, + "cvar95_kl": 9.543217658996582, + "delta_nll": 3.4792888164520264, + "max_kl": 12.426045417785645, + "mean_kl": 3.487220525741577, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.2637362778186798 + }, + { + "candidate_nll": 3.4351937770843506, + "code_tokens": 105, + "cvar95_kl": 7.135111331939697, + "delta_nll": 2.671454429626465, + "max_kl": 9.159980773925781, + "mean_kl": 2.8141908645629883, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.3523809611797333 + }, + { + "candidate_nll": 3.923891067504883, + "code_tokens": 49, + "cvar95_kl": 8.41783618927002, + "delta_nll": 3.2766385078430176, + "max_kl": 9.111981391906738, + "mean_kl": 3.078900098800659, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.935849189758301, + "code_tokens": 27, + "cvar95_kl": 8.382678985595703, + "delta_nll": 2.4736862182617188, + "max_kl": 8.712491989135742, + "mean_kl": 2.770604133605957, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 3.1055004596710205, + "code_tokens": 25, + "cvar95_kl": 6.44755744934082, + "delta_nll": 2.0651237964630127, + "max_kl": 6.912771224975586, + "mean_kl": 2.330697774887085, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.6129097938537598, + "code_tokens": 98, + "cvar95_kl": 8.240939140319824, + "delta_nll": 2.3178553581237793, + "max_kl": 9.248461723327637, + "mean_kl": 2.5995595455169678, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.33673468232154846 + }, + { + "candidate_nll": 3.2958180904388428, + "code_tokens": 23, + "cvar95_kl": 5.179538726806641, + "delta_nll": 2.646036386489868, + "max_kl": 5.279382228851318, + "mean_kl": 2.5151517391204834, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 5.042038440704346, + "code_tokens": 31, + "cvar95_kl": 7.8667802810668945, + "delta_nll": 3.1766648292541504, + "max_kl": 7.885072231292725, + "mean_kl": 3.816328525543213, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 3.539789915084839, + "code_tokens": 36, + "cvar95_kl": 9.860128402709961, + "delta_nll": 2.791696786880493, + "max_kl": 11.4192476272583, + "mean_kl": 2.7731728553771973, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 3.825580358505249, + "code_tokens": 38, + "cvar95_kl": 6.685905456542969, + "delta_nll": 2.3919029235839844, + "max_kl": 6.8442583084106445, + "mean_kl": 2.938861608505249, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.2631579041481018 + }, + { + "candidate_nll": 3.0732970237731934, + "code_tokens": 25, + "cvar95_kl": 4.127978324890137, + "delta_nll": 1.4672071933746338, + "max_kl": 5.1190104484558105, + "mean_kl": 1.479907512664795, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 3.096405506134033, + "code_tokens": 26, + "cvar95_kl": 5.722417831420898, + "delta_nll": 2.216956377029419, + "max_kl": 6.202585220336914, + "mean_kl": 2.6522610187530518, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 4.400309085845947, + "code_tokens": 62, + "cvar95_kl": 7.940323829650879, + "delta_nll": 3.657866954803467, + "max_kl": 8.996227264404297, + "mean_kl": 3.4946324825286865, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.24193547666072845 + }, + { + "candidate_nll": 4.416086196899414, + "code_tokens": 36, + "cvar95_kl": 6.762676239013672, + "delta_nll": 2.6775670051574707, + "max_kl": 7.114258289337158, + "mean_kl": 3.152003526687622, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.311461448669434, + "code_tokens": 119, + "cvar95_kl": 7.713494777679443, + "delta_nll": 2.980872631072998, + "max_kl": 9.32614803314209, + "mean_kl": 3.1180686950683594, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.859308958053589, + "code_tokens": 37, + "cvar95_kl": 8.598276138305664, + "delta_nll": 3.5466012954711914, + "max_kl": 9.279450416564941, + "mean_kl": 3.3296196460723877, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 4.086911201477051, + "code_tokens": 28, + "cvar95_kl": 9.06125259399414, + "delta_nll": 3.304177761077881, + "max_kl": 9.383522987365723, + "mean_kl": 3.5673060417175293, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.696498155593872, + "code_tokens": 143, + "cvar95_kl": 8.760497093200684, + "delta_nll": 3.1301512718200684, + "max_kl": 11.397586822509766, + "mean_kl": 3.1846792697906494, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.35664334893226624 + }, + { + "candidate_nll": 4.030398845672607, + "code_tokens": 29, + "cvar95_kl": 8.047343254089355, + "delta_nll": 2.2480926513671875, + "max_kl": 8.246639251708984, + "mean_kl": 2.866730213165283, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 4.758358001708984, + "code_tokens": 28, + "cvar95_kl": 9.376272201538086, + "delta_nll": 3.6858696937561035, + "max_kl": 10.144646644592285, + "mean_kl": 3.594282388687134, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 4.760482311248779, + "code_tokens": 30, + "cvar95_kl": 10.591886520385742, + "delta_nll": 3.4964447021484375, + "max_kl": 11.937950134277344, + "mean_kl": 4.004704475402832, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.9982805252075195, + "code_tokens": 85, + "cvar95_kl": 10.44526195526123, + "delta_nll": 3.576470375061035, + "max_kl": 13.309831619262695, + "mean_kl": 3.4726788997650146, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.3764705955982208 + }, + { + "candidate_nll": 4.6009602546691895, + "code_tokens": 44, + "cvar95_kl": 9.411850929260254, + "delta_nll": 3.6237165927886963, + "max_kl": 10.022635459899902, + "mean_kl": 3.5574235916137695, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.34090909361839294 + }, + { + "candidate_nll": 4.397696018218994, + "code_tokens": 19, + "cvar95_kl": 10.082947731018066, + "delta_nll": 3.5935938358306885, + "max_kl": 10.082947731018066, + "mean_kl": 3.2247488498687744, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 0.4736842215061188 + }, + { + "candidate_nll": 3.903355121612549, + "code_tokens": 113, + "cvar95_kl": 7.333526611328125, + "delta_nll": 3.1659321784973145, + "max_kl": 8.274863243103027, + "mean_kl": 3.077901840209961, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.32743361592292786 + }, + { + "candidate_nll": 4.129490852355957, + "code_tokens": 40, + "cvar95_kl": 8.713447570800781, + "delta_nll": 2.654296398162842, + "max_kl": 10.362902641296387, + "mean_kl": 3.260939836502075, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 5.398182392120361, + "code_tokens": 51, + "cvar95_kl": 9.772160530090332, + "delta_nll": 3.79152250289917, + "max_kl": 10.414891242980957, + "mean_kl": 4.181020736694336, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 4.423774719238281, + "code_tokens": 55, + "cvar95_kl": 8.180773735046387, + "delta_nll": 2.8427422046661377, + "max_kl": 9.483924865722656, + "mean_kl": 3.1388192176818848, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.23636363446712494 + }, + { + "candidate_nll": 2.9348113536834717, + "code_tokens": 27, + "cvar95_kl": 4.930449485778809, + "delta_nll": 1.8047430515289307, + "max_kl": 4.9424896240234375, + "mean_kl": 1.7280102968215942, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 0.6296296119689941 + }, + { + "candidate_nll": 4.840608596801758, + "code_tokens": 32, + "cvar95_kl": 8.62011432647705, + "delta_nll": 4.000802516937256, + "max_kl": 9.645954132080078, + "mean_kl": 3.9404635429382324, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 3.2516489028930664, + "code_tokens": 64, + "cvar95_kl": 7.569957733154297, + "delta_nll": 2.5991108417510986, + "max_kl": 8.616204261779785, + "mean_kl": 2.686995267868042, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.421875 + }, + { + "candidate_nll": 4.449107646942139, + "code_tokens": 35, + "cvar95_kl": 9.433780670166016, + "delta_nll": 3.3994534015655518, + "max_kl": 9.892304420471191, + "mean_kl": 3.3995418548583984, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.5983781814575195, + "code_tokens": 152, + "cvar95_kl": 9.19632625579834, + "delta_nll": 4.146530628204346, + "max_kl": 10.344772338867188, + "mean_kl": 3.9511373043060303, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.21710526943206787 + }, + { + "candidate_nll": 4.307426929473877, + "code_tokens": 77, + "cvar95_kl": 7.858419418334961, + "delta_nll": 3.0460925102233887, + "max_kl": 9.064900398254395, + "mean_kl": 2.9328339099884033, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.31168830394744873 + }, + { + "candidate_nll": 3.531881809234619, + "code_tokens": 77, + "cvar95_kl": 8.309663772583008, + "delta_nll": 2.9636006355285645, + "max_kl": 10.657027244567871, + "mean_kl": 2.9059054851531982, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.350649356842041 + }, + { + "candidate_nll": 2.691770315170288, + "code_tokens": 71, + "cvar95_kl": 6.6993303298950195, + "delta_nll": 2.1296815872192383, + "max_kl": 8.014695167541504, + "mean_kl": 2.2305727005004883, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.4647887349128723 + }, + { + "candidate_nll": 5.932209014892578, + "code_tokens": 112, + "cvar95_kl": 10.70285701751709, + "delta_nll": 5.053829669952393, + "max_kl": 11.971454620361328, + "mean_kl": 4.8836493492126465, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 2.930839776992798, + "code_tokens": 111, + "cvar95_kl": 6.527050495147705, + "delta_nll": 2.2109360694885254, + "max_kl": 7.319986343383789, + "mean_kl": 2.217437267303467, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.4954954981803894 + }, + { + "candidate_nll": 4.213109016418457, + "code_tokens": 62, + "cvar95_kl": 9.468558311462402, + "delta_nll": 3.425212860107422, + "max_kl": 10.510729789733887, + "mean_kl": 3.5166471004486084, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.33870968222618103 + }, + { + "candidate_nll": 2.347506523132324, + "code_tokens": 52, + "cvar95_kl": 5.063136577606201, + "delta_nll": 1.736926555633545, + "max_kl": 5.728111743927002, + "mean_kl": 1.7054775953292847, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 3.962583303451538, + "code_tokens": 24, + "cvar95_kl": 5.308831214904785, + "delta_nll": 2.067356586456299, + "max_kl": 5.8017802238464355, + "mean_kl": 2.4170773029327393, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.6365959644317627, + "code_tokens": 48, + "cvar95_kl": 8.409268379211426, + "delta_nll": 2.744826078414917, + "max_kl": 9.195842742919922, + "mean_kl": 3.111039400100708, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.3541666567325592 + }, + { + "candidate_nll": 3.9240360260009766, + "code_tokens": 98, + "cvar95_kl": 8.4772310256958, + "delta_nll": 2.8737430572509766, + "max_kl": 10.135868072509766, + "mean_kl": 2.9820854663848877, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.30612245202064514 + }, + { + "candidate_nll": 3.463618278503418, + "code_tokens": 48, + "cvar95_kl": 7.141387939453125, + "delta_nll": 2.470932722091675, + "max_kl": 7.8366217613220215, + "mean_kl": 2.5731656551361084, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.041098594665527, + "code_tokens": 56, + "cvar95_kl": 11.217723846435547, + "delta_nll": 4.0951032638549805, + "max_kl": 12.00446891784668, + "mean_kl": 4.253112316131592, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.1964285671710968 + }, + { + "candidate_nll": 4.636012077331543, + "code_tokens": 88, + "cvar95_kl": 10.030560493469238, + "delta_nll": 4.034735202789307, + "max_kl": 12.170021057128906, + "mean_kl": 3.9523508548736572, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.23863635957241058 + }, + { + "candidate_nll": 4.6148362159729, + "code_tokens": 50, + "cvar95_kl": 8.575749397277832, + "delta_nll": 2.9813060760498047, + "max_kl": 9.317227363586426, + "mean_kl": 3.113088369369507, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 2.641106605529785, + "code_tokens": 85, + "cvar95_kl": 6.125792503356934, + "delta_nll": 2.2041194438934326, + "max_kl": 9.757617950439453, + "mean_kl": 2.0375537872314453, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.5176470875740051 + }, + { + "candidate_nll": 4.48350715637207, + "code_tokens": 27, + "cvar95_kl": 8.183242797851562, + "delta_nll": 3.4092941284179688, + "max_kl": 8.239941596984863, + "mean_kl": 3.751373529434204, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.25925925374031067 + }, + { + "candidate_nll": 4.387979030609131, + "code_tokens": 48, + "cvar95_kl": 9.607941627502441, + "delta_nll": 3.2703604698181152, + "max_kl": 11.039531707763672, + "mean_kl": 3.6071135997772217, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.0765380859375, + "code_tokens": 29, + "cvar95_kl": 9.803777694702148, + "delta_nll": 3.4286909103393555, + "max_kl": 10.304572105407715, + "mean_kl": 3.5836381912231445, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 4.116324424743652, + "code_tokens": 63, + "cvar95_kl": 8.600378036499023, + "delta_nll": 3.1781091690063477, + "max_kl": 10.631144523620605, + "mean_kl": 3.355180501937866, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.319093704223633, + "code_tokens": 89, + "cvar95_kl": 7.565637111663818, + "delta_nll": 3.6440351009368896, + "max_kl": 9.221755027770996, + "mean_kl": 3.5577759742736816, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.2584269642829895 + }, + { + "candidate_nll": 4.782802581787109, + "code_tokens": 133, + "cvar95_kl": 8.218850135803223, + "delta_nll": 3.4385242462158203, + "max_kl": 9.37411880493164, + "mean_kl": 3.5072762966156006, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.24812030792236328 + }, + { + "candidate_nll": 2.649731159210205, + "code_tokens": 25, + "cvar95_kl": 4.411849021911621, + "delta_nll": 1.701194405555725, + "max_kl": 4.41336727142334, + "mean_kl": 1.79408597946167, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.4399999976158142 + }, + { + "candidate_nll": 3.2049715518951416, + "code_tokens": 33, + "cvar95_kl": 6.097731113433838, + "delta_nll": 2.1676712036132812, + "max_kl": 6.777646064758301, + "mean_kl": 2.543184757232666, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.72031831741333, + "code_tokens": 93, + "cvar95_kl": 7.057049751281738, + "delta_nll": 1.96354341506958, + "max_kl": 7.932240962982178, + "mean_kl": 2.2403383255004883, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.4301075339317322 + }, + { + "candidate_nll": 4.1528401374816895, + "code_tokens": 101, + "cvar95_kl": 7.750070095062256, + "delta_nll": 3.1844406127929688, + "max_kl": 9.682236671447754, + "mean_kl": 3.068723678588867, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.32673266530036926 + }, + { + "candidate_nll": 4.574173450469971, + "code_tokens": 152, + "cvar95_kl": 10.51678466796875, + "delta_nll": 4.009410381317139, + "max_kl": 11.808062553405762, + "mean_kl": 3.9581387042999268, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.2763157784938812 + }, + { + "candidate_nll": 3.925682306289673, + "code_tokens": 33, + "cvar95_kl": 7.1884613037109375, + "delta_nll": 2.524810791015625, + "max_kl": 7.533471584320068, + "mean_kl": 2.887241840362549, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.6756834983825684, + "code_tokens": 29, + "cvar95_kl": 6.77570915222168, + "delta_nll": 2.389220714569092, + "max_kl": 7.397458553314209, + "mean_kl": 2.59285044670105, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.49614143371582, + "code_tokens": 27, + "cvar95_kl": 9.631362915039062, + "delta_nll": 3.038071632385254, + "max_kl": 10.850263595581055, + "mean_kl": 3.4703521728515625, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.40740740299224854 + }, + { + "candidate_nll": 3.823972225189209, + "code_tokens": 33, + "cvar95_kl": 7.458729267120361, + "delta_nll": 3.513932228088379, + "max_kl": 7.855655670166016, + "mean_kl": 3.4396872520446777, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.6173160076141357, + "code_tokens": 27, + "cvar95_kl": 7.212612152099609, + "delta_nll": 2.8209176063537598, + "max_kl": 7.624324798583984, + "mean_kl": 3.011335849761963, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.29629629850387573 + }, + { + "candidate_nll": 3.916788101196289, + "code_tokens": 84, + "cvar95_kl": 9.050057411193848, + "delta_nll": 3.2128946781158447, + "max_kl": 10.055676460266113, + "mean_kl": 3.359126567840576, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.2976190447807312 + }, + { + "candidate_nll": 3.608377456665039, + "code_tokens": 103, + "cvar95_kl": 6.286457061767578, + "delta_nll": 2.5629076957702637, + "max_kl": 8.048223495483398, + "mean_kl": 2.584554672241211, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.3300970792770386 + }, + { + "candidate_nll": 4.248344898223877, + "code_tokens": 88, + "cvar95_kl": 8.02261734008789, + "delta_nll": 3.3319976329803467, + "max_kl": 8.549182891845703, + "mean_kl": 3.283386468887329, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.2954545319080353 + }, + { + "candidate_nll": 5.50300931930542, + "code_tokens": 62, + "cvar95_kl": 9.160257339477539, + "delta_nll": 4.19407844543457, + "max_kl": 10.05349349975586, + "mean_kl": 3.886956214904785, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.19354838132858276 + }, + { + "candidate_nll": 4.5191473960876465, + "code_tokens": 24, + "cvar95_kl": 5.93216609954834, + "delta_nll": 2.982438564300537, + "max_kl": 6.170562267303467, + "mean_kl": 3.0594139099121094, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 4.068112373352051, + "code_tokens": 34, + "cvar95_kl": 7.576630115509033, + "delta_nll": 2.7296602725982666, + "max_kl": 7.674261093139648, + "mean_kl": 2.6478383541107178, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.279271125793457, + "code_tokens": 43, + "cvar95_kl": 7.323395252227783, + "delta_nll": 3.559525728225708, + "max_kl": 7.4863433837890625, + "mean_kl": 3.5518226623535156, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 4.862390518188477, + "code_tokens": 54, + "cvar95_kl": 9.728126525878906, + "delta_nll": 3.3730852603912354, + "max_kl": 11.563558578491211, + "mean_kl": 3.365175485610962, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.24074074625968933 + }, + { + "candidate_nll": 3.564471960067749, + "code_tokens": 50, + "cvar95_kl": 6.669740200042725, + "delta_nll": 2.8345866203308105, + "max_kl": 7.271946907043457, + "mean_kl": 2.8101744651794434, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 4.533908367156982, + "code_tokens": 235, + "cvar95_kl": 9.094620704650879, + "delta_nll": 3.776397943496704, + "max_kl": 12.205676078796387, + "mean_kl": 3.7723934650421143, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.31489360332489014 + }, + { + "candidate_nll": 4.828789710998535, + "code_tokens": 33, + "cvar95_kl": 6.809917449951172, + "delta_nll": 2.8001346588134766, + "max_kl": 7.232245922088623, + "mean_kl": 3.401658058166504, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 4.153748512268066, + "code_tokens": 45, + "cvar95_kl": 7.954519748687744, + "delta_nll": 3.167246103286743, + "max_kl": 8.083971977233887, + "mean_kl": 3.14068603515625, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.510209560394287, + "code_tokens": 28, + "cvar95_kl": 12.882226943969727, + "delta_nll": 3.887249708175659, + "max_kl": 14.7774076461792, + "mean_kl": 3.7129967212677, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.5054006576538086, + "code_tokens": 49, + "cvar95_kl": 7.019104480743408, + "delta_nll": 2.827136278152466, + "max_kl": 7.970180988311768, + "mean_kl": 2.878164291381836, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 0.40816327929496765 + }, + { + "candidate_nll": 4.765048503875732, + "code_tokens": 107, + "cvar95_kl": 11.186854362487793, + "delta_nll": 4.5156331062316895, + "max_kl": 12.833584785461426, + "mean_kl": 4.31416654586792, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.25233644247055054 + }, + { + "candidate_nll": 4.5356221199035645, + "code_tokens": 48, + "cvar95_kl": 9.347851753234863, + "delta_nll": 3.3147120475769043, + "max_kl": 9.88271713256836, + "mean_kl": 3.558807134628296, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.2708333432674408 + }, + { + "candidate_nll": 3.3299381732940674, + "code_tokens": 40, + "cvar95_kl": 5.647253036499023, + "delta_nll": 1.490397334098816, + "max_kl": 5.708614826202393, + "mean_kl": 1.8685283660888672, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 3.116544723510742, + "code_tokens": 46, + "cvar95_kl": 7.795646667480469, + "delta_nll": 2.6470611095428467, + "max_kl": 10.0137939453125, + "mean_kl": 2.792222499847412, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.095095157623291, + "code_tokens": 31, + "cvar95_kl": 7.33782434463501, + "delta_nll": 3.052645444869995, + "max_kl": 7.410007476806641, + "mean_kl": 3.0309579372406006, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 2.070950984954834, + "code_tokens": 58, + "cvar95_kl": 5.976633548736572, + "delta_nll": 1.6544665098190308, + "max_kl": 6.836325168609619, + "mean_kl": 1.6580966711044312, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.568965494632721 + }, + { + "candidate_nll": 3.4317870140075684, + "code_tokens": 49, + "cvar95_kl": 7.685487270355225, + "delta_nll": 2.3140172958374023, + "max_kl": 11.503105163574219, + "mean_kl": 2.471754550933838, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.3469387888908386 + }, + { + "candidate_nll": 3.0461912155151367, + "code_tokens": 18, + "cvar95_kl": 7.359262466430664, + "delta_nll": 2.099597692489624, + "max_kl": 7.359262466430664, + "mean_kl": 2.106541633605957, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 3.425178289413452, + "code_tokens": 28, + "cvar95_kl": 5.484388828277588, + "delta_nll": 2.2657666206359863, + "max_kl": 5.7342305183410645, + "mean_kl": 2.5417423248291016, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 0.3928571343421936 + }, + { + "candidate_nll": 3.984658718109131, + "code_tokens": 57, + "cvar95_kl": 6.9815216064453125, + "delta_nll": 3.2425918579101562, + "max_kl": 7.086539268493652, + "mean_kl": 3.284390449523926, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.24561403691768646 + }, + { + "candidate_nll": 3.6269729137420654, + "code_tokens": 88, + "cvar95_kl": 7.992048740386963, + "delta_nll": 2.926862955093384, + "max_kl": 9.502729415893555, + "mean_kl": 3.0651886463165283, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.3295454680919647 + }, + { + "candidate_nll": 4.427010536193848, + "code_tokens": 23, + "cvar95_kl": 9.60407829284668, + "delta_nll": 3.2282238006591797, + "max_kl": 9.87773323059082, + "mean_kl": 3.4240267276763916, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.3913043439388275 + }, + { + "candidate_nll": 5.225218772888184, + "code_tokens": 40, + "cvar95_kl": 10.948299407958984, + "delta_nll": 3.965761423110962, + "max_kl": 12.029520988464355, + "mean_kl": 4.144801139831543, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.22499999403953552 + }, + { + "candidate_nll": 3.8633036613464355, + "code_tokens": 20, + "cvar95_kl": 8.649494171142578, + "delta_nll": 2.7020885944366455, + "max_kl": 8.649494171142578, + "mean_kl": 3.192800521850586, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.0522613525390625, + "code_tokens": 63, + "cvar95_kl": 7.232001781463623, + "delta_nll": 3.036475658416748, + "max_kl": 7.617827892303467, + "mean_kl": 3.077618360519409, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.3650793731212616 + }, + { + "candidate_nll": 3.939974308013916, + "code_tokens": 39, + "cvar95_kl": 9.018871307373047, + "delta_nll": 2.987293004989624, + "max_kl": 9.353965759277344, + "mean_kl": 3.0929932594299316, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.740537643432617, + "code_tokens": 50, + "cvar95_kl": 9.137890815734863, + "delta_nll": 3.3286728858947754, + "max_kl": 9.777397155761719, + "mean_kl": 3.4647982120513916, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 4.196115493774414, + "code_tokens": 63, + "cvar95_kl": 8.735008239746094, + "delta_nll": 3.228638172149658, + "max_kl": 9.45785903930664, + "mean_kl": 3.5239179134368896, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.871257781982422, + "code_tokens": 46, + "cvar95_kl": 7.892362594604492, + "delta_nll": 3.898695707321167, + "max_kl": 8.455352783203125, + "mean_kl": 3.9613115787506104, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 3.460775852203369, + "code_tokens": 25, + "cvar95_kl": 8.163870811462402, + "delta_nll": 2.781881332397461, + "max_kl": 8.680957794189453, + "mean_kl": 2.9074506759643555, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.616760492324829, + "code_tokens": 111, + "cvar95_kl": 6.144054889678955, + "delta_nll": 2.802494525909424, + "max_kl": 6.909591197967529, + "mean_kl": 2.8691904544830322, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.27927929162979126 + }, + { + "candidate_nll": 3.964010000228882, + "code_tokens": 71, + "cvar95_kl": 7.0361809730529785, + "delta_nll": 3.235452175140381, + "max_kl": 9.11245346069336, + "mean_kl": 3.0726962089538574, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.2535211145877838 + }, + { + "candidate_nll": 5.094749927520752, + "code_tokens": 28, + "cvar95_kl": 8.745904922485352, + "delta_nll": 3.7240262031555176, + "max_kl": 10.127215385437012, + "mean_kl": 3.992933750152588, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 4.499088287353516, + "code_tokens": 25, + "cvar95_kl": 9.782880783081055, + "delta_nll": 3.2986323833465576, + "max_kl": 10.592630386352539, + "mean_kl": 3.460158586502075, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.3199999928474426 + }, + { + "candidate_nll": 3.648460865020752, + "code_tokens": 40, + "cvar95_kl": 6.786896228790283, + "delta_nll": 2.345689058303833, + "max_kl": 7.616542816162109, + "mean_kl": 2.4241597652435303, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.42500001192092896 + }, + { + "candidate_nll": 4.043676376342773, + "code_tokens": 177, + "cvar95_kl": 8.043177604675293, + "delta_nll": 2.999232292175293, + "max_kl": 10.383049011230469, + "mean_kl": 3.2854807376861572, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.32203391194343567 + }, + { + "candidate_nll": 3.7546467781066895, + "code_tokens": 51, + "cvar95_kl": 6.422715663909912, + "delta_nll": 2.663468360900879, + "max_kl": 7.68163537979126, + "mean_kl": 2.675055742263794, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 2.758876085281372, + "code_tokens": 69, + "cvar95_kl": 6.916262626647949, + "delta_nll": 2.304415702819824, + "max_kl": 7.931644439697266, + "mean_kl": 2.2587549686431885, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.49275362491607666 + }, + { + "candidate_nll": 3.4060487747192383, + "code_tokens": 26, + "cvar95_kl": 7.611032485961914, + "delta_nll": 2.6577887535095215, + "max_kl": 7.793169021606445, + "mean_kl": 2.652899742126465, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.416278123855591, + "code_tokens": 24, + "cvar95_kl": 4.783596515655518, + "delta_nll": 1.897938847541809, + "max_kl": 5.3328375816345215, + "mean_kl": 2.1025984287261963, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.412638187408447, + "code_tokens": 15, + "cvar95_kl": 9.071919441223145, + "delta_nll": 3.262162208557129, + "max_kl": 9.071919441223145, + "mean_kl": 3.037976026535034, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.087111473083496, + "code_tokens": 43, + "cvar95_kl": 9.573158264160156, + "delta_nll": 2.937762498855591, + "max_kl": 10.630472183227539, + "mean_kl": 3.2438199520111084, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 0.3720930218696594 + }, + { + "candidate_nll": 3.566969633102417, + "code_tokens": 18, + "cvar95_kl": 4.9060869216918945, + "delta_nll": 2.4746439456939697, + "max_kl": 4.9060869216918945, + "mean_kl": 2.359306812286377, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.431206464767456, + "code_tokens": 49, + "cvar95_kl": 7.22119140625, + "delta_nll": 2.7032129764556885, + "max_kl": 8.254953384399414, + "mean_kl": 2.6924307346343994, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.36734694242477417 + }, + { + "candidate_nll": 3.634838342666626, + "code_tokens": 59, + "cvar95_kl": 7.208563327789307, + "delta_nll": 2.7670822143554688, + "max_kl": 8.839435577392578, + "mean_kl": 2.865492582321167, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.3050847351551056 + }, + { + "candidate_nll": 2.8888986110687256, + "code_tokens": 221, + "cvar95_kl": 7.264266490936279, + "delta_nll": 2.3036324977874756, + "max_kl": 8.755000114440918, + "mean_kl": 2.3615994453430176, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.4524886906147003 + }, + { + "candidate_nll": 4.760019302368164, + "code_tokens": 25, + "cvar95_kl": 9.261526107788086, + "delta_nll": 3.5581462383270264, + "max_kl": 9.363036155700684, + "mean_kl": 3.705427885055542, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 3.5761380195617676, + "code_tokens": 36, + "cvar95_kl": 8.012529373168945, + "delta_nll": 2.852217435836792, + "max_kl": 8.828361511230469, + "mean_kl": 3.0787675380706787, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 4.41171932220459, + "code_tokens": 158, + "cvar95_kl": 10.435530662536621, + "delta_nll": 3.7675745487213135, + "max_kl": 14.330744743347168, + "mean_kl": 3.731595277786255, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.14556962251663208 + }, + { + "candidate_nll": 4.4435811042785645, + "code_tokens": 92, + "cvar95_kl": 8.67955207824707, + "delta_nll": 3.1280460357666016, + "max_kl": 10.552438735961914, + "mean_kl": 3.115156412124634, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.4021739065647125 + }, + { + "candidate_nll": 3.1147351264953613, + "code_tokens": 102, + "cvar95_kl": 5.949742794036865, + "delta_nll": 2.2006924152374268, + "max_kl": 6.974618911743164, + "mean_kl": 2.1640686988830566, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.4019607901573181 + }, + { + "candidate_nll": 4.17245626449585, + "code_tokens": 171, + "cvar95_kl": 9.335532188415527, + "delta_nll": 3.562486171722412, + "max_kl": 10.911084175109863, + "mean_kl": 3.5478315353393555, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.28654971718788147 + }, + { + "candidate_nll": 4.117778301239014, + "code_tokens": 40, + "cvar95_kl": 9.919681549072266, + "delta_nll": 2.912914276123047, + "max_kl": 10.982097625732422, + "mean_kl": 3.1289889812469482, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 4.338740825653076, + "code_tokens": 48, + "cvar95_kl": 7.213616847991943, + "delta_nll": 3.102726936340332, + "max_kl": 8.454635620117188, + "mean_kl": 3.1076581478118896, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.3958333432674408 + }, + { + "candidate_nll": 4.753991603851318, + "code_tokens": 24, + "cvar95_kl": 8.748641967773438, + "delta_nll": 3.149071216583252, + "max_kl": 9.45405101776123, + "mean_kl": 3.749749183654785, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.635478973388672, + "code_tokens": 48, + "cvar95_kl": 6.850613117218018, + "delta_nll": 2.6483747959136963, + "max_kl": 7.25504207611084, + "mean_kl": 2.4848735332489014, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.198665142059326, + "code_tokens": 25, + "cvar95_kl": 7.195574760437012, + "delta_nll": 1.8117820024490356, + "max_kl": 7.717532634735107, + "mean_kl": 1.9783635139465332, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 4.77855110168457, + "code_tokens": 30, + "cvar95_kl": 8.025188446044922, + "delta_nll": 3.279806137084961, + "max_kl": 8.419042587280273, + "mean_kl": 3.119933843612671, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 0.4333333373069763 + }, + { + "candidate_nll": 3.4909470081329346, + "code_tokens": 19, + "cvar95_kl": 7.1393280029296875, + "delta_nll": 2.064129114151001, + "max_kl": 7.1393280029296875, + "mean_kl": 2.3033227920532227, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 0.42105263471603394 + }, + { + "candidate_nll": 2.8180196285247803, + "code_tokens": 12, + "cvar95_kl": 4.249669075012207, + "delta_nll": 1.7389471530914307, + "max_kl": 4.249669075012207, + "mean_kl": 1.9539265632629395, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.197563171386719, + "code_tokens": 37, + "cvar95_kl": 9.93562126159668, + "delta_nll": 2.8572535514831543, + "max_kl": 10.113757133483887, + "mean_kl": 3.1309657096862793, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 3.1713600158691406, + "code_tokens": 33, + "cvar95_kl": 5.020965576171875, + "delta_nll": 1.9659302234649658, + "max_kl": 5.321965217590332, + "mean_kl": 2.086520195007324, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 5.007024765014648, + "code_tokens": 94, + "cvar95_kl": 9.6576509475708, + "delta_nll": 4.154914855957031, + "max_kl": 10.733749389648438, + "mean_kl": 4.320420265197754, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.25531914830207825 + }, + { + "candidate_nll": 4.14772367477417, + "code_tokens": 90, + "cvar95_kl": 10.358808517456055, + "delta_nll": 3.522958755493164, + "max_kl": 13.715937614440918, + "mean_kl": 3.5629096031188965, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.628603935241699, + "code_tokens": 47, + "cvar95_kl": 6.182680130004883, + "delta_nll": 2.180840015411377, + "max_kl": 6.913046836853027, + "mean_kl": 2.554387092590332, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 0.2978723347187042 + }, + { + "candidate_nll": 3.7228832244873047, + "code_tokens": 61, + "cvar95_kl": 7.807807922363281, + "delta_nll": 2.799049139022827, + "max_kl": 9.868509292602539, + "mean_kl": 3.0939106941223145, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.31147539615631104 + }, + { + "candidate_nll": 3.9421374797821045, + "code_tokens": 39, + "cvar95_kl": 6.523212432861328, + "delta_nll": 2.7186226844787598, + "max_kl": 6.7033467292785645, + "mean_kl": 2.855170249938965, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 4.121978759765625, + "code_tokens": 152, + "cvar95_kl": 8.425084114074707, + "delta_nll": 3.189816951751709, + "max_kl": 9.317445755004883, + "mean_kl": 3.2738711833953857, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.30263158679008484 + }, + { + "candidate_nll": 4.452208042144775, + "code_tokens": 100, + "cvar95_kl": 9.484735488891602, + "delta_nll": 3.4832255840301514, + "max_kl": 10.066777229309082, + "mean_kl": 3.4475014209747314, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.25999999046325684 + }, + { + "candidate_nll": 3.4196231365203857, + "code_tokens": 89, + "cvar95_kl": 7.975677490234375, + "delta_nll": 2.7986953258514404, + "max_kl": 8.539787292480469, + "mean_kl": 2.5494375228881836, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.449438214302063 + }, + { + "candidate_nll": 4.548503875732422, + "code_tokens": 40, + "cvar95_kl": 9.474424362182617, + "delta_nll": 3.2375025749206543, + "max_kl": 10.215795516967773, + "mean_kl": 3.4997658729553223, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 3.6583476066589355, + "code_tokens": 76, + "cvar95_kl": 7.677002429962158, + "delta_nll": 2.9567806720733643, + "max_kl": 8.248167991638184, + "mean_kl": 2.9498467445373535, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.2763157784938812 + }, + { + "candidate_nll": 3.821094036102295, + "code_tokens": 56, + "cvar95_kl": 7.732815265655518, + "delta_nll": 2.5721068382263184, + "max_kl": 9.393145561218262, + "mean_kl": 2.6279149055480957, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.3392857015132904 + }, + { + "candidate_nll": 5.31335973739624, + "code_tokens": 34, + "cvar95_kl": 9.718441009521484, + "delta_nll": 3.8239293098449707, + "max_kl": 10.528529167175293, + "mean_kl": 3.733424425125122, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.20588235557079315 + }, + { + "candidate_nll": 5.169817924499512, + "code_tokens": 96, + "cvar95_kl": 11.119587898254395, + "delta_nll": 4.491706371307373, + "max_kl": 12.048755645751953, + "mean_kl": 4.3527703285217285, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.2604166567325592 + }, + { + "candidate_nll": 3.355297803878784, + "code_tokens": 451, + "cvar95_kl": 9.167081832885742, + "delta_nll": 2.656602621078491, + "max_kl": 15.762304306030273, + "mean_kl": 2.624727487564087, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.36807096004486084 + }, + { + "candidate_nll": 4.812944412231445, + "code_tokens": 37, + "cvar95_kl": 9.363646507263184, + "delta_nll": 3.200634717941284, + "max_kl": 11.92435359954834, + "mean_kl": 3.4442007541656494, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 4.1187920570373535, + "code_tokens": 42, + "cvar95_kl": 8.451628684997559, + "delta_nll": 2.8771941661834717, + "max_kl": 10.694687843322754, + "mean_kl": 3.1870357990264893, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 0.261904776096344 + }, + { + "candidate_nll": 3.786756992340088, + "code_tokens": 33, + "cvar95_kl": 6.7056884765625, + "delta_nll": 2.77000093460083, + "max_kl": 6.8975019454956055, + "mean_kl": 2.710300922393799, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.3030303120613098 + }, + { + "candidate_nll": 4.030956268310547, + "code_tokens": 43, + "cvar95_kl": 7.819770812988281, + "delta_nll": 3.1049046516418457, + "max_kl": 8.754584312438965, + "mean_kl": 3.071894884109497, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 3.9843978881835938, + "code_tokens": 73, + "cvar95_kl": 8.168511390686035, + "delta_nll": 3.2479615211486816, + "max_kl": 9.058589935302734, + "mean_kl": 3.313755750656128, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.2876712381839752 + }, + { + "candidate_nll": 3.206409454345703, + "code_tokens": 15, + "cvar95_kl": 5.998850345611572, + "delta_nll": 2.293933391571045, + "max_kl": 5.998850345611572, + "mean_kl": 2.111158847808838, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.488107681274414, + "code_tokens": 34, + "cvar95_kl": 8.503981590270996, + "delta_nll": 3.22902774810791, + "max_kl": 8.577134132385254, + "mean_kl": 3.8142614364624023, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 4.348793983459473, + "code_tokens": 136, + "cvar95_kl": 8.126666069030762, + "delta_nll": 3.2403106689453125, + "max_kl": 10.485770225524902, + "mean_kl": 3.1457481384277344, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 2.769252061843872, + "code_tokens": 17, + "cvar95_kl": 3.900637149810791, + "delta_nll": 1.523095965385437, + "max_kl": 3.900637149810791, + "mean_kl": 1.8659559488296509, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.4117647111415863 + }, + { + "candidate_nll": 4.267989158630371, + "code_tokens": 33, + "cvar95_kl": 10.936059951782227, + "delta_nll": 3.4749345779418945, + "max_kl": 11.979005813598633, + "mean_kl": 3.379603385925293, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.411950588226318, + "code_tokens": 38, + "cvar95_kl": 7.651898384094238, + "delta_nll": 3.5695037841796875, + "max_kl": 7.901939868927002, + "mean_kl": 3.4756686687469482, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.2368421107530594 + }, + { + "candidate_nll": 3.7596750259399414, + "code_tokens": 61, + "cvar95_kl": 9.475969314575195, + "delta_nll": 3.3621504306793213, + "max_kl": 10.762118339538574, + "mean_kl": 3.42740797996521, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.24590164422988892 + }, + { + "candidate_nll": 3.500070571899414, + "code_tokens": 143, + "cvar95_kl": 7.455878257751465, + "delta_nll": 2.7643511295318604, + "max_kl": 8.710003852844238, + "mean_kl": 2.901580572128296, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.3916083872318268 + }, + { + "candidate_nll": 3.3099851608276367, + "code_tokens": 41, + "cvar95_kl": 8.2288818359375, + "delta_nll": 2.220919132232666, + "max_kl": 8.75992488861084, + "mean_kl": 2.550264358520508, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.3658536672592163 + }, + { + "candidate_nll": 2.7376186847686768, + "code_tokens": 67, + "cvar95_kl": 8.532747268676758, + "delta_nll": 2.372936964035034, + "max_kl": 9.214317321777344, + "mean_kl": 2.6396422386169434, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.46268656849861145 + }, + { + "candidate_nll": 4.836615085601807, + "code_tokens": 76, + "cvar95_kl": 9.260400772094727, + "delta_nll": 3.446355104446411, + "max_kl": 10.682518005371094, + "mean_kl": 3.3897628784179688, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 3.841155767440796, + "code_tokens": 137, + "cvar95_kl": 8.073051452636719, + "delta_nll": 3.205303192138672, + "max_kl": 10.951790809631348, + "mean_kl": 3.1524202823638916, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.33576643466949463 + } + ], + "policy": { + "default_bits": 4, + "name": "uniform_int4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2433024, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2433024, + "resident_compression_ratio": 7.757575757575758 + }, + "task_delta_nll_distribution": { + "maximum": 5.0876946449279785, + "median": 2.9581104516983032, + "minimum": 0.9846757650375366, + "q25": 2.486187994480133, + "q75": 3.3768213391304016 + }, + "task_macro": { + "candidate_nll": 3.963789122581482, + "delta_nll": 2.949742543697357, + "mean_kl": 3.0408298342227935, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.3330292352586985 + }, + "token_weighted": { + "candidate_nll": 4.004589080810547, + "cvar95_kl": 9.002206802368164, + "delta_nll": 3.0877513885498047, + "max_kl": 16.425533294677734, + "mean_kl": 3.1499691009521484, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.3211545944213867 + } + }, + "uniform_int8_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 0.00709843111038208, + "macro_mean_kl": 0.01141075879149139, + "macro_top1_agreement": 0.9741976313591003, + "maximum_code_tokens": 30, + "minimum_code_tokens": 11, + "task_count": 125 + }, + "Q2": { + "macro_delta_nll": 0.010783773183822632, + "macro_mean_kl": 0.013967912949621677, + "macro_top1_agreement": 0.9710763902664185, + "maximum_code_tokens": 48, + "minimum_code_tokens": 30, + "task_count": 125 + }, + "Q3": { + "macro_delta_nll": 0.017099417924880983, + "macro_mean_kl": 0.01838173795491457, + "macro_top1_agreement": 0.9626756963729859, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 125 + }, + "Q4": { + "macro_delta_nll": 0.03385586380958557, + "macro_mean_kl": 0.031905529409646985, + "macro_top1_agreement": 0.9505300631523133, + "maximum_code_tokens": 451, + "minimum_code_tokens": 76, + "task_count": 125 + } + }, + "per_task": [ + { + "candidate_nll": 0.7303756475448608, + "code_tokens": 98, + "cvar95_kl": 0.13508905470371246, + "delta_nll": 0.0035199522972106934, + "max_kl": 0.16827626526355743, + "mean_kl": 0.01848014071583748, + "reference_nll": 0.7268556952476501, + "task_id": 11, + "token_count": 98, + "top1_agreement": 0.9897959232330322 + }, + { + "candidate_nll": 1.170810341835022, + "code_tokens": 19, + "cvar95_kl": 0.03613688051700592, + "delta_nll": 0.011397242546081543, + "max_kl": 0.03613688051700592, + "mean_kl": 0.0064887539483606815, + "reference_nll": 1.1594130992889404, + "task_id": 12, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9649032950401306, + "code_tokens": 39, + "cvar95_kl": 0.11870085448026657, + "delta_nll": -0.05044597387313843, + "max_kl": 0.14969758689403534, + "mean_kl": 0.016321973875164986, + "reference_nll": 1.015349268913269, + "task_id": 13, + "token_count": 39, + "top1_agreement": 0.9743589758872986 + }, + { + "candidate_nll": 1.3995054960250854, + "code_tokens": 24, + "cvar95_kl": 0.08196324110031128, + "delta_nll": -0.046784162521362305, + "max_kl": 0.12636372447013855, + "mean_kl": 0.016114989295601845, + "reference_nll": 1.4462896585464478, + "task_id": 14, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 1.4392350912094116, + "code_tokens": 25, + "cvar95_kl": 0.06915412843227386, + "delta_nll": 0.031476736068725586, + "max_kl": 0.10494361817836761, + "mean_kl": 0.01274560485035181, + "reference_nll": 1.407758355140686, + "task_id": 15, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.909285306930542, + "code_tokens": 55, + "cvar95_kl": 0.047497984021902084, + "delta_nll": 0.013673722743988037, + "max_kl": 0.05659465864300728, + "mean_kl": 0.009337624534964561, + "reference_nll": 0.895611584186554, + "task_id": 16, + "token_count": 55, + "top1_agreement": 0.9818181991577148 + }, + { + "candidate_nll": 0.83961421251297, + "code_tokens": 16, + "cvar95_kl": 0.02004585973918438, + "delta_nll": 0.010173618793487549, + "max_kl": 0.02004585973918438, + "mean_kl": 0.004075487609952688, + "reference_nll": 0.8294405937194824, + "task_id": 17, + "token_count": 16, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2322083711624146, + "code_tokens": 199, + "cvar95_kl": 0.3167828917503357, + "delta_nll": 0.01031494140625, + "max_kl": 0.6550754308700562, + "mean_kl": 0.03782374784350395, + "reference_nll": 1.2218934297561646, + "task_id": 18, + "token_count": 199, + "top1_agreement": 0.9145728349685669 + }, + { + "candidate_nll": 1.5864008665084839, + "code_tokens": 28, + "cvar95_kl": 0.039230525493621826, + "delta_nll": 0.020322799682617188, + "max_kl": 0.04426715523004532, + "mean_kl": 0.01017265859991312, + "reference_nll": 1.5660780668258667, + "task_id": 19, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 0.9174427390098572, + "code_tokens": 87, + "cvar95_kl": 0.38930439949035645, + "delta_nll": 0.04446011781692505, + "max_kl": 1.3800705671310425, + "mean_kl": 0.04030923917889595, + "reference_nll": 0.8729826211929321, + "task_id": 20, + "token_count": 87, + "top1_agreement": 0.9425287246704102 + }, + { + "candidate_nll": 1.2220391035079956, + "code_tokens": 35, + "cvar95_kl": 0.0644833967089653, + "delta_nll": -0.010925054550170898, + "max_kl": 0.06917877495288849, + "mean_kl": 0.014502760022878647, + "reference_nll": 1.2329641580581665, + "task_id": 21, + "token_count": 35, + "top1_agreement": 0.9714285731315613 + }, + { + "candidate_nll": 0.655078113079071, + "code_tokens": 62, + "cvar95_kl": 0.1585526317358017, + "delta_nll": 0.01578652858734131, + "max_kl": 0.2549428939819336, + "mean_kl": 0.01725442335009575, + "reference_nll": 0.6392915844917297, + "task_id": 22, + "token_count": 62, + "top1_agreement": 0.9516128897666931 + }, + { + "candidate_nll": 1.1119842529296875, + "code_tokens": 58, + "cvar95_kl": 0.16379857063293457, + "delta_nll": 0.006926536560058594, + "max_kl": 0.37132343649864197, + "mean_kl": 0.017417574301362038, + "reference_nll": 1.105057716369629, + "task_id": 23, + "token_count": 58, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 1.1610227823257446, + "code_tokens": 79, + "cvar95_kl": 0.1345561444759369, + "delta_nll": 0.0019745826721191406, + "max_kl": 0.16966059803962708, + "mean_kl": 0.024569327011704445, + "reference_nll": 1.1590481996536255, + "task_id": 24, + "token_count": 79, + "top1_agreement": 0.9367088675498962 + }, + { + "candidate_nll": 1.118152379989624, + "code_tokens": 58, + "cvar95_kl": 0.062340885400772095, + "delta_nll": 0.025032520294189453, + "max_kl": 0.06849651038646698, + "mean_kl": 0.012996862642467022, + "reference_nll": 1.0931198596954346, + "task_id": 25, + "token_count": 58, + "top1_agreement": 0.9137930870056152 + }, + { + "candidate_nll": 1.167366623878479, + "code_tokens": 48, + "cvar95_kl": 0.08859491348266602, + "delta_nll": 0.005168795585632324, + "max_kl": 0.12154804915189743, + "mean_kl": 0.014337733387947083, + "reference_nll": 1.1621978282928467, + "task_id": 26, + "token_count": 48, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 1.1495749950408936, + "code_tokens": 37, + "cvar95_kl": 0.06092211604118347, + "delta_nll": 0.02414250373840332, + "max_kl": 0.0710364505648613, + "mean_kl": 0.010390897281467915, + "reference_nll": 1.1254324913024902, + "task_id": 27, + "token_count": 37, + "top1_agreement": 0.9189189076423645 + }, + { + "candidate_nll": 0.7362630367279053, + "code_tokens": 61, + "cvar95_kl": 0.11543451249599457, + "delta_nll": 0.028855860233306885, + "max_kl": 0.29879048466682434, + "mean_kl": 0.013628797605633736, + "reference_nll": 0.7074071764945984, + "task_id": 28, + "token_count": 61, + "top1_agreement": 0.9508196711540222 + }, + { + "candidate_nll": 0.7970921993255615, + "code_tokens": 77, + "cvar95_kl": 0.4115562438964844, + "delta_nll": 0.027372658252716064, + "max_kl": 0.6827960014343262, + "mean_kl": 0.03619532659649849, + "reference_nll": 0.7697195410728455, + "task_id": 29, + "token_count": 77, + "top1_agreement": 0.9870129823684692 + }, + { + "candidate_nll": 1.0105679035186768, + "code_tokens": 98, + "cvar95_kl": 0.12295398861169815, + "delta_nll": -0.011135220527648926, + "max_kl": 0.13934780657291412, + "mean_kl": 0.02256907895207405, + "reference_nll": 1.0217031240463257, + "task_id": 30, + "token_count": 98, + "top1_agreement": 0.9897959232330322 + }, + { + "candidate_nll": 0.6519662737846375, + "code_tokens": 162, + "cvar95_kl": 0.5127643346786499, + "delta_nll": 0.0668150782585144, + "max_kl": 1.4732556343078613, + "mean_kl": 0.054624494165182114, + "reference_nll": 0.585151195526123, + "task_id": 31, + "token_count": 162, + "top1_agreement": 0.9259259104728699 + }, + { + "candidate_nll": 0.7825482487678528, + "code_tokens": 103, + "cvar95_kl": 0.27220770716667175, + "delta_nll": 0.038082242012023926, + "max_kl": 0.6126030087471008, + "mean_kl": 0.02460607886314392, + "reference_nll": 0.7444660067558289, + "task_id": 32, + "token_count": 103, + "top1_agreement": 0.9514563083648682 + }, + { + "candidate_nll": 1.391046166419983, + "code_tokens": 73, + "cvar95_kl": 0.20010635256767273, + "delta_nll": 0.0720757246017456, + "max_kl": 0.4022406339645386, + "mean_kl": 0.027633918449282646, + "reference_nll": 1.3189704418182373, + "task_id": 33, + "token_count": 73, + "top1_agreement": 0.931506872177124 + }, + { + "candidate_nll": 1.0100759267807007, + "code_tokens": 119, + "cvar95_kl": 0.38897690176963806, + "delta_nll": 0.027102291584014893, + "max_kl": 0.6158857941627502, + "mean_kl": 0.03867809474468231, + "reference_nll": 0.9829736351966858, + "task_id": 34, + "token_count": 119, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 1.1595003604888916, + "code_tokens": 17, + "cvar95_kl": 0.05711754038929939, + "delta_nll": -0.03002297878265381, + "max_kl": 0.05711754038929939, + "mean_kl": 0.011637450195848942, + "reference_nll": 1.1895233392715454, + "task_id": 35, + "token_count": 17, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.4591100215911865, + "code_tokens": 56, + "cvar95_kl": 0.1135697141289711, + "delta_nll": 0.033408284187316895, + "max_kl": 0.12381391227245331, + "mean_kl": 0.0243150033056736, + "reference_nll": 1.4257017374038696, + "task_id": 36, + "token_count": 56, + "top1_agreement": 0.9107142686843872 + }, + { + "candidate_nll": 0.4801763594150543, + "code_tokens": 57, + "cvar95_kl": 0.07043429464101791, + "delta_nll": 0.01281583309173584, + "max_kl": 0.09291214495897293, + "mean_kl": 0.009914911352097988, + "reference_nll": 0.4673605263233185, + "task_id": 37, + "token_count": 57, + "top1_agreement": 0.9824561476707458 + }, + { + "candidate_nll": 0.6615414023399353, + "code_tokens": 63, + "cvar95_kl": 0.13791689276695251, + "delta_nll": 0.02879619598388672, + "max_kl": 0.1888631284236908, + "mean_kl": 0.017740346491336823, + "reference_nll": 0.6327452063560486, + "task_id": 38, + "token_count": 63, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8481513857841492, + "code_tokens": 200, + "cvar95_kl": 0.45427340269088745, + "delta_nll": 0.010215818881988525, + "max_kl": 1.1194424629211426, + "mean_kl": 0.04403680935502052, + "reference_nll": 0.8379355669021606, + "task_id": 39, + "token_count": 200, + "top1_agreement": 0.9350000023841858 + }, + { + "candidate_nll": 1.0631366968154907, + "code_tokens": 30, + "cvar95_kl": 0.06113705039024353, + "delta_nll": -0.021065950393676758, + "max_kl": 0.07287552952766418, + "mean_kl": 0.013477585278451443, + "reference_nll": 1.0842026472091675, + "task_id": 40, + "token_count": 30, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5913982391357422, + "code_tokens": 28, + "cvar95_kl": 0.01218867301940918, + "delta_nll": -0.024952828884124756, + "max_kl": 0.013300392776727676, + "mean_kl": 0.0026359292678534985, + "reference_nll": 0.6163510680198669, + "task_id": 41, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 0.809496283531189, + "code_tokens": 26, + "cvar95_kl": 0.08271096646785736, + "delta_nll": 0.03853964805603027, + "max_kl": 0.0948687493801117, + "mean_kl": 0.015500715002417564, + "reference_nll": 0.7709566354751587, + "task_id": 42, + "token_count": 26, + "top1_agreement": 0.8846153616905212 + }, + { + "candidate_nll": 0.8425158262252808, + "code_tokens": 54, + "cvar95_kl": 0.08317889273166656, + "delta_nll": 0.009574472904205322, + "max_kl": 0.09347321093082428, + "mean_kl": 0.014892959967255592, + "reference_nll": 0.8329413533210754, + "task_id": 43, + "token_count": 54, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 0.966856837272644, + "code_tokens": 47, + "cvar95_kl": 0.07018662244081497, + "delta_nll": 0.010120689868927002, + "max_kl": 0.08610522001981735, + "mean_kl": 0.010905792936682701, + "reference_nll": 0.956736147403717, + "task_id": 44, + "token_count": 47, + "top1_agreement": 0.936170220375061 + }, + { + "candidate_nll": 0.795110285282135, + "code_tokens": 93, + "cvar95_kl": 0.128584086894989, + "delta_nll": 0.06058073043823242, + "max_kl": 0.1541401445865631, + "mean_kl": 0.01882798597216606, + "reference_nll": 0.7345295548439026, + "task_id": 45, + "token_count": 93, + "top1_agreement": 0.9462365508079529 + }, + { + "candidate_nll": 0.8842098712921143, + "code_tokens": 30, + "cvar95_kl": 0.04821290075778961, + "delta_nll": 0.024835288524627686, + "max_kl": 0.05108967423439026, + "mean_kl": 0.01141685713082552, + "reference_nll": 0.8593745827674866, + "task_id": 46, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 1.0920023918151855, + "code_tokens": 89, + "cvar95_kl": 0.12980090081691742, + "delta_nll": 0.023499131202697754, + "max_kl": 0.2836276888847351, + "mean_kl": 0.01959419995546341, + "reference_nll": 1.0685032606124878, + "task_id": 47, + "token_count": 89, + "top1_agreement": 0.966292142868042 + }, + { + "candidate_nll": 0.9896074533462524, + "code_tokens": 70, + "cvar95_kl": 0.15014959871768951, + "delta_nll": 0.025358140468597412, + "max_kl": 0.20418210327625275, + "mean_kl": 0.021574119105935097, + "reference_nll": 0.964249312877655, + "task_id": 48, + "token_count": 70, + "top1_agreement": 0.9571428298950195 + }, + { + "candidate_nll": 1.6564089059829712, + "code_tokens": 26, + "cvar95_kl": 0.03424878790974617, + "delta_nll": 0.043181777000427246, + "max_kl": 0.042801130563020706, + "mean_kl": 0.010413101874291897, + "reference_nll": 1.613227128982544, + "task_id": 49, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9665414690971375, + "code_tokens": 48, + "cvar95_kl": 0.110567606985569, + "delta_nll": 0.008834600448608398, + "max_kl": 0.16235306859016418, + "mean_kl": 0.016660815104842186, + "reference_nll": 0.957706868648529, + "task_id": 50, + "token_count": 48, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.8677156567573547, + "code_tokens": 29, + "cvar95_kl": 0.01895114965736866, + "delta_nll": -0.0014323592185974121, + "max_kl": 0.021374806761741638, + "mean_kl": 0.004783522803336382, + "reference_nll": 0.8691480159759521, + "task_id": 51, + "token_count": 29, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0775179862976074, + "code_tokens": 17, + "cvar95_kl": 0.010332036763429642, + "delta_nll": -0.035645127296447754, + "max_kl": 0.010332036763429642, + "mean_kl": 0.0028900359757244587, + "reference_nll": 1.1131631135940552, + "task_id": 52, + "token_count": 17, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9315062761306763, + "code_tokens": 37, + "cvar95_kl": 0.0518275611102581, + "delta_nll": 0.011290132999420166, + "max_kl": 0.06260928511619568, + "mean_kl": 0.007329113781452179, + "reference_nll": 0.9202161431312561, + "task_id": 53, + "token_count": 37, + "top1_agreement": 0.9729729890823364 + }, + { + "candidate_nll": 0.5574541091918945, + "code_tokens": 121, + "cvar95_kl": 0.42427191138267517, + "delta_nll": 0.046052396297454834, + "max_kl": 0.9412640333175659, + "mean_kl": 0.03851114585995674, + "reference_nll": 0.5114017128944397, + "task_id": 54, + "token_count": 121, + "top1_agreement": 0.93388432264328 + }, + { + "candidate_nll": 1.4256136417388916, + "code_tokens": 30, + "cvar95_kl": 0.07870058715343475, + "delta_nll": 0.02676987648010254, + "max_kl": 0.09150835126638412, + "mean_kl": 0.016224829480051994, + "reference_nll": 1.398843765258789, + "task_id": 55, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 0.8991823792457581, + "code_tokens": 74, + "cvar95_kl": 0.11031309515237808, + "delta_nll": 0.0007106661796569824, + "max_kl": 0.13659797608852386, + "mean_kl": 0.0145348459482193, + "reference_nll": 0.8984717130661011, + "task_id": 56, + "token_count": 74, + "top1_agreement": 0.9729729890823364 + }, + { + "candidate_nll": 0.8638786673545837, + "code_tokens": 53, + "cvar95_kl": 0.09372228384017944, + "delta_nll": 0.029454827308654785, + "max_kl": 0.11512858420610428, + "mean_kl": 0.016471246257424355, + "reference_nll": 0.834423840045929, + "task_id": 57, + "token_count": 53, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.8545043468475342, + "code_tokens": 21, + "cvar95_kl": 0.052153944969177246, + "delta_nll": -0.004891395568847656, + "max_kl": 0.081003338098526, + "mean_kl": 0.0080454396083951, + "reference_nll": 1.8593957424163818, + "task_id": 58, + "token_count": 21, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3559309244155884, + "code_tokens": 20, + "cvar95_kl": 0.07684168219566345, + "delta_nll": -0.006482720375061035, + "max_kl": 0.07684168219566345, + "mean_kl": 0.01580420881509781, + "reference_nll": 1.3624136447906494, + "task_id": 59, + "token_count": 20, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 0.8735544085502625, + "code_tokens": 122, + "cvar95_kl": 0.28098437190055847, + "delta_nll": 0.05455601215362549, + "max_kl": 0.592315673828125, + "mean_kl": 0.03402591496706009, + "reference_nll": 0.818998396396637, + "task_id": 60, + "token_count": 122, + "top1_agreement": 0.9426229596138 + }, + { + "candidate_nll": 0.8636236190795898, + "code_tokens": 94, + "cvar95_kl": 0.1716153919696808, + "delta_nll": 0.02348381280899048, + "max_kl": 0.2577202320098877, + "mean_kl": 0.0267892237752676, + "reference_nll": 0.8401398062705994, + "task_id": 61, + "token_count": 94, + "top1_agreement": 0.978723406791687 + }, + { + "candidate_nll": 1.085481882095337, + "code_tokens": 12, + "cvar95_kl": 0.011499770916998386, + "delta_nll": 0.020180344581604004, + "max_kl": 0.011499770916998386, + "mean_kl": 0.0029877498745918274, + "reference_nll": 1.065301537513733, + "task_id": 62, + "token_count": 12, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.5124791860580444, + "code_tokens": 38, + "cvar95_kl": 0.04518405348062515, + "delta_nll": 0.006078958511352539, + "max_kl": 0.0531487874686718, + "mean_kl": 0.009268837980926037, + "reference_nll": 1.506400227546692, + "task_id": 63, + "token_count": 38, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8930755257606506, + "code_tokens": 58, + "cvar95_kl": 0.05258774384856224, + "delta_nll": -0.011711597442626953, + "max_kl": 0.05342841148376465, + "mean_kl": 0.007918545976281166, + "reference_nll": 0.9047871232032776, + "task_id": 64, + "token_count": 58, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6436004042625427, + "code_tokens": 54, + "cvar95_kl": 0.13577057421207428, + "delta_nll": 0.047336578369140625, + "max_kl": 0.2127564549446106, + "mean_kl": 0.015880897641181946, + "reference_nll": 0.5962638258934021, + "task_id": 65, + "token_count": 54, + "top1_agreement": 0.9814814925193787 + }, + { + "candidate_nll": 0.9273276329040527, + "code_tokens": 40, + "cvar95_kl": 0.01907806098461151, + "delta_nll": -0.0042122602462768555, + "max_kl": 0.020595187321305275, + "mean_kl": 0.003718684194609523, + "reference_nll": 0.9315398931503296, + "task_id": 66, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5206365585327148, + "code_tokens": 115, + "cvar95_kl": 0.29594382643699646, + "delta_nll": 0.008090615272521973, + "max_kl": 0.4111235439777374, + "mean_kl": 0.026325233280658722, + "reference_nll": 0.5125459432601929, + "task_id": 67, + "token_count": 115, + "top1_agreement": 0.9304347634315491 + }, + { + "candidate_nll": 0.5485146045684814, + "code_tokens": 60, + "cvar95_kl": 0.15575706958770752, + "delta_nll": 0.023900926113128662, + "max_kl": 0.2004082053899765, + "mean_kl": 0.017739150673151016, + "reference_nll": 0.5246136784553528, + "task_id": 68, + "token_count": 60, + "top1_agreement": 0.9833333492279053 + }, + { + "candidate_nll": 0.7961963415145874, + "code_tokens": 124, + "cvar95_kl": 0.3429846465587616, + "delta_nll": 0.026209235191345215, + "max_kl": 0.8604524731636047, + "mean_kl": 0.03922838345170021, + "reference_nll": 0.7699871063232422, + "task_id": 69, + "token_count": 124, + "top1_agreement": 0.9032257795333862 + }, + { + "candidate_nll": 0.7944657206535339, + "code_tokens": 91, + "cvar95_kl": 0.10858277976512909, + "delta_nll": 0.023717045783996582, + "max_kl": 0.2436714470386505, + "mean_kl": 0.013639695942401886, + "reference_nll": 0.7707486748695374, + "task_id": 70, + "token_count": 91, + "top1_agreement": 0.9560439586639404 + }, + { + "candidate_nll": 0.8356055021286011, + "code_tokens": 125, + "cvar95_kl": 0.17785954475402832, + "delta_nll": 0.01568812131881714, + "max_kl": 0.3957410454750061, + "mean_kl": 0.023328373208642006, + "reference_nll": 0.8199173808097839, + "task_id": 71, + "token_count": 125, + "top1_agreement": 0.9679999947547913 + }, + { + "candidate_nll": 1.2049670219421387, + "code_tokens": 26, + "cvar95_kl": 0.0847049206495285, + "delta_nll": 0.06909048557281494, + "max_kl": 0.09378682076931, + "mean_kl": 0.015413755550980568, + "reference_nll": 1.1358765363693237, + "task_id": 72, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.295048236846924, + "code_tokens": 24, + "cvar95_kl": 0.150682732462883, + "delta_nll": 0.05462312698364258, + "max_kl": 0.15096478164196014, + "mean_kl": 0.03297318145632744, + "reference_nll": 2.2404251098632812, + "task_id": 73, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9200271368026733, + "code_tokens": 180, + "cvar95_kl": 0.38579902052879333, + "delta_nll": 0.0658453106880188, + "max_kl": 0.5137471556663513, + "mean_kl": 0.04577277973294258, + "reference_nll": 0.8541818261146545, + "task_id": 74, + "token_count": 180, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.3838077783584595, + "code_tokens": 40, + "cvar95_kl": 0.0991126075387001, + "delta_nll": 0.008999228477478027, + "max_kl": 0.11464756727218628, + "mean_kl": 0.012050293385982513, + "reference_nll": 1.3748085498809814, + "task_id": 75, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0058878660202026, + "code_tokens": 72, + "cvar95_kl": 0.18116626143455505, + "delta_nll": 0.030891001224517822, + "max_kl": 0.31926092505455017, + "mean_kl": 0.02611645497381687, + "reference_nll": 0.9749968647956848, + "task_id": 76, + "token_count": 72, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 1.5457720756530762, + "code_tokens": 20, + "cvar95_kl": 0.09525889903306961, + "delta_nll": -0.0317990779876709, + "max_kl": 0.09525889903306961, + "mean_kl": 0.017236648127436638, + "reference_nll": 1.577571153640747, + "task_id": 77, + "token_count": 20, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0316953659057617, + "code_tokens": 76, + "cvar95_kl": 0.21048316359519958, + "delta_nll": 0.04105103015899658, + "max_kl": 0.33110904693603516, + "mean_kl": 0.02857571467757225, + "reference_nll": 0.9906443357467651, + "task_id": 78, + "token_count": 76, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 1.336522102355957, + "code_tokens": 42, + "cvar95_kl": 0.05231620743870735, + "delta_nll": -0.03521931171417236, + "max_kl": 0.05734030902385712, + "mean_kl": 0.009630455635488033, + "reference_nll": 1.3717414140701294, + "task_id": 79, + "token_count": 42, + "top1_agreement": 0.976190447807312 + }, + { + "candidate_nll": 0.6114948987960815, + "code_tokens": 28, + "cvar95_kl": 0.09413042664527893, + "delta_nll": 0.028271734714508057, + "max_kl": 0.10932847857475281, + "mean_kl": 0.01013279054313898, + "reference_nll": 0.5832231640815735, + "task_id": 80, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0583676099777222, + "code_tokens": 58, + "cvar95_kl": 0.12472521513700485, + "delta_nll": 0.024890780448913574, + "max_kl": 0.14327889680862427, + "mean_kl": 0.021377304568886757, + "reference_nll": 1.0334768295288086, + "task_id": 81, + "token_count": 58, + "top1_agreement": 0.982758641242981 + }, + { + "candidate_nll": 0.8546289801597595, + "code_tokens": 25, + "cvar95_kl": 0.019587436690926552, + "delta_nll": -0.014020442962646484, + "max_kl": 0.019936606287956238, + "mean_kl": 0.005718192085623741, + "reference_nll": 0.868649423122406, + "task_id": 82, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0637911558151245, + "code_tokens": 88, + "cvar95_kl": 0.6216155290603638, + "delta_nll": 0.09878331422805786, + "max_kl": 1.75636887550354, + "mean_kl": 0.055053018033504486, + "reference_nll": 0.9650078415870667, + "task_id": 83, + "token_count": 88, + "top1_agreement": 0.9204545617103577 + }, + { + "candidate_nll": 1.1221495866775513, + "code_tokens": 42, + "cvar95_kl": 0.13517127931118011, + "delta_nll": -0.005367636680603027, + "max_kl": 0.159673273563385, + "mean_kl": 0.02214982733130455, + "reference_nll": 1.1275172233581543, + "task_id": 84, + "token_count": 42, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 0.835040807723999, + "code_tokens": 24, + "cvar95_kl": 0.037389349192380905, + "delta_nll": 0.015490055084228516, + "max_kl": 0.041228342801332474, + "mean_kl": 0.007212849799543619, + "reference_nll": 0.8195507526397705, + "task_id": 85, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.699725866317749, + "code_tokens": 24, + "cvar95_kl": 0.035677649080753326, + "delta_nll": 0.014854490756988525, + "max_kl": 0.05376433953642845, + "mean_kl": 0.0077397748827934265, + "reference_nll": 0.6848713755607605, + "task_id": 86, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 1.0933729410171509, + "code_tokens": 43, + "cvar95_kl": 0.03877802565693855, + "delta_nll": 0.026193857192993164, + "max_kl": 0.051284290850162506, + "mean_kl": 0.0072237057611346245, + "reference_nll": 1.0671790838241577, + "task_id": 87, + "token_count": 43, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9086310267448425, + "code_tokens": 24, + "cvar95_kl": 0.04189204424619675, + "delta_nll": 0.022411346435546875, + "max_kl": 0.060692526400089264, + "mean_kl": 0.008439191617071629, + "reference_nll": 0.8862196803092957, + "task_id": 88, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3201351165771484, + "code_tokens": 14, + "cvar95_kl": 0.03707623481750488, + "delta_nll": 0.007509469985961914, + "max_kl": 0.03707623481750488, + "mean_kl": 0.007463677320629358, + "reference_nll": 1.3126256465911865, + "task_id": 89, + "token_count": 14, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 0.5136525630950928, + "code_tokens": 41, + "cvar95_kl": 0.05094480887055397, + "delta_nll": 0.022609293460845947, + "max_kl": 0.08581598103046417, + "mean_kl": 0.009656337089836597, + "reference_nll": 0.4910432696342468, + "task_id": 90, + "token_count": 41, + "top1_agreement": 0.9512194991111755 + }, + { + "candidate_nll": 0.9900891184806824, + "code_tokens": 32, + "cvar95_kl": 0.031703799962997437, + "delta_nll": 0.0017492175102233887, + "max_kl": 0.04238611459732056, + "mean_kl": 0.006714640650898218, + "reference_nll": 0.988339900970459, + "task_id": 91, + "token_count": 32, + "top1_agreement": 0.96875 + }, + { + "candidate_nll": 0.7858836054801941, + "code_tokens": 53, + "cvar95_kl": 0.04299309477210045, + "delta_nll": 0.024188697338104248, + "max_kl": 0.05402224138379097, + "mean_kl": 0.00845135934650898, + "reference_nll": 0.7616949081420898, + "task_id": 92, + "token_count": 53, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.41477397084236145, + "code_tokens": 51, + "cvar95_kl": 0.20540131628513336, + "delta_nll": 0.03648287057876587, + "max_kl": 0.4584619402885437, + "mean_kl": 0.019988451153039932, + "reference_nll": 0.3782911002635956, + "task_id": 93, + "token_count": 51, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 1.7097346782684326, + "code_tokens": 36, + "cvar95_kl": 0.06695758551359177, + "delta_nll": 0.017579317092895508, + "max_kl": 0.07028105109930038, + "mean_kl": 0.013926519080996513, + "reference_nll": 1.692155361175537, + "task_id": 94, + "token_count": 36, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 1.5282455682754517, + "code_tokens": 24, + "cvar95_kl": 0.04584892466664314, + "delta_nll": -0.032506704330444336, + "max_kl": 0.06012102961540222, + "mean_kl": 0.008756505325436592, + "reference_nll": 1.560752272605896, + "task_id": 95, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.302522897720337, + "code_tokens": 39, + "cvar95_kl": 0.06570790708065033, + "delta_nll": -0.011811494827270508, + "max_kl": 0.06983443349599838, + "mean_kl": 0.014587810263037682, + "reference_nll": 1.3143343925476074, + "task_id": 96, + "token_count": 39, + "top1_agreement": 0.9487179517745972 + }, + { + "candidate_nll": 0.7162724733352661, + "code_tokens": 84, + "cvar95_kl": 0.0824974924325943, + "delta_nll": 0.01785987615585327, + "max_kl": 0.1599540114402771, + "mean_kl": 0.012312708422541618, + "reference_nll": 0.6984125971794128, + "task_id": 97, + "token_count": 84, + "top1_agreement": 0.988095223903656 + }, + { + "candidate_nll": 0.9448277950286865, + "code_tokens": 32, + "cvar95_kl": 0.02884177677333355, + "delta_nll": 0.02205955982208252, + "max_kl": 0.038144566118717194, + "mean_kl": 0.006878299172967672, + "reference_nll": 0.922768235206604, + "task_id": 98, + "token_count": 32, + "top1_agreement": 0.9375 + }, + { + "candidate_nll": 1.3334046602249146, + "code_tokens": 19, + "cvar95_kl": 0.04142870754003525, + "delta_nll": 0.023241043090820312, + "max_kl": 0.04142870754003525, + "mean_kl": 0.006109209731221199, + "reference_nll": 1.3101636171340942, + "task_id": 99, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8146919012069702, + "code_tokens": 50, + "cvar95_kl": 0.31939321756362915, + "delta_nll": -0.0032848715782165527, + "max_kl": 0.6818132400512695, + "mean_kl": 0.03451798856258392, + "reference_nll": 0.8179767727851868, + "task_id": 100, + "token_count": 50, + "top1_agreement": 0.8999999761581421 + }, + { + "candidate_nll": 0.5393460988998413, + "code_tokens": 71, + "cvar95_kl": 0.363267719745636, + "delta_nll": 0.039532482624053955, + "max_kl": 0.4263126850128174, + "mean_kl": 0.04541124403476715, + "reference_nll": 0.49981361627578735, + "task_id": 101, + "token_count": 71, + "top1_agreement": 0.9718309640884399 + }, + { + "candidate_nll": 1.3197507858276367, + "code_tokens": 28, + "cvar95_kl": 0.03516513109207153, + "delta_nll": -0.06821763515472412, + "max_kl": 0.04043719172477722, + "mean_kl": 0.008249783888459206, + "reference_nll": 1.3879684209823608, + "task_id": 102, + "token_count": 28, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 0.8435997366905212, + "code_tokens": 76, + "cvar95_kl": 0.062001295387744904, + "delta_nll": -0.0256616473197937, + "max_kl": 0.07021024823188782, + "mean_kl": 0.01367997657507658, + "reference_nll": 0.8692613840103149, + "task_id": 103, + "token_count": 76, + "top1_agreement": 0.9868420958518982 + }, + { + "candidate_nll": 0.9526077508926392, + "code_tokens": 34, + "cvar95_kl": 0.05240423232316971, + "delta_nll": -0.009977459907531738, + "max_kl": 0.06947250664234161, + "mean_kl": 0.012638763524591923, + "reference_nll": 0.9625852108001709, + "task_id": 104, + "token_count": 34, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 2.1012589931488037, + "code_tokens": 11, + "cvar95_kl": 0.02709655649960041, + "delta_nll": -0.046132802963256836, + "max_kl": 0.02709655649960041, + "mean_kl": 0.006265569943934679, + "reference_nll": 2.1473917961120605, + "task_id": 105, + "token_count": 11, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3954291343688965, + "code_tokens": 31, + "cvar95_kl": 0.06168582662940025, + "delta_nll": -0.020838379859924316, + "max_kl": 0.09877268970012665, + "mean_kl": 0.009982689283788204, + "reference_nll": 1.4162675142288208, + "task_id": 106, + "token_count": 31, + "top1_agreement": 0.9354838728904724 + }, + { + "candidate_nll": 0.8629112243652344, + "code_tokens": 118, + "cvar95_kl": 0.35224828124046326, + "delta_nll": 0.016134977340698242, + "max_kl": 0.7643479704856873, + "mean_kl": 0.03997105732560158, + "reference_nll": 0.8467762470245361, + "task_id": 107, + "token_count": 118, + "top1_agreement": 0.9152542352676392 + }, + { + "candidate_nll": 0.7671003341674805, + "code_tokens": 60, + "cvar95_kl": 0.025304250419139862, + "delta_nll": -0.00859534740447998, + "max_kl": 0.028871694579720497, + "mean_kl": 0.005343298893421888, + "reference_nll": 0.7756956815719604, + "task_id": 108, + "token_count": 60, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9099680185317993, + "code_tokens": 47, + "cvar95_kl": 0.05628103017807007, + "delta_nll": 0.01451808214187622, + "max_kl": 0.06309367716312408, + "mean_kl": 0.013431820087134838, + "reference_nll": 0.8954499363899231, + "task_id": 109, + "token_count": 47, + "top1_agreement": 0.978723406791687 + }, + { + "candidate_nll": 1.168602466583252, + "code_tokens": 84, + "cvar95_kl": 0.2814013361930847, + "delta_nll": 0.01658809185028076, + "max_kl": 0.4367305040359497, + "mean_kl": 0.0393073670566082, + "reference_nll": 1.1520143747329712, + "task_id": 110, + "token_count": 84, + "top1_agreement": 0.8690476417541504 + }, + { + "candidate_nll": 1.0545023679733276, + "code_tokens": 27, + "cvar95_kl": 0.08516751229763031, + "delta_nll": 0.06047630310058594, + "max_kl": 0.11000338196754456, + "mean_kl": 0.015271913260221481, + "reference_nll": 0.9940260648727417, + "task_id": 111, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.6448314189910889, + "code_tokens": 19, + "cvar95_kl": 0.0971062034368515, + "delta_nll": 0.03292429447174072, + "max_kl": 0.0971062034368515, + "mean_kl": 0.012107712216675282, + "reference_nll": 1.6119071245193481, + "task_id": 112, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7730337977409363, + "code_tokens": 114, + "cvar95_kl": 0.1300615817308426, + "delta_nll": 2.1278858184814453e-05, + "max_kl": 0.2782489061355591, + "mean_kl": 0.019434422254562378, + "reference_nll": 0.7730125188827515, + "task_id": 113, + "token_count": 114, + "top1_agreement": 0.9912280440330505 + }, + { + "candidate_nll": 1.7469165325164795, + "code_tokens": 40, + "cvar95_kl": 0.20495757460594177, + "delta_nll": 0.01902329921722412, + "max_kl": 0.22896608710289001, + "mean_kl": 0.022183051332831383, + "reference_nll": 1.7278932332992554, + "task_id": 114, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.4426264762878418, + "code_tokens": 25, + "cvar95_kl": 0.0412004292011261, + "delta_nll": -0.043051719665527344, + "max_kl": 0.05105271935462952, + "mean_kl": 0.008265433833003044, + "reference_nll": 1.4856781959533691, + "task_id": 115, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.7456321120262146, + "code_tokens": 23, + "cvar95_kl": 0.06569425761699677, + "delta_nll": -0.025056958198547363, + "max_kl": 0.10613600164651871, + "mean_kl": 0.010636899620294571, + "reference_nll": 0.770689070224762, + "task_id": 116, + "token_count": 23, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0245609283447266, + "code_tokens": 77, + "cvar95_kl": 0.13851217925548553, + "delta_nll": 0.008211374282836914, + "max_kl": 0.14281794428825378, + "mean_kl": 0.02360093779861927, + "reference_nll": 1.0163495540618896, + "task_id": 117, + "token_count": 77, + "top1_agreement": 0.9610389471054077 + }, + { + "candidate_nll": 1.1005898714065552, + "code_tokens": 19, + "cvar95_kl": 0.028458595275878906, + "delta_nll": -0.00707554817199707, + "max_kl": 0.028458595275878906, + "mean_kl": 0.007480348460376263, + "reference_nll": 1.1076654195785522, + "task_id": 118, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.976451575756073, + "code_tokens": 37, + "cvar95_kl": 0.05177726596593857, + "delta_nll": -0.006792187690734863, + "max_kl": 0.06511953473091125, + "mean_kl": 0.009155580773949623, + "reference_nll": 0.9832437634468079, + "task_id": 119, + "token_count": 37, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3659489154815674, + "code_tokens": 33, + "cvar95_kl": 0.10391820222139359, + "delta_nll": -0.017215967178344727, + "max_kl": 0.11866727471351624, + "mean_kl": 0.0159289687871933, + "reference_nll": 1.383164882659912, + "task_id": 120, + "token_count": 33, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7046144008636475, + "code_tokens": 95, + "cvar95_kl": 0.24419239163398743, + "delta_nll": 0.044772982597351074, + "max_kl": 0.40727484226226807, + "mean_kl": 0.0321914367377758, + "reference_nll": 0.6598414182662964, + "task_id": 121, + "token_count": 95, + "top1_agreement": 0.9368420839309692 + }, + { + "candidate_nll": 1.0926512479782104, + "code_tokens": 125, + "cvar95_kl": 0.3462030291557312, + "delta_nll": 0.07949090003967285, + "max_kl": 0.7800798416137695, + "mean_kl": 0.0461714081466198, + "reference_nll": 1.0131603479385376, + "task_id": 122, + "token_count": 125, + "top1_agreement": 0.9359999895095825 + }, + { + "candidate_nll": 0.7579429149627686, + "code_tokens": 168, + "cvar95_kl": 0.35012564063072205, + "delta_nll": 0.044894516468048096, + "max_kl": 0.8321583867073059, + "mean_kl": 0.04649940878152847, + "reference_nll": 0.7130483984947205, + "task_id": 123, + "token_count": 168, + "top1_agreement": 0.9404761791229248 + }, + { + "candidate_nll": 1.358396053314209, + "code_tokens": 31, + "cvar95_kl": 0.0432329960167408, + "delta_nll": 0.042993903160095215, + "max_kl": 0.052951015532016754, + "mean_kl": 0.009078028611838818, + "reference_nll": 1.3154021501541138, + "task_id": 124, + "token_count": 31, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.810555636882782, + "code_tokens": 83, + "cvar95_kl": 0.25892823934555054, + "delta_nll": 0.04884892702102661, + "max_kl": 0.39830201864242554, + "mean_kl": 0.031812626868486404, + "reference_nll": 0.7617067098617554, + "task_id": 125, + "token_count": 83, + "top1_agreement": 0.9397590160369873 + }, + { + "candidate_nll": 0.7043254971504211, + "code_tokens": 50, + "cvar95_kl": 0.11854153871536255, + "delta_nll": 0.00438302755355835, + "max_kl": 0.25568893551826477, + "mean_kl": 0.0126386359333992, + "reference_nll": 0.6999424695968628, + "task_id": 126, + "token_count": 50, + "top1_agreement": 0.9399999976158142 + }, + { + "candidate_nll": 0.36634448170661926, + "code_tokens": 69, + "cvar95_kl": 0.06565694510936737, + "delta_nll": -0.000837475061416626, + "max_kl": 0.07301229983568192, + "mean_kl": 0.012558399699628353, + "reference_nll": 0.3671819567680359, + "task_id": 127, + "token_count": 69, + "top1_agreement": 0.95652174949646 + }, + { + "candidate_nll": 1.085649013519287, + "code_tokens": 50, + "cvar95_kl": 0.09815805405378342, + "delta_nll": 0.0007528066635131836, + "max_kl": 0.21098075807094574, + "mean_kl": 0.01176211517304182, + "reference_nll": 1.084896206855774, + "task_id": 128, + "token_count": 50, + "top1_agreement": 0.9399999976158142 + }, + { + "candidate_nll": 1.0279442071914673, + "code_tokens": 161, + "cvar95_kl": 0.36731603741645813, + "delta_nll": 0.02416253089904785, + "max_kl": 0.5428264141082764, + "mean_kl": 0.0494859404861927, + "reference_nll": 1.0037816762924194, + "task_id": 129, + "token_count": 161, + "top1_agreement": 0.9627329111099243 + }, + { + "candidate_nll": 0.6617211103439331, + "code_tokens": 53, + "cvar95_kl": 0.08596113324165344, + "delta_nll": 0.005944967269897461, + "max_kl": 0.11487188190221786, + "mean_kl": 0.012781397439539433, + "reference_nll": 0.6557761430740356, + "task_id": 130, + "token_count": 53, + "top1_agreement": 0.9811320900917053 + }, + { + "candidate_nll": 0.4657652974128723, + "code_tokens": 97, + "cvar95_kl": 0.5213609933853149, + "delta_nll": 0.020237237215042114, + "max_kl": 1.1843793392181396, + "mean_kl": 0.044014833867549896, + "reference_nll": 0.4455280601978302, + "task_id": 131, + "token_count": 97, + "top1_agreement": 0.9587628841400146 + }, + { + "candidate_nll": 1.3431541919708252, + "code_tokens": 22, + "cvar95_kl": 0.09664665162563324, + "delta_nll": 0.02934122085571289, + "max_kl": 0.17174102365970612, + "mean_kl": 0.013310780748724937, + "reference_nll": 1.3138129711151123, + "task_id": 132, + "token_count": 22, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8299505710601807, + "code_tokens": 34, + "cvar95_kl": 0.1001550629734993, + "delta_nll": -0.038953542709350586, + "max_kl": 0.15361452102661133, + "mean_kl": 0.012796497903764248, + "reference_nll": 0.8689041137695312, + "task_id": 133, + "token_count": 34, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8553503751754761, + "code_tokens": 81, + "cvar95_kl": 0.1243412047624588, + "delta_nll": -0.003292858600616455, + "max_kl": 0.1785905510187149, + "mean_kl": 0.019397631287574768, + "reference_nll": 0.8586432337760925, + "task_id": 134, + "token_count": 81, + "top1_agreement": 0.9382715821266174 + }, + { + "candidate_nll": 1.2711421251296997, + "code_tokens": 17, + "cvar95_kl": 0.09637020528316498, + "delta_nll": 0.009038805961608887, + "max_kl": 0.09637020528316498, + "mean_kl": 0.011124211363494396, + "reference_nll": 1.2621033191680908, + "task_id": 135, + "token_count": 17, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.848625659942627, + "code_tokens": 181, + "cvar95_kl": 0.18131645023822784, + "delta_nll": 0.023368239402770996, + "max_kl": 0.3662872612476349, + "mean_kl": 0.024028919637203217, + "reference_nll": 0.825257420539856, + "task_id": 136, + "token_count": 181, + "top1_agreement": 0.939226508140564 + }, + { + "candidate_nll": 0.956621527671814, + "code_tokens": 63, + "cvar95_kl": 0.15893739461898804, + "delta_nll": -0.023786962032318115, + "max_kl": 0.21966232359409332, + "mean_kl": 0.017625290900468826, + "reference_nll": 0.9804084897041321, + "task_id": 137, + "token_count": 63, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.696343719959259, + "code_tokens": 35, + "cvar95_kl": 0.02632366679608822, + "delta_nll": 0.002820611000061035, + "max_kl": 0.028263360261917114, + "mean_kl": 0.00470143835991621, + "reference_nll": 0.693523108959198, + "task_id": 138, + "token_count": 35, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8196250796318054, + "code_tokens": 25, + "cvar95_kl": 0.05328430235385895, + "delta_nll": 0.01483762264251709, + "max_kl": 0.06226477771997452, + "mean_kl": 0.008671948686242104, + "reference_nll": 0.8047874569892883, + "task_id": 139, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0842305421829224, + "code_tokens": 60, + "cvar95_kl": 0.22999875247478485, + "delta_nll": -0.0041348934173583984, + "max_kl": 0.4604836404323578, + "mean_kl": 0.019447477534413338, + "reference_nll": 1.0883654356002808, + "task_id": 140, + "token_count": 60, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 0.9542588591575623, + "code_tokens": 86, + "cvar95_kl": 0.34441977739334106, + "delta_nll": 0.04261195659637451, + "max_kl": 0.9365884065628052, + "mean_kl": 0.037562668323516846, + "reference_nll": 0.9116469025611877, + "task_id": 141, + "token_count": 86, + "top1_agreement": 0.9069767594337463 + }, + { + "candidate_nll": 0.9202831387519836, + "code_tokens": 40, + "cvar95_kl": 0.07476456463336945, + "delta_nll": -0.005882620811462402, + "max_kl": 0.08649321645498276, + "mean_kl": 0.010993857868015766, + "reference_nll": 0.926165759563446, + "task_id": 142, + "token_count": 40, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 1.5913984775543213, + "code_tokens": 27, + "cvar95_kl": 0.09197138249874115, + "delta_nll": 0.02783823013305664, + "max_kl": 0.10203124582767487, + "mean_kl": 0.02145189791917801, + "reference_nll": 1.5635602474212646, + "task_id": 143, + "token_count": 27, + "top1_agreement": 0.9259259104728699 + }, + { + "candidate_nll": 1.1272876262664795, + "code_tokens": 52, + "cvar95_kl": 0.34284552931785583, + "delta_nll": 0.0377047061920166, + "max_kl": 0.44278863072395325, + "mean_kl": 0.03218446671962738, + "reference_nll": 1.089582920074463, + "task_id": 144, + "token_count": 52, + "top1_agreement": 0.942307710647583 + }, + { + "candidate_nll": 0.750076949596405, + "code_tokens": 71, + "cvar95_kl": 0.1313343495130539, + "delta_nll": 0.008202850818634033, + "max_kl": 0.15071649849414825, + "mean_kl": 0.016292056068778038, + "reference_nll": 0.741874098777771, + "task_id": 145, + "token_count": 71, + "top1_agreement": 0.98591548204422 + }, + { + "candidate_nll": 1.149425745010376, + "code_tokens": 25, + "cvar95_kl": 0.04569534212350845, + "delta_nll": 0.04757404327392578, + "max_kl": 0.0552520677447319, + "mean_kl": 0.007000432815402746, + "reference_nll": 1.1018517017364502, + "task_id": 146, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6563074588775635, + "code_tokens": 103, + "cvar95_kl": 0.4390580952167511, + "delta_nll": 0.06181180477142334, + "max_kl": 0.8915877938270569, + "mean_kl": 0.04762720689177513, + "reference_nll": 0.5944956541061401, + "task_id": 147, + "token_count": 103, + "top1_agreement": 0.9514563083648682 + }, + { + "candidate_nll": 1.088447093963623, + "code_tokens": 116, + "cvar95_kl": 0.2054053395986557, + "delta_nll": 0.030909061431884766, + "max_kl": 0.2872523367404938, + "mean_kl": 0.03277267515659332, + "reference_nll": 1.0575380325317383, + "task_id": 148, + "token_count": 116, + "top1_agreement": 0.9741379022598267 + }, + { + "candidate_nll": 0.6953474879264832, + "code_tokens": 107, + "cvar95_kl": 0.19168657064437866, + "delta_nll": 0.04700124263763428, + "max_kl": 0.3779585063457489, + "mean_kl": 0.02614414133131504, + "reference_nll": 0.6483462452888489, + "task_id": 149, + "token_count": 107, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9592768549919128, + "code_tokens": 54, + "cvar95_kl": 0.07017656415700912, + "delta_nll": 0.01623910665512085, + "max_kl": 0.08317282050848007, + "mean_kl": 0.01671728678047657, + "reference_nll": 0.943037748336792, + "task_id": 150, + "token_count": 54, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 0.6619073748588562, + "code_tokens": 46, + "cvar95_kl": 0.05695221945643425, + "delta_nll": 0.02851492166519165, + "max_kl": 0.06288282573223114, + "mean_kl": 0.011976583860814571, + "reference_nll": 0.6333924531936646, + "task_id": 151, + "token_count": 46, + "top1_agreement": 0.97826087474823 + }, + { + "candidate_nll": 0.35764214396476746, + "code_tokens": 170, + "cvar95_kl": 0.2566520571708679, + "delta_nll": 0.028495371341705322, + "max_kl": 1.0390961170196533, + "mean_kl": 0.023964067921042442, + "reference_nll": 0.32914677262306213, + "task_id": 152, + "token_count": 170, + "top1_agreement": 0.9764705896377563 + }, + { + "candidate_nll": 1.429614543914795, + "code_tokens": 45, + "cvar95_kl": 0.19061650335788727, + "delta_nll": 0.0029222965240478516, + "max_kl": 0.3113153576850891, + "mean_kl": 0.031217796728014946, + "reference_nll": 1.426692247390747, + "task_id": 153, + "token_count": 45, + "top1_agreement": 0.9777777791023254 + }, + { + "candidate_nll": 1.3891710042953491, + "code_tokens": 24, + "cvar95_kl": 0.062423184514045715, + "delta_nll": -0.019132137298583984, + "max_kl": 0.07845759391784668, + "mean_kl": 0.010852258652448654, + "reference_nll": 1.408303141593933, + "task_id": 154, + "token_count": 24, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.1514794826507568, + "code_tokens": 78, + "cvar95_kl": 0.17010948061943054, + "delta_nll": -0.010480403900146484, + "max_kl": 0.27418041229248047, + "mean_kl": 0.023200465366244316, + "reference_nll": 1.1619598865509033, + "task_id": 155, + "token_count": 78, + "top1_agreement": 0.9871794581413269 + }, + { + "candidate_nll": 0.6626686453819275, + "code_tokens": 33, + "cvar95_kl": 0.08466114103794098, + "delta_nll": 0.03211158514022827, + "max_kl": 0.10702353715896606, + "mean_kl": 0.012856557965278625, + "reference_nll": 0.6305570602416992, + "task_id": 156, + "token_count": 33, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1069246530532837, + "code_tokens": 32, + "cvar95_kl": 0.06703774631023407, + "delta_nll": -0.008744955062866211, + "max_kl": 0.06991538405418396, + "mean_kl": 0.014471743255853653, + "reference_nll": 1.11566960811615, + "task_id": 157, + "token_count": 32, + "top1_agreement": 0.90625 + }, + { + "candidate_nll": 1.0163607597351074, + "code_tokens": 77, + "cvar95_kl": 0.19300803542137146, + "delta_nll": 0.027610957622528076, + "max_kl": 0.2215922772884369, + "mean_kl": 0.035686567425727844, + "reference_nll": 0.9887498021125793, + "task_id": 158, + "token_count": 77, + "top1_agreement": 0.9610389471054077 + }, + { + "candidate_nll": 0.43587562441825867, + "code_tokens": 222, + "cvar95_kl": 0.22744791209697723, + "delta_nll": 0.013625025749206543, + "max_kl": 0.7051600813865662, + "mean_kl": 0.022994237020611763, + "reference_nll": 0.4222505986690521, + "task_id": 159, + "token_count": 222, + "top1_agreement": 0.977477490901947 + }, + { + "candidate_nll": 1.0751713514328003, + "code_tokens": 86, + "cvar95_kl": 0.1472378671169281, + "delta_nll": 0.026576519012451172, + "max_kl": 0.2629011869430542, + "mean_kl": 0.026742752641439438, + "reference_nll": 1.0485948324203491, + "task_id": 160, + "token_count": 86, + "top1_agreement": 0.9767441749572754 + }, + { + "candidate_nll": 0.3297203779220581, + "code_tokens": 31, + "cvar95_kl": 0.021063242107629776, + "delta_nll": -0.004329115152359009, + "max_kl": 0.022107090801000595, + "mean_kl": 0.0032711748499423265, + "reference_nll": 0.3340494930744171, + "task_id": 161, + "token_count": 31, + "top1_agreement": 0.9677419066429138 + }, + { + "candidate_nll": 0.5587046146392822, + "code_tokens": 34, + "cvar95_kl": 0.08774743974208832, + "delta_nll": 0.04710453748703003, + "max_kl": 0.11584483087062836, + "mean_kl": 0.010845405980944633, + "reference_nll": 0.5116000771522522, + "task_id": 162, + "token_count": 34, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 1.4207301139831543, + "code_tokens": 38, + "cvar95_kl": 0.14879849553108215, + "delta_nll": 0.03326976299285889, + "max_kl": 0.1825178563594818, + "mean_kl": 0.022296695038676262, + "reference_nll": 1.3874603509902954, + "task_id": 163, + "token_count": 38, + "top1_agreement": 0.9210526347160339 + }, + { + "candidate_nll": 1.0165644884109497, + "code_tokens": 93, + "cvar95_kl": 0.19395574927330017, + "delta_nll": 0.05351990461349487, + "max_kl": 0.3747785687446594, + "mean_kl": 0.024356884881854057, + "reference_nll": 0.9630445837974548, + "task_id": 164, + "token_count": 93, + "top1_agreement": 0.9247311949729919 + }, + { + "candidate_nll": 0.7907591462135315, + "code_tokens": 70, + "cvar95_kl": 0.09471829235553741, + "delta_nll": 0.012153446674346924, + "max_kl": 0.16135744750499725, + "mean_kl": 0.01354183442890644, + "reference_nll": 0.7786056995391846, + "task_id": 165, + "token_count": 70, + "top1_agreement": 0.9714285731315613 + }, + { + "candidate_nll": 0.820806622505188, + "code_tokens": 67, + "cvar95_kl": 0.04672534018754959, + "delta_nll": 0.010314583778381348, + "max_kl": 0.07670657336711884, + "mean_kl": 0.008531823754310608, + "reference_nll": 0.8104920387268066, + "task_id": 166, + "token_count": 67, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3144588470458984, + "code_tokens": 66, + "cvar95_kl": 0.07976183295249939, + "delta_nll": 0.028331518173217773, + "max_kl": 0.10420050472021103, + "mean_kl": 0.01638997718691826, + "reference_nll": 1.2861273288726807, + "task_id": 167, + "token_count": 66, + "top1_agreement": 0.9848484992980957 + }, + { + "candidate_nll": 1.0022873878479004, + "code_tokens": 34, + "cvar95_kl": 0.05674033612012863, + "delta_nll": -0.01000678539276123, + "max_kl": 0.08798657357692719, + "mean_kl": 0.007727882359176874, + "reference_nll": 1.0122941732406616, + "task_id": 168, + "token_count": 34, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9716688394546509, + "code_tokens": 64, + "cvar95_kl": 0.19187802076339722, + "delta_nll": 0.06164348125457764, + "max_kl": 0.35933563113212585, + "mean_kl": 0.02995060384273529, + "reference_nll": 0.9100253582000732, + "task_id": 169, + "token_count": 64, + "top1_agreement": 0.96875 + }, + { + "candidate_nll": 1.3131797313690186, + "code_tokens": 56, + "cvar95_kl": 0.08919859677553177, + "delta_nll": 0.0028241872787475586, + "max_kl": 0.12111788988113403, + "mean_kl": 0.016529370099306107, + "reference_nll": 1.310355544090271, + "task_id": 170, + "token_count": 56, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 1.1532632112503052, + "code_tokens": 21, + "cvar95_kl": 0.03725045174360275, + "delta_nll": 0.028180837631225586, + "max_kl": 0.04516642168164253, + "mean_kl": 0.009610004723072052, + "reference_nll": 1.1250823736190796, + "task_id": 171, + "token_count": 21, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6859344840049744, + "code_tokens": 60, + "cvar95_kl": 0.12443554401397705, + "delta_nll": 0.020594120025634766, + "max_kl": 0.1374540477991104, + "mean_kl": 0.016887495294213295, + "reference_nll": 0.6653403639793396, + "task_id": 172, + "token_count": 60, + "top1_agreement": 0.9833333492279053 + }, + { + "candidate_nll": 1.3914880752563477, + "code_tokens": 29, + "cvar95_kl": 0.04935797303915024, + "delta_nll": 0.03771209716796875, + "max_kl": 0.0741412565112114, + "mean_kl": 0.007636839523911476, + "reference_nll": 1.353775978088379, + "task_id": 173, + "token_count": 29, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7291414141654968, + "code_tokens": 34, + "cvar95_kl": 0.0323595330119133, + "delta_nll": 0.007096529006958008, + "max_kl": 0.04022804647684097, + "mean_kl": 0.005777974147349596, + "reference_nll": 0.7220448851585388, + "task_id": 174, + "token_count": 34, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0062140226364136, + "code_tokens": 88, + "cvar95_kl": 0.19341956079006195, + "delta_nll": 0.08710652589797974, + "max_kl": 0.26848381757736206, + "mean_kl": 0.025945184752345085, + "reference_nll": 0.9191074967384338, + "task_id": 175, + "token_count": 88, + "top1_agreement": 0.9204545617103577 + }, + { + "candidate_nll": 0.5580580830574036, + "code_tokens": 17, + "cvar95_kl": 0.018623322248458862, + "delta_nll": 0.014834463596343994, + "max_kl": 0.018623322248458862, + "mean_kl": 0.003931140527129173, + "reference_nll": 0.5432236194610596, + "task_id": 176, + "token_count": 17, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.8927594423294067, + "code_tokens": 35, + "cvar95_kl": 0.03589785099029541, + "delta_nll": 0.0022122859954833984, + "max_kl": 0.037147827446460724, + "mean_kl": 0.012798544019460678, + "reference_nll": 1.8905471563339233, + "task_id": 177, + "token_count": 35, + "top1_agreement": 0.9428571462631226 + }, + { + "candidate_nll": 0.8864004611968994, + "code_tokens": 48, + "cvar95_kl": 0.0444582998752594, + "delta_nll": -0.0033760666847229004, + "max_kl": 0.05535239353775978, + "mean_kl": 0.006467301864176989, + "reference_nll": 0.8897765278816223, + "task_id": 178, + "token_count": 48, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0797395706176758, + "code_tokens": 130, + "cvar95_kl": 0.27927032113075256, + "delta_nll": 0.08946716785430908, + "max_kl": 0.4932416081428528, + "mean_kl": 0.04968521371483803, + "reference_nll": 0.9902724027633667, + "task_id": 179, + "token_count": 130, + "top1_agreement": 0.8769230842590332 + }, + { + "candidate_nll": 1.1828922033309937, + "code_tokens": 65, + "cvar95_kl": 0.6781758666038513, + "delta_nll": 0.08795273303985596, + "max_kl": 1.8646786212921143, + "mean_kl": 0.060973651707172394, + "reference_nll": 1.0949394702911377, + "task_id": 180, + "token_count": 65, + "top1_agreement": 0.9230769276618958 + }, + { + "candidate_nll": 0.8599193096160889, + "code_tokens": 145, + "cvar95_kl": 0.15036025643348694, + "delta_nll": -0.015095651149749756, + "max_kl": 0.2620132863521576, + "mean_kl": 0.02092103101313114, + "reference_nll": 0.8750149607658386, + "task_id": 181, + "token_count": 145, + "top1_agreement": 0.9862068891525269 + }, + { + "candidate_nll": 0.7244014143943787, + "code_tokens": 80, + "cvar95_kl": 0.09121755510568619, + "delta_nll": 0.008702099323272705, + "max_kl": 0.12537220120429993, + "mean_kl": 0.01049826666712761, + "reference_nll": 0.715699315071106, + "task_id": 182, + "token_count": 80, + "top1_agreement": 0.9750000238418579 + }, + { + "candidate_nll": 0.6157228350639343, + "code_tokens": 70, + "cvar95_kl": 0.16858160495758057, + "delta_nll": 0.03358602523803711, + "max_kl": 0.2778235673904419, + "mean_kl": 0.02104162611067295, + "reference_nll": 0.5821368098258972, + "task_id": 183, + "token_count": 70, + "top1_agreement": 0.9857142567634583 + }, + { + "candidate_nll": 1.526713252067566, + "code_tokens": 25, + "cvar95_kl": 0.14842930436134338, + "delta_nll": 0.03842425346374512, + "max_kl": 0.2243214249610901, + "mean_kl": 0.01813885197043419, + "reference_nll": 1.4882889986038208, + "task_id": 184, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 1.6070866584777832, + "code_tokens": 50, + "cvar95_kl": 0.16930989921092987, + "delta_nll": -0.047296762466430664, + "max_kl": 0.2722146213054657, + "mean_kl": 0.029366016387939453, + "reference_nll": 1.6543834209442139, + "task_id": 185, + "token_count": 50, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 0.8143222332000732, + "code_tokens": 48, + "cvar95_kl": 0.06645497679710388, + "delta_nll": 0.016279280185699463, + "max_kl": 0.10383414477109909, + "mean_kl": 0.00804347638040781, + "reference_nll": 0.7980429530143738, + "task_id": 186, + "token_count": 48, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.3967968225479126, + "code_tokens": 104, + "cvar95_kl": 0.3512706458568573, + "delta_nll": 0.05272442102432251, + "max_kl": 0.7471973896026611, + "mean_kl": 0.02984154038131237, + "reference_nll": 0.3440724015235901, + "task_id": 187, + "token_count": 104, + "top1_agreement": 0.9711538553237915 + }, + { + "candidate_nll": 1.002343773841858, + "code_tokens": 68, + "cvar95_kl": 0.18962903320789337, + "delta_nll": 0.012349486351013184, + "max_kl": 0.4029476046562195, + "mean_kl": 0.023615362122654915, + "reference_nll": 0.9899942874908447, + "task_id": 188, + "token_count": 68, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 0.8051608204841614, + "code_tokens": 145, + "cvar95_kl": 0.20623886585235596, + "delta_nll": 0.050248682498931885, + "max_kl": 0.3528914451599121, + "mean_kl": 0.033667776733636856, + "reference_nll": 0.7549121379852295, + "task_id": 189, + "token_count": 145, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 0.8192042708396912, + "code_tokens": 41, + "cvar95_kl": 0.06287894397974014, + "delta_nll": -0.02568352222442627, + "max_kl": 0.11295510828495026, + "mean_kl": 0.010265215300023556, + "reference_nll": 0.8448877930641174, + "task_id": 190, + "token_count": 41, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7760279774665833, + "code_tokens": 53, + "cvar95_kl": 0.11391810327768326, + "delta_nll": 0.014905691146850586, + "max_kl": 0.2624571919441223, + "mean_kl": 0.011356521397829056, + "reference_nll": 0.7611222863197327, + "task_id": 191, + "token_count": 53, + "top1_agreement": 0.9433962106704712 + }, + { + "candidate_nll": 0.8070585131645203, + "code_tokens": 58, + "cvar95_kl": 0.05262140557169914, + "delta_nll": 0.014828741550445557, + "max_kl": 0.0720796138048172, + "mean_kl": 0.00817890651524067, + "reference_nll": 0.7922297716140747, + "task_id": 192, + "token_count": 58, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.7261995077133179, + "code_tokens": 24, + "cvar95_kl": 0.0719340369105339, + "delta_nll": -0.005860567092895508, + "max_kl": 0.1017761379480362, + "mean_kl": 0.012648685835301876, + "reference_nll": 1.7320600748062134, + "task_id": 193, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0063754320144653, + "code_tokens": 89, + "cvar95_kl": 0.15092888474464417, + "delta_nll": 0.014521479606628418, + "max_kl": 0.22943846881389618, + "mean_kl": 0.02337472140789032, + "reference_nll": 0.9918539524078369, + "task_id": 194, + "token_count": 89, + "top1_agreement": 0.9101123809814453 + }, + { + "candidate_nll": 0.5223227143287659, + "code_tokens": 101, + "cvar95_kl": 0.11533913016319275, + "delta_nll": 0.037246525287628174, + "max_kl": 0.1734357625246048, + "mean_kl": 0.014692813158035278, + "reference_nll": 0.4850761890411377, + "task_id": 195, + "token_count": 101, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.48482346534729, + "code_tokens": 33, + "cvar95_kl": 0.08521343767642975, + "delta_nll": 0.02726566791534424, + "max_kl": 0.09981763362884521, + "mean_kl": 0.013455976732075214, + "reference_nll": 1.4575577974319458, + "task_id": 196, + "token_count": 33, + "top1_agreement": 0.939393937587738 + }, + { + "candidate_nll": 1.043990969657898, + "code_tokens": 50, + "cvar95_kl": 0.05402359366416931, + "delta_nll": -0.002674579620361328, + "max_kl": 0.0663384348154068, + "mean_kl": 0.009079700335860252, + "reference_nll": 1.0466655492782593, + "task_id": 197, + "token_count": 50, + "top1_agreement": 0.9800000190734863 + }, + { + "candidate_nll": 1.477325439453125, + "code_tokens": 58, + "cvar95_kl": 0.23442701995372772, + "delta_nll": 0.06857264041900635, + "max_kl": 0.3104332685470581, + "mean_kl": 0.03066365234553814, + "reference_nll": 1.4087527990341187, + "task_id": 198, + "token_count": 58, + "top1_agreement": 0.8793103694915771 + }, + { + "candidate_nll": 1.0035998821258545, + "code_tokens": 62, + "cvar95_kl": 0.11086013913154602, + "delta_nll": 0.0244409441947937, + "max_kl": 0.22699394822120667, + "mean_kl": 0.015336215496063232, + "reference_nll": 0.9791589379310608, + "task_id": 199, + "token_count": 62, + "top1_agreement": 0.9838709831237793 + }, + { + "candidate_nll": 0.7177208662033081, + "code_tokens": 42, + "cvar95_kl": 0.043366700410842896, + "delta_nll": 0.026030898094177246, + "max_kl": 0.05039721727371216, + "mean_kl": 0.009174838662147522, + "reference_nll": 0.6916899681091309, + "task_id": 200, + "token_count": 42, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9481451511383057, + "code_tokens": 15, + "cvar95_kl": 0.04327481985092163, + "delta_nll": 0.023404181003570557, + "max_kl": 0.04327481985092163, + "mean_kl": 0.00911701936274767, + "reference_nll": 0.9247409701347351, + "task_id": 201, + "token_count": 15, + "top1_agreement": 0.8666666746139526 + }, + { + "candidate_nll": 0.7792495489120483, + "code_tokens": 57, + "cvar95_kl": 0.08930715173482895, + "delta_nll": 0.04046738147735596, + "max_kl": 0.10167964547872543, + "mean_kl": 0.01224775891751051, + "reference_nll": 0.7387821674346924, + "task_id": 202, + "token_count": 57, + "top1_agreement": 0.9824561476707458 + }, + { + "candidate_nll": 1.0769174098968506, + "code_tokens": 58, + "cvar95_kl": 0.13590091466903687, + "delta_nll": 0.03338146209716797, + "max_kl": 0.20398111641407013, + "mean_kl": 0.014771481975913048, + "reference_nll": 1.0435359477996826, + "task_id": 203, + "token_count": 58, + "top1_agreement": 0.982758641242981 + }, + { + "candidate_nll": 0.890396773815155, + "code_tokens": 45, + "cvar95_kl": 0.021716296672821045, + "delta_nll": 0.01842564344406128, + "max_kl": 0.027186071500182152, + "mean_kl": 0.006468243896961212, + "reference_nll": 0.8719711303710938, + "task_id": 204, + "token_count": 45, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.052309513092041, + "code_tokens": 32, + "cvar95_kl": 0.09373478591442108, + "delta_nll": 0.02307915687561035, + "max_kl": 0.09755130857229233, + "mean_kl": 0.023801608011126518, + "reference_nll": 2.0292303562164307, + "task_id": 205, + "token_count": 32, + "top1_agreement": 0.90625 + }, + { + "candidate_nll": 1.3176850080490112, + "code_tokens": 39, + "cvar95_kl": 0.10308480262756348, + "delta_nll": -0.002981424331665039, + "max_kl": 0.10648398846387863, + "mean_kl": 0.018076062202453613, + "reference_nll": 1.3206664323806763, + "task_id": 206, + "token_count": 39, + "top1_agreement": 0.9743589758872986 + }, + { + "candidate_nll": 0.47168096899986267, + "code_tokens": 136, + "cvar95_kl": 0.49404749274253845, + "delta_nll": 0.06091967225074768, + "max_kl": 1.6326924562454224, + "mean_kl": 0.03985569253563881, + "reference_nll": 0.410761296749115, + "task_id": 207, + "token_count": 136, + "top1_agreement": 0.9632353186607361 + }, + { + "candidate_nll": 1.2545093297958374, + "code_tokens": 50, + "cvar95_kl": 0.06325799971818924, + "delta_nll": 0.0337526798248291, + "max_kl": 0.06908475607633591, + "mean_kl": 0.013022353872656822, + "reference_nll": 1.2207566499710083, + "task_id": 208, + "token_count": 50, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.8670791983604431, + "code_tokens": 34, + "cvar95_kl": 0.09096074104309082, + "delta_nll": 0.001163482666015625, + "max_kl": 0.10687169432640076, + "mean_kl": 0.019301733002066612, + "reference_nll": 0.8659157156944275, + "task_id": 209, + "token_count": 34, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 1.224428653717041, + "code_tokens": 44, + "cvar95_kl": 0.21004492044448853, + "delta_nll": 0.045638442039489746, + "max_kl": 0.36780300736427307, + "mean_kl": 0.02818625420331955, + "reference_nll": 1.1787902116775513, + "task_id": 210, + "token_count": 44, + "top1_agreement": 0.9772727489471436 + }, + { + "candidate_nll": 1.4386091232299805, + "code_tokens": 36, + "cvar95_kl": 0.22796814143657684, + "delta_nll": 0.005187273025512695, + "max_kl": 0.24844418466091156, + "mean_kl": 0.018831899389624596, + "reference_nll": 1.4334218502044678, + "task_id": 211, + "token_count": 36, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 0.7561736106872559, + "code_tokens": 46, + "cvar95_kl": 0.06721463799476624, + "delta_nll": -0.04698359966278076, + "max_kl": 0.10795515030622482, + "mean_kl": 0.008395674638450146, + "reference_nll": 0.8031572103500366, + "task_id": 212, + "token_count": 46, + "top1_agreement": 0.97826087474823 + }, + { + "candidate_nll": 1.169716238975525, + "code_tokens": 48, + "cvar95_kl": 0.11036282032728195, + "delta_nll": -0.010623693466186523, + "max_kl": 0.17137077450752258, + "mean_kl": 0.013569104485213757, + "reference_nll": 1.1803399324417114, + "task_id": 213, + "token_count": 48, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.9302911162376404, + "code_tokens": 25, + "cvar95_kl": 0.06488487124443054, + "delta_nll": 0.03488004207611084, + "max_kl": 0.08935421705245972, + "mean_kl": 0.010580794885754585, + "reference_nll": 0.8954110741615295, + "task_id": 214, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.5690504312515259, + "code_tokens": 69, + "cvar95_kl": 0.2743968963623047, + "delta_nll": 0.06477141380310059, + "max_kl": 0.3428688049316406, + "mean_kl": 0.052509043365716934, + "reference_nll": 1.5042790174484253, + "task_id": 215, + "token_count": 69, + "top1_agreement": 0.9275362491607666 + }, + { + "candidate_nll": 0.9869621992111206, + "code_tokens": 60, + "cvar95_kl": 0.09032172709703445, + "delta_nll": 0.021504521369934082, + "max_kl": 0.10716209560632706, + "mean_kl": 0.014770526438951492, + "reference_nll": 0.9654576778411865, + "task_id": 216, + "token_count": 60, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 0.9450049996376038, + "code_tokens": 49, + "cvar95_kl": 0.3075001537799835, + "delta_nll": 0.03052753210067749, + "max_kl": 0.5980578064918518, + "mean_kl": 0.025667134672403336, + "reference_nll": 0.9144774675369263, + "task_id": 217, + "token_count": 49, + "top1_agreement": 0.9591836929321289 + }, + { + "candidate_nll": 1.7199658155441284, + "code_tokens": 44, + "cvar95_kl": 0.12357372045516968, + "delta_nll": 0.019437551498413086, + "max_kl": 0.19891254603862762, + "mean_kl": 0.018080906942486763, + "reference_nll": 1.7005282640457153, + "task_id": 218, + "token_count": 44, + "top1_agreement": 0.9090909361839294 + }, + { + "candidate_nll": 1.1807938814163208, + "code_tokens": 81, + "cvar95_kl": 0.1671801060438156, + "delta_nll": 0.035379648208618164, + "max_kl": 0.28339046239852905, + "mean_kl": 0.02279585599899292, + "reference_nll": 1.1454142332077026, + "task_id": 219, + "token_count": 81, + "top1_agreement": 0.9753086566925049 + }, + { + "candidate_nll": 1.4130301475524902, + "code_tokens": 26, + "cvar95_kl": 0.15223784744739532, + "delta_nll": -0.023705124855041504, + "max_kl": 0.20396928489208221, + "mean_kl": 0.021599356085062027, + "reference_nll": 1.4367352724075317, + "task_id": 220, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8968881368637085, + "code_tokens": 32, + "cvar95_kl": 0.0216642115265131, + "delta_nll": 0.016967058181762695, + "max_kl": 0.026817820966243744, + "mean_kl": 0.00474198954179883, + "reference_nll": 0.8799210786819458, + "task_id": 221, + "token_count": 32, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9883370399475098, + "code_tokens": 48, + "cvar95_kl": 0.11706414818763733, + "delta_nll": 0.04059511423110962, + "max_kl": 0.16666996479034424, + "mean_kl": 0.012294181622564793, + "reference_nll": 0.9477419257164001, + "task_id": 222, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 0.7356714606285095, + "code_tokens": 180, + "cvar95_kl": 0.2724216878414154, + "delta_nll": 0.0363503098487854, + "max_kl": 0.6077927947044373, + "mean_kl": 0.03248777985572815, + "reference_nll": 0.6993211507797241, + "task_id": 223, + "token_count": 180, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 0.757544755935669, + "code_tokens": 38, + "cvar95_kl": 0.07646464556455612, + "delta_nll": 0.03578317165374756, + "max_kl": 0.0917268693447113, + "mean_kl": 0.010229659266769886, + "reference_nll": 0.7217615842819214, + "task_id": 224, + "token_count": 38, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 0.7046133279800415, + "code_tokens": 90, + "cvar95_kl": 0.1429987996816635, + "delta_nll": 0.009551644325256348, + "max_kl": 0.251743346452713, + "mean_kl": 0.015939004719257355, + "reference_nll": 0.6950616836547852, + "task_id": 225, + "token_count": 90, + "top1_agreement": 0.9777777791023254 + }, + { + "candidate_nll": 0.593610942363739, + "code_tokens": 44, + "cvar95_kl": 0.046527501195669174, + "delta_nll": 0.018980860710144043, + "max_kl": 0.06349539011716843, + "mean_kl": 0.009026930667459965, + "reference_nll": 0.574630081653595, + "task_id": 226, + "token_count": 44, + "top1_agreement": 0.9545454382896423 + }, + { + "candidate_nll": 0.7445516586303711, + "code_tokens": 60, + "cvar95_kl": 0.03287943825125694, + "delta_nll": 0.0006514787673950195, + "max_kl": 0.037831876426935196, + "mean_kl": 0.007794437929987907, + "reference_nll": 0.7439001798629761, + "task_id": 227, + "token_count": 60, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 1.0416145324707031, + "code_tokens": 69, + "cvar95_kl": 0.08711522072553635, + "delta_nll": 0.03881645202636719, + "max_kl": 0.11568035185337067, + "mean_kl": 0.015331679955124855, + "reference_nll": 1.002798080444336, + "task_id": 228, + "token_count": 69, + "top1_agreement": 0.95652174949646 + }, + { + "candidate_nll": 0.5475552082061768, + "code_tokens": 71, + "cvar95_kl": 0.08801580220460892, + "delta_nll": 0.029993414878845215, + "max_kl": 0.1719771772623062, + "mean_kl": 0.013337667100131512, + "reference_nll": 0.5175617933273315, + "task_id": 229, + "token_count": 71, + "top1_agreement": 0.98591548204422 + }, + { + "candidate_nll": 1.2566879987716675, + "code_tokens": 22, + "cvar95_kl": 0.05906015634536743, + "delta_nll": 0.014014840126037598, + "max_kl": 0.06663081794977188, + "mean_kl": 0.012938239611685276, + "reference_nll": 1.2426731586456299, + "task_id": 230, + "token_count": 22, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5818655490875244, + "code_tokens": 215, + "cvar95_kl": 0.3518262803554535, + "delta_nll": 0.028086066246032715, + "max_kl": 0.6824665069580078, + "mean_kl": 0.03762124851346016, + "reference_nll": 0.5537794828414917, + "task_id": 231, + "token_count": 215, + "top1_agreement": 0.9581395387649536 + }, + { + "candidate_nll": 1.1442958116531372, + "code_tokens": 26, + "cvar95_kl": 0.1278950423002243, + "delta_nll": 0.041506171226501465, + "max_kl": 0.1590590476989746, + "mean_kl": 0.014564533717930317, + "reference_nll": 1.1027896404266357, + "task_id": 232, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9074442982673645, + "code_tokens": 33, + "cvar95_kl": 0.027417317032814026, + "delta_nll": 0.03919762372970581, + "max_kl": 0.027459489181637764, + "mean_kl": 0.00559984240680933, + "reference_nll": 0.8682466745376587, + "task_id": 233, + "token_count": 33, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 0.9665151238441467, + "code_tokens": 18, + "cvar95_kl": 0.01866849698126316, + "delta_nll": 0.030367016792297363, + "max_kl": 0.01866849698126316, + "mean_kl": 0.00387908099219203, + "reference_nll": 0.9361481070518494, + "task_id": 234, + "token_count": 18, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2724621295928955, + "code_tokens": 71, + "cvar95_kl": 0.08477944880723953, + "delta_nll": -0.007599592208862305, + "max_kl": 0.12211053818464279, + "mean_kl": 0.016150981187820435, + "reference_nll": 1.2800617218017578, + "task_id": 235, + "token_count": 71, + "top1_agreement": 0.9718309640884399 + }, + { + "candidate_nll": 0.815497875213623, + "code_tokens": 107, + "cvar95_kl": 0.13198481500148773, + "delta_nll": 0.03409641981124878, + "max_kl": 0.26963046193122864, + "mean_kl": 0.019494881853461266, + "reference_nll": 0.7814014554023743, + "task_id": 236, + "token_count": 107, + "top1_agreement": 0.9345794320106506 + }, + { + "candidate_nll": 2.0853917598724365, + "code_tokens": 40, + "cvar95_kl": 0.1325688511133194, + "delta_nll": 0.030367136001586914, + "max_kl": 0.1665385216474533, + "mean_kl": 0.022636298090219498, + "reference_nll": 2.0550246238708496, + "task_id": 237, + "token_count": 40, + "top1_agreement": 0.875 + }, + { + "candidate_nll": 1.3811978101730347, + "code_tokens": 32, + "cvar95_kl": 0.05500086769461632, + "delta_nll": 0.01690089702606201, + "max_kl": 0.06804084777832031, + "mean_kl": 0.009330025874078274, + "reference_nll": 1.3642969131469727, + "task_id": 238, + "token_count": 32, + "top1_agreement": 0.90625 + }, + { + "candidate_nll": 0.7469038367271423, + "code_tokens": 130, + "cvar95_kl": 0.30498987436294556, + "delta_nll": 0.04765009880065918, + "max_kl": 0.5483905673027039, + "mean_kl": 0.0413372665643692, + "reference_nll": 0.6992537379264832, + "task_id": 239, + "token_count": 130, + "top1_agreement": 0.9615384340286255 + }, + { + "candidate_nll": 1.4902390241622925, + "code_tokens": 27, + "cvar95_kl": 0.09661456197500229, + "delta_nll": 0.01881730556488037, + "max_kl": 0.09723567962646484, + "mean_kl": 0.017629163339734077, + "reference_nll": 1.471421718597412, + "task_id": 240, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9685587882995605, + "code_tokens": 42, + "cvar95_kl": 0.05774541571736336, + "delta_nll": -0.004381537437438965, + "max_kl": 0.07072749733924866, + "mean_kl": 0.009707793593406677, + "reference_nll": 0.9729403257369995, + "task_id": 241, + "token_count": 42, + "top1_agreement": 0.976190447807312 + }, + { + "candidate_nll": 0.7589508295059204, + "code_tokens": 30, + "cvar95_kl": 0.11145144701004028, + "delta_nll": 0.01343756914138794, + "max_kl": 0.19195573031902313, + "mean_kl": 0.012907450087368488, + "reference_nll": 0.7455132603645325, + "task_id": 242, + "token_count": 30, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 1.426547646522522, + "code_tokens": 58, + "cvar95_kl": 0.13623236119747162, + "delta_nll": 0.007314801216125488, + "max_kl": 0.15686801075935364, + "mean_kl": 0.03174802288413048, + "reference_nll": 1.4192328453063965, + "task_id": 243, + "token_count": 58, + "top1_agreement": 0.8965517282485962 + }, + { + "candidate_nll": 1.2553317546844482, + "code_tokens": 34, + "cvar95_kl": 0.11715291440486908, + "delta_nll": 0.025104522705078125, + "max_kl": 0.12744444608688354, + "mean_kl": 0.013649707660079002, + "reference_nll": 1.2302272319793701, + "task_id": 244, + "token_count": 34, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 0.7546148896217346, + "code_tokens": 208, + "cvar95_kl": 0.3351515829563141, + "delta_nll": 0.053152382373809814, + "max_kl": 0.6987546682357788, + "mean_kl": 0.037108078598976135, + "reference_nll": 0.7014625072479248, + "task_id": 245, + "token_count": 208, + "top1_agreement": 0.9519230723381042 + }, + { + "candidate_nll": 0.9592627882957458, + "code_tokens": 83, + "cvar95_kl": 0.2621619403362274, + "delta_nll": 0.06035703420639038, + "max_kl": 0.3926297426223755, + "mean_kl": 0.04303685203194618, + "reference_nll": 0.8989057540893555, + "task_id": 246, + "token_count": 83, + "top1_agreement": 0.9156626462936401 + }, + { + "candidate_nll": 0.5983474254608154, + "code_tokens": 170, + "cvar95_kl": 0.6420617699623108, + "delta_nll": 0.07579177618026733, + "max_kl": 1.0071650743484497, + "mean_kl": 0.06521298736333847, + "reference_nll": 0.5225556492805481, + "task_id": 247, + "token_count": 170, + "top1_agreement": 0.9235293865203857 + }, + { + "candidate_nll": 0.8281319737434387, + "code_tokens": 40, + "cvar95_kl": 0.4161178767681122, + "delta_nll": 0.04671144485473633, + "max_kl": 0.7250508069992065, + "mean_kl": 0.03186020627617836, + "reference_nll": 0.7814205288887024, + "task_id": 248, + "token_count": 40, + "top1_agreement": 0.8999999761581421 + }, + { + "candidate_nll": 1.2646621465682983, + "code_tokens": 31, + "cvar95_kl": 0.06477034837007523, + "delta_nll": 0.017952561378479004, + "max_kl": 0.06540334969758987, + "mean_kl": 0.01020401157438755, + "reference_nll": 1.2467095851898193, + "task_id": 249, + "token_count": 31, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8328479528427124, + "code_tokens": 42, + "cvar95_kl": 0.023015856742858887, + "delta_nll": 0.0380779504776001, + "max_kl": 0.025921210646629333, + "mean_kl": 0.005562147591263056, + "reference_nll": 0.7947700023651123, + "task_id": 250, + "token_count": 42, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 1.826490879058838, + "code_tokens": 27, + "cvar95_kl": 0.2652459144592285, + "delta_nll": 0.05742490291595459, + "max_kl": 0.4264928102493286, + "mean_kl": 0.0271224956959486, + "reference_nll": 1.7690659761428833, + "task_id": 251, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.182363271713257, + "code_tokens": 25, + "cvar95_kl": 0.083458811044693, + "delta_nll": -0.017447233200073242, + "max_kl": 0.08822454512119293, + "mean_kl": 0.019222550094127655, + "reference_nll": 2.19981050491333, + "task_id": 252, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5833754539489746, + "code_tokens": 40, + "cvar95_kl": 0.022437192499637604, + "delta_nll": 0.014989018440246582, + "max_kl": 0.02465960942208767, + "mean_kl": 0.0043617067858576775, + "reference_nll": 0.568386435508728, + "task_id": 253, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3921664953231812, + "code_tokens": 24, + "cvar95_kl": 0.046399690210819244, + "delta_nll": 0.007811903953552246, + "max_kl": 0.05789554491639137, + "mean_kl": 0.010018565692007542, + "reference_nll": 1.384354591369629, + "task_id": 254, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8933620452880859, + "code_tokens": 28, + "cvar95_kl": 0.03368114307522774, + "delta_nll": 0.010617077350616455, + "max_kl": 0.04656984657049179, + "mean_kl": 0.007369654718786478, + "reference_nll": 0.8827449679374695, + "task_id": 255, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 0.49737828969955444, + "code_tokens": 71, + "cvar95_kl": 0.07234930247068405, + "delta_nll": 0.018513768911361694, + "max_kl": 0.11026468873023987, + "mean_kl": 0.010840476490557194, + "reference_nll": 0.47886452078819275, + "task_id": 256, + "token_count": 71, + "top1_agreement": 0.98591548204422 + }, + { + "candidate_nll": 0.5462310314178467, + "code_tokens": 24, + "cvar95_kl": 0.028377732262015343, + "delta_nll": 0.01687026023864746, + "max_kl": 0.032051362097263336, + "mean_kl": 0.004564752336591482, + "reference_nll": 0.5293607711791992, + "task_id": 257, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2650172710418701, + "code_tokens": 34, + "cvar95_kl": 0.18845850229263306, + "delta_nll": -0.006563782691955566, + "max_kl": 0.19432871043682098, + "mean_kl": 0.017599478363990784, + "reference_nll": 1.2715810537338257, + "task_id": 258, + "token_count": 34, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 1.1104015111923218, + "code_tokens": 64, + "cvar95_kl": 0.1669597625732422, + "delta_nll": -0.008892416954040527, + "max_kl": 0.2524101436138153, + "mean_kl": 0.02325761318206787, + "reference_nll": 1.1192939281463623, + "task_id": 259, + "token_count": 64, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7383589148521423, + "code_tokens": 45, + "cvar95_kl": 0.1636464148759842, + "delta_nll": 0.03176760673522949, + "max_kl": 0.27229180932044983, + "mean_kl": 0.01649208553135395, + "reference_nll": 0.7065913081169128, + "task_id": 260, + "token_count": 45, + "top1_agreement": 0.9555555582046509 + }, + { + "candidate_nll": 1.1632640361785889, + "code_tokens": 48, + "cvar95_kl": 0.035967592149972916, + "delta_nll": -0.03544008731842041, + "max_kl": 0.04670627415180206, + "mean_kl": 0.008134373463690281, + "reference_nll": 1.1987041234970093, + "task_id": 261, + "token_count": 48, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.700315535068512, + "code_tokens": 21, + "cvar95_kl": 0.04272149130702019, + "delta_nll": 0.00011438131332397461, + "max_kl": 0.05155440419912338, + "mean_kl": 0.0070027695037424564, + "reference_nll": 0.700201153755188, + "task_id": 262, + "token_count": 21, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7300011515617371, + "code_tokens": 24, + "cvar95_kl": 0.030669931322336197, + "delta_nll": -0.024412989616394043, + "max_kl": 0.04408050328493118, + "mean_kl": 0.0058592804707586765, + "reference_nll": 0.7544141411781311, + "task_id": 263, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 1.5851120948791504, + "code_tokens": 63, + "cvar95_kl": 0.10220500826835632, + "delta_nll": 0.013184785842895508, + "max_kl": 0.16760005056858063, + "mean_kl": 0.017532989382743835, + "reference_nll": 1.5719273090362549, + "task_id": 264, + "token_count": 63, + "top1_agreement": 0.920634925365448 + }, + { + "candidate_nll": 1.2545428276062012, + "code_tokens": 22, + "cvar95_kl": 0.07356076687574387, + "delta_nll": -0.024216294288635254, + "max_kl": 0.09511867165565491, + "mean_kl": 0.012750239111483097, + "reference_nll": 1.2787591218948364, + "task_id": 265, + "token_count": 22, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.115394115447998, + "code_tokens": 25, + "cvar95_kl": 0.063703253865242, + "delta_nll": -0.013268232345581055, + "max_kl": 0.06617457419633865, + "mean_kl": 0.013399746268987656, + "reference_nll": 1.128662347793579, + "task_id": 266, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.9245328903198242, + "code_tokens": 21, + "cvar95_kl": 0.24654391407966614, + "delta_nll": -0.03579282760620117, + "max_kl": 0.40789541602134705, + "mean_kl": 0.03035898506641388, + "reference_nll": 1.9603257179260254, + "task_id": 267, + "token_count": 21, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 1.444280743598938, + "code_tokens": 24, + "cvar95_kl": 0.17649874091148376, + "delta_nll": -0.022051334381103516, + "max_kl": 0.23360759019851685, + "mean_kl": 0.024244507774710655, + "reference_nll": 1.4663320779800415, + "task_id": 268, + "token_count": 24, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.8155454397201538, + "code_tokens": 15, + "cvar95_kl": 0.07768116891384125, + "delta_nll": 0.022057652473449707, + "max_kl": 0.07768116891384125, + "mean_kl": 0.011576362885534763, + "reference_nll": 1.793487787246704, + "task_id": 269, + "token_count": 15, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 0.8259164094924927, + "code_tokens": 59, + "cvar95_kl": 0.14911825954914093, + "delta_nll": 0.04505997896194458, + "max_kl": 0.17105750739574432, + "mean_kl": 0.02153564803302288, + "reference_nll": 0.7808564305305481, + "task_id": 270, + "token_count": 59, + "top1_agreement": 0.9322034120559692 + }, + { + "candidate_nll": 0.9475165605545044, + "code_tokens": 54, + "cvar95_kl": 0.0905202254652977, + "delta_nll": 0.013398349285125732, + "max_kl": 0.15906456112861633, + "mean_kl": 0.012838184833526611, + "reference_nll": 0.9341182112693787, + "task_id": 271, + "token_count": 54, + "top1_agreement": 0.9814814925193787 + }, + { + "candidate_nll": 1.9383805990219116, + "code_tokens": 28, + "cvar95_kl": 0.03513442724943161, + "delta_nll": -0.013478517532348633, + "max_kl": 0.053535256534814835, + "mean_kl": 0.006933960597962141, + "reference_nll": 1.9518591165542603, + "task_id": 272, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1059930324554443, + "code_tokens": 46, + "cvar95_kl": 0.12743939459323883, + "delta_nll": 0.021267056465148926, + "max_kl": 0.21708768606185913, + "mean_kl": 0.01821739412844181, + "reference_nll": 1.0847259759902954, + "task_id": 273, + "token_count": 46, + "top1_agreement": 0.97826087474823 + }, + { + "candidate_nll": 1.4836907386779785, + "code_tokens": 27, + "cvar95_kl": 0.05108633637428284, + "delta_nll": -0.031131625175476074, + "max_kl": 0.053166136145591736, + "mean_kl": 0.008891264908015728, + "reference_nll": 1.5148223638534546, + "task_id": 274, + "token_count": 27, + "top1_agreement": 0.9629629850387573 + }, + { + "candidate_nll": 1.3185395002365112, + "code_tokens": 97, + "cvar95_kl": 0.19633172452449799, + "delta_nll": 0.06110882759094238, + "max_kl": 0.23900717496871948, + "mean_kl": 0.03221386298537254, + "reference_nll": 1.2574306726455688, + "task_id": 275, + "token_count": 97, + "top1_agreement": 0.938144326210022 + }, + { + "candidate_nll": 0.7912057042121887, + "code_tokens": 24, + "cvar95_kl": 0.04637271165847778, + "delta_nll": -0.011026740074157715, + "max_kl": 0.054961204528808594, + "mean_kl": 0.008965600281953812, + "reference_nll": 0.8022324442863464, + "task_id": 276, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3088558912277222, + "code_tokens": 30, + "cvar95_kl": 0.20467102527618408, + "delta_nll": 0.04370725154876709, + "max_kl": 0.25056707859039307, + "mean_kl": 0.01849093846976757, + "reference_nll": 1.265148639678955, + "task_id": 277, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 1.4591341018676758, + "code_tokens": 38, + "cvar95_kl": 0.3534621596336365, + "delta_nll": 0.026064276695251465, + "max_kl": 0.5330673456192017, + "mean_kl": 0.034155089408159256, + "reference_nll": 1.4330698251724243, + "task_id": 278, + "token_count": 38, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 1.4896142482757568, + "code_tokens": 21, + "cvar95_kl": 0.08165368437767029, + "delta_nll": 0.0018352270126342773, + "max_kl": 0.09619525820016861, + "mean_kl": 0.021432848647236824, + "reference_nll": 1.4877790212631226, + "task_id": 279, + "token_count": 21, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 0.5331955552101135, + "code_tokens": 65, + "cvar95_kl": 0.10709179937839508, + "delta_nll": -0.0037662386894226074, + "max_kl": 0.18983428180217743, + "mean_kl": 0.013725055381655693, + "reference_nll": 0.5369617938995361, + "task_id": 280, + "token_count": 65, + "top1_agreement": 0.9692307710647583 + }, + { + "candidate_nll": 0.8133689761161804, + "code_tokens": 27, + "cvar95_kl": 0.1837712675333023, + "delta_nll": 0.028842687606811523, + "max_kl": 0.29789695143699646, + "mean_kl": 0.01852179318666458, + "reference_nll": 0.7845262885093689, + "task_id": 281, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7614670991897583, + "code_tokens": 35, + "cvar95_kl": 0.04900834709405899, + "delta_nll": 0.037651002407073975, + "max_kl": 0.05431350693106651, + "mean_kl": 0.010137069970369339, + "reference_nll": 0.7238160967826843, + "task_id": 282, + "token_count": 35, + "top1_agreement": 0.9714285731315613 + }, + { + "candidate_nll": 0.9303459525108337, + "code_tokens": 75, + "cvar95_kl": 0.1064162477850914, + "delta_nll": 0.005821824073791504, + "max_kl": 0.1154273971915245, + "mean_kl": 0.017466096207499504, + "reference_nll": 0.9245241284370422, + "task_id": 283, + "token_count": 75, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 1.6510555744171143, + "code_tokens": 25, + "cvar95_kl": 0.03869602084159851, + "delta_nll": 0.00932002067565918, + "max_kl": 0.042307764291763306, + "mean_kl": 0.009754207916557789, + "reference_nll": 1.641735553741455, + "task_id": 284, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9663410186767578, + "code_tokens": 52, + "cvar95_kl": 0.07066798955202103, + "delta_nll": 0.04235208034515381, + "max_kl": 0.07250350713729858, + "mean_kl": 0.013021860271692276, + "reference_nll": 0.923988938331604, + "task_id": 285, + "token_count": 52, + "top1_agreement": 0.9807692170143127 + }, + { + "candidate_nll": 0.6124125123023987, + "code_tokens": 110, + "cvar95_kl": 0.2973022758960724, + "delta_nll": 0.0017524361610412598, + "max_kl": 0.4677589535713196, + "mean_kl": 0.03245244175195694, + "reference_nll": 0.6106600761413574, + "task_id": 286, + "token_count": 110, + "top1_agreement": 0.9727272987365723 + }, + { + "candidate_nll": 1.2317779064178467, + "code_tokens": 25, + "cvar95_kl": 0.14282794296741486, + "delta_nll": -0.0010440349578857422, + "max_kl": 0.2232845276594162, + "mean_kl": 0.024314487352967262, + "reference_nll": 1.2328219413757324, + "task_id": 287, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.0290435552597046, + "code_tokens": 56, + "cvar95_kl": 0.1888788789510727, + "delta_nll": 0.01205754280090332, + "max_kl": 0.2329382598400116, + "mean_kl": 0.030549170449376106, + "reference_nll": 1.0169860124588013, + "task_id": 288, + "token_count": 56, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 1.5963526964187622, + "code_tokens": 108, + "cvar95_kl": 0.2765786051750183, + "delta_nll": 0.04250442981719971, + "max_kl": 0.8728927373886108, + "mean_kl": 0.040780480951070786, + "reference_nll": 1.5538482666015625, + "task_id": 289, + "token_count": 108, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.3646117448806763, + "code_tokens": 47, + "cvar95_kl": 0.059291958808898926, + "delta_nll": -0.007128119468688965, + "max_kl": 0.06999614089727402, + "mean_kl": 0.012084024958312511, + "reference_nll": 1.3717398643493652, + "task_id": 290, + "token_count": 47, + "top1_agreement": 0.978723406791687 + }, + { + "candidate_nll": 1.0170948505401611, + "code_tokens": 102, + "cvar95_kl": 0.2220470905303955, + "delta_nll": 0.018686234951019287, + "max_kl": 0.37844234704971313, + "mean_kl": 0.024354606866836548, + "reference_nll": 0.9984086155891418, + "task_id": 291, + "token_count": 102, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 1.6943740844726562, + "code_tokens": 18, + "cvar95_kl": 0.08930838108062744, + "delta_nll": 0.06874847412109375, + "max_kl": 0.08930838108062744, + "mean_kl": 0.009738761931657791, + "reference_nll": 1.6256256103515625, + "task_id": 292, + "token_count": 18, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0609642267227173, + "code_tokens": 28, + "cvar95_kl": 0.03307639807462692, + "delta_nll": -0.038699984550476074, + "max_kl": 0.03731895238161087, + "mean_kl": 0.008647533133625984, + "reference_nll": 1.0996642112731934, + "task_id": 293, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.55195152759552, + "code_tokens": 30, + "cvar95_kl": 0.09268026053905487, + "delta_nll": 0.008550643920898438, + "max_kl": 0.10808173567056656, + "mean_kl": 0.013489676639437675, + "reference_nll": 1.5434008836746216, + "task_id": 294, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 0.4739150106906891, + "code_tokens": 48, + "cvar95_kl": 0.11785370856523514, + "delta_nll": -0.0012019872665405273, + "max_kl": 0.14424589276313782, + "mean_kl": 0.01528071891516447, + "reference_nll": 0.4751169979572296, + "task_id": 295, + "token_count": 48, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.5466382503509521, + "code_tokens": 59, + "cvar95_kl": 0.03366344794631004, + "delta_nll": 0.009623229503631592, + "max_kl": 0.03768809512257576, + "mean_kl": 0.00598830496892333, + "reference_nll": 0.5370150208473206, + "task_id": 296, + "token_count": 59, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0829696655273438, + "code_tokens": 102, + "cvar95_kl": 0.18562333285808563, + "delta_nll": 0.04077577590942383, + "max_kl": 0.24863030016422272, + "mean_kl": 0.027431625872850418, + "reference_nll": 1.04219388961792, + "task_id": 297, + "token_count": 102, + "top1_agreement": 0.9607843160629272 + }, + { + "candidate_nll": 0.7265756726264954, + "code_tokens": 36, + "cvar95_kl": 0.12820783257484436, + "delta_nll": -0.013789951801300049, + "max_kl": 0.2022916078567505, + "mean_kl": 0.017081240192055702, + "reference_nll": 0.7403656244277954, + "task_id": 298, + "token_count": 36, + "top1_agreement": 0.9722222089767456 + }, + { + "candidate_nll": 0.9223048686981201, + "code_tokens": 51, + "cvar95_kl": 0.1542099416255951, + "delta_nll": 0.033373355865478516, + "max_kl": 0.2752964496612549, + "mean_kl": 0.01987362466752529, + "reference_nll": 0.8889315128326416, + "task_id": 299, + "token_count": 51, + "top1_agreement": 0.9803921580314636 + }, + { + "candidate_nll": 1.2807544469833374, + "code_tokens": 62, + "cvar95_kl": 0.1655554473400116, + "delta_nll": 0.03507578372955322, + "max_kl": 0.30606362223625183, + "mean_kl": 0.026282545179128647, + "reference_nll": 1.2456786632537842, + "task_id": 300, + "token_count": 62, + "top1_agreement": 0.9516128897666931 + }, + { + "candidate_nll": 0.5904662013053894, + "code_tokens": 39, + "cvar95_kl": 0.2843458652496338, + "delta_nll": 0.014184117317199707, + "max_kl": 0.42672398686408997, + "mean_kl": 0.03267572820186615, + "reference_nll": 0.5762820839881897, + "task_id": 301, + "token_count": 39, + "top1_agreement": 0.9487179517745972 + }, + { + "candidate_nll": 0.7777959108352661, + "code_tokens": 76, + "cvar95_kl": 0.29079481959342957, + "delta_nll": 0.03811156749725342, + "max_kl": 0.4786742031574249, + "mean_kl": 0.03516437113285065, + "reference_nll": 0.7396843433380127, + "task_id": 302, + "token_count": 76, + "top1_agreement": 0.9210526347160339 + }, + { + "candidate_nll": 1.5160623788833618, + "code_tokens": 58, + "cvar95_kl": 0.28883013129234314, + "delta_nll": 0.011411309242248535, + "max_kl": 0.48641422390937805, + "mean_kl": 0.03982424736022949, + "reference_nll": 1.5046510696411133, + "task_id": 303, + "token_count": 58, + "top1_agreement": 0.9482758641242981 + }, + { + "candidate_nll": 1.1668747663497925, + "code_tokens": 93, + "cvar95_kl": 0.3310495913028717, + "delta_nll": 0.03947949409484863, + "max_kl": 0.4407699704170227, + "mean_kl": 0.03974595665931702, + "reference_nll": 1.1273952722549438, + "task_id": 304, + "token_count": 93, + "top1_agreement": 0.9139785170555115 + }, + { + "candidate_nll": 1.3475698232650757, + "code_tokens": 45, + "cvar95_kl": 0.09734532237052917, + "delta_nll": -0.007675647735595703, + "max_kl": 0.16516532003879547, + "mean_kl": 0.016233138740062714, + "reference_nll": 1.3552454710006714, + "task_id": 305, + "token_count": 45, + "top1_agreement": 0.9555555582046509 + }, + { + "candidate_nll": 0.5743474960327148, + "code_tokens": 209, + "cvar95_kl": 0.45288923382759094, + "delta_nll": 0.017372071743011475, + "max_kl": 1.3597054481506348, + "mean_kl": 0.04129744693636894, + "reference_nll": 0.5569754242897034, + "task_id": 306, + "token_count": 209, + "top1_agreement": 0.9330143332481384 + }, + { + "candidate_nll": 1.0209332704544067, + "code_tokens": 46, + "cvar95_kl": 0.11891517043113708, + "delta_nll": 0.052465617656707764, + "max_kl": 0.18307971954345703, + "mean_kl": 0.01757819764316082, + "reference_nll": 0.968467652797699, + "task_id": 307, + "token_count": 46, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7174152135848999, + "code_tokens": 39, + "cvar95_kl": 0.09613239020109177, + "delta_nll": -0.0033717751502990723, + "max_kl": 0.12555499374866486, + "mean_kl": 0.012369691394269466, + "reference_nll": 0.720786988735199, + "task_id": 308, + "token_count": 39, + "top1_agreement": 0.9743589758872986 + }, + { + "candidate_nll": 0.9477680325508118, + "code_tokens": 25, + "cvar95_kl": 0.012415453791618347, + "delta_nll": 0.006072640419006348, + "max_kl": 0.012908902019262314, + "mean_kl": 0.002957127057015896, + "reference_nll": 0.9416953921318054, + "task_id": 309, + "token_count": 25, + "top1_agreement": 0.9200000166893005 + }, + { + "candidate_nll": 1.1396607160568237, + "code_tokens": 28, + "cvar95_kl": 0.033272698521614075, + "delta_nll": -0.035535573959350586, + "max_kl": 0.038835227489471436, + "mean_kl": 0.009617792442440987, + "reference_nll": 1.1751962900161743, + "task_id": 310, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2691900730133057, + "code_tokens": 84, + "cvar95_kl": 0.19050855934619904, + "delta_nll": 0.015267491340637207, + "max_kl": 0.3078611493110657, + "mean_kl": 0.03242058306932449, + "reference_nll": 1.2539225816726685, + "task_id": 311, + "token_count": 84, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 0.8255850672721863, + "code_tokens": 34, + "cvar95_kl": 0.07175995409488678, + "delta_nll": 0.01330476999282837, + "max_kl": 0.09084892272949219, + "mean_kl": 0.013552838005125523, + "reference_nll": 0.8122802972793579, + "task_id": 312, + "token_count": 34, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3003734350204468, + "code_tokens": 28, + "cvar95_kl": 0.09452224522829056, + "delta_nll": 0.0453263521194458, + "max_kl": 0.15153826773166656, + "mean_kl": 0.01297932118177414, + "reference_nll": 1.255047082901001, + "task_id": 313, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 1.1329410076141357, + "code_tokens": 92, + "cvar95_kl": 0.3766676187515259, + "delta_nll": 0.03409552574157715, + "max_kl": 0.6361127495765686, + "mean_kl": 0.04402878135442734, + "reference_nll": 1.0988454818725586, + "task_id": 314, + "token_count": 92, + "top1_agreement": 0.945652186870575 + }, + { + "candidate_nll": 0.8487433195114136, + "code_tokens": 185, + "cvar95_kl": 0.5008316040039062, + "delta_nll": 0.11862868070602417, + "max_kl": 1.5652185678482056, + "mean_kl": 0.053893495351076126, + "reference_nll": 0.7301146388053894, + "task_id": 315, + "token_count": 185, + "top1_agreement": 0.9459459185600281 + }, + { + "candidate_nll": 0.36951518058776855, + "code_tokens": 106, + "cvar95_kl": 0.13586309552192688, + "delta_nll": 0.01652061939239502, + "max_kl": 0.31775057315826416, + "mean_kl": 0.013613050803542137, + "reference_nll": 0.35299456119537354, + "task_id": 316, + "token_count": 106, + "top1_agreement": 0.9811320900917053 + }, + { + "candidate_nll": 1.5448065996170044, + "code_tokens": 64, + "cvar95_kl": 0.22003993391990662, + "delta_nll": 0.002252340316772461, + "max_kl": 0.3298512399196625, + "mean_kl": 0.03392772004008293, + "reference_nll": 1.542554259300232, + "task_id": 317, + "token_count": 64, + "top1_agreement": 0.921875 + }, + { + "candidate_nll": 1.1352624893188477, + "code_tokens": 74, + "cvar95_kl": 0.10974113643169403, + "delta_nll": 0.024478435516357422, + "max_kl": 0.23238250613212585, + "mean_kl": 0.0188768170773983, + "reference_nll": 1.1107840538024902, + "task_id": 318, + "token_count": 74, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9985869526863098, + "code_tokens": 27, + "cvar95_kl": 0.07318924367427826, + "delta_nll": 0.011183083057403564, + "max_kl": 0.08791238814592361, + "mean_kl": 0.01723477989435196, + "reference_nll": 0.9874038696289062, + "task_id": 319, + "token_count": 27, + "top1_agreement": 0.8888888955116272 + }, + { + "candidate_nll": 0.6278055906295776, + "code_tokens": 72, + "cvar95_kl": 0.19188976287841797, + "delta_nll": 0.003967702388763428, + "max_kl": 0.2517572343349457, + "mean_kl": 0.021268753334879875, + "reference_nll": 0.6238378882408142, + "task_id": 320, + "token_count": 72, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.9173759818077087, + "code_tokens": 62, + "cvar95_kl": 0.2418258935213089, + "delta_nll": -0.015926361083984375, + "max_kl": 0.5483251214027405, + "mean_kl": 0.028020713478326797, + "reference_nll": 0.9333023428916931, + "task_id": 321, + "token_count": 62, + "top1_agreement": 0.9516128897666931 + }, + { + "candidate_nll": 0.7490004897117615, + "code_tokens": 42, + "cvar95_kl": 0.10039535909891129, + "delta_nll": 0.05083411931991577, + "max_kl": 0.13054072856903076, + "mean_kl": 0.01471069734543562, + "reference_nll": 0.6981663703918457, + "task_id": 322, + "token_count": 42, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.893073320388794, + "code_tokens": 238, + "cvar95_kl": 0.2776794731616974, + "delta_nll": 0.005740463733673096, + "max_kl": 0.4170212745666504, + "mean_kl": 0.037741728127002716, + "reference_nll": 0.8873328566551208, + "task_id": 323, + "token_count": 238, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 0.9143219590187073, + "code_tokens": 68, + "cvar95_kl": 0.08763806521892548, + "delta_nll": -0.00018978118896484375, + "max_kl": 0.1460747867822647, + "mean_kl": 0.011641522869467735, + "reference_nll": 0.9145117402076721, + "task_id": 324, + "token_count": 68, + "top1_agreement": 0.9852941036224365 + }, + { + "candidate_nll": 1.0255357027053833, + "code_tokens": 83, + "cvar95_kl": 0.24782665073871613, + "delta_nll": 0.04138582944869995, + "max_kl": 0.31994763016700745, + "mean_kl": 0.0354788564145565, + "reference_nll": 0.9841498732566833, + "task_id": 325, + "token_count": 83, + "top1_agreement": 0.9277108311653137 + }, + { + "candidate_nll": 1.239659070968628, + "code_tokens": 64, + "cvar95_kl": 0.043962374329566956, + "delta_nll": -0.024108409881591797, + "max_kl": 0.0688987523317337, + "mean_kl": 0.009426138363778591, + "reference_nll": 1.2637674808502197, + "task_id": 326, + "token_count": 64, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6621934771537781, + "code_tokens": 37, + "cvar95_kl": 0.02807730808854103, + "delta_nll": 0.005393564701080322, + "max_kl": 0.030303604900836945, + "mean_kl": 0.006095089018344879, + "reference_nll": 0.6567999124526978, + "task_id": 327, + "token_count": 37, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.5153355598449707, + "code_tokens": 27, + "cvar95_kl": 0.04862125962972641, + "delta_nll": 0.0054084062576293945, + "max_kl": 0.07166334986686707, + "mean_kl": 0.009985399432480335, + "reference_nll": 1.5099271535873413, + "task_id": 328, + "token_count": 27, + "top1_agreement": 0.9629629850387573 + }, + { + "candidate_nll": 0.9137356877326965, + "code_tokens": 40, + "cvar95_kl": 0.015970150008797646, + "delta_nll": 0.02201932668685913, + "max_kl": 0.01691940613090992, + "mean_kl": 0.0036457316018640995, + "reference_nll": 0.8917163610458374, + "task_id": 329, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0224825143814087, + "code_tokens": 28, + "cvar95_kl": 0.061375755816698074, + "delta_nll": 0.02470982074737549, + "max_kl": 0.06143457442522049, + "mean_kl": 0.01531421672552824, + "reference_nll": 0.9977726936340332, + "task_id": 330, + "token_count": 28, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 0.8743178844451904, + "code_tokens": 59, + "cvar95_kl": 0.0897982120513916, + "delta_nll": 0.01968824863433838, + "max_kl": 0.13744278252124786, + "mean_kl": 0.014186255633831024, + "reference_nll": 0.854629635810852, + "task_id": 331, + "token_count": 59, + "top1_agreement": 0.9661017060279846 + }, + { + "candidate_nll": 0.5966822504997253, + "code_tokens": 57, + "cvar95_kl": 0.10842012614011765, + "delta_nll": 0.026682138442993164, + "max_kl": 0.16882257163524628, + "mean_kl": 0.015678251162171364, + "reference_nll": 0.5700001120567322, + "task_id": 332, + "token_count": 57, + "top1_agreement": 0.9649122953414917 + }, + { + "candidate_nll": 1.4330857992172241, + "code_tokens": 25, + "cvar95_kl": 0.04315762221813202, + "delta_nll": 0.04914569854736328, + "max_kl": 0.05357225611805916, + "mean_kl": 0.009501941502094269, + "reference_nll": 1.3839401006698608, + "task_id": 333, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7824879288673401, + "code_tokens": 48, + "cvar95_kl": 0.245189368724823, + "delta_nll": 0.03533560037612915, + "max_kl": 0.45635026693344116, + "mean_kl": 0.021671727299690247, + "reference_nll": 0.7471523284912109, + "task_id": 334, + "token_count": 48, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 0.8493776917457581, + "code_tokens": 35, + "cvar95_kl": 0.03651416301727295, + "delta_nll": -0.012398123741149902, + "max_kl": 0.038169581443071365, + "mean_kl": 0.00602731853723526, + "reference_nll": 0.861775815486908, + "task_id": 335, + "token_count": 35, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6693041920661926, + "code_tokens": 30, + "cvar95_kl": 0.05866566300392151, + "delta_nll": 0.0245969295501709, + "max_kl": 0.061297912150621414, + "mean_kl": 0.00878565851598978, + "reference_nll": 0.6447072625160217, + "task_id": 336, + "token_count": 30, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 1.1382536888122559, + "code_tokens": 49, + "cvar95_kl": 0.03998522832989693, + "delta_nll": 0.015176177024841309, + "max_kl": 0.043124355375766754, + "mean_kl": 0.008713734336197376, + "reference_nll": 1.1230775117874146, + "task_id": 337, + "token_count": 49, + "top1_agreement": 0.9387755393981934 + }, + { + "candidate_nll": 1.0246520042419434, + "code_tokens": 98, + "cvar95_kl": 0.24515771865844727, + "delta_nll": 0.00843513011932373, + "max_kl": 0.3983985185623169, + "mean_kl": 0.022786863148212433, + "reference_nll": 1.0162168741226196, + "task_id": 338, + "token_count": 98, + "top1_agreement": 0.9795918464660645 + }, + { + "candidate_nll": 1.3699432611465454, + "code_tokens": 25, + "cvar95_kl": 0.04016021266579628, + "delta_nll": -0.010863065719604492, + "max_kl": 0.04127763956785202, + "mean_kl": 0.010038338601589203, + "reference_nll": 1.38080632686615, + "task_id": 339, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.7912485599517822, + "code_tokens": 27, + "cvar95_kl": 0.10825206339359283, + "delta_nll": 0.01352536678314209, + "max_kl": 0.18841427564620972, + "mean_kl": 0.013693369925022125, + "reference_nll": 0.7777231931686401, + "task_id": 340, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0430772304534912, + "code_tokens": 20, + "cvar95_kl": 0.024303613230586052, + "delta_nll": -0.0020074844360351562, + "max_kl": 0.024303613230586052, + "mean_kl": 0.006495853420346975, + "reference_nll": 1.0450847148895264, + "task_id": 341, + "token_count": 20, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.765764594078064, + "code_tokens": 253, + "cvar95_kl": 0.5500823259353638, + "delta_nll": 0.07309460639953613, + "max_kl": 1.1106146574020386, + "mean_kl": 0.06368852406740189, + "reference_nll": 0.6926699876785278, + "task_id": 342, + "token_count": 253, + "top1_agreement": 0.9209486246109009 + }, + { + "candidate_nll": 0.8812485337257385, + "code_tokens": 55, + "cvar95_kl": 0.14085529744625092, + "delta_nll": 0.05212825536727905, + "max_kl": 0.1483965367078781, + "mean_kl": 0.01910315454006195, + "reference_nll": 0.8291202783584595, + "task_id": 343, + "token_count": 55, + "top1_agreement": 0.9454545378684998 + }, + { + "candidate_nll": 1.1769118309020996, + "code_tokens": 31, + "cvar95_kl": 0.07913221418857574, + "delta_nll": -0.0295865535736084, + "max_kl": 0.09416764229536057, + "mean_kl": 0.015607703477144241, + "reference_nll": 1.206498384475708, + "task_id": 344, + "token_count": 31, + "top1_agreement": 0.9677419066429138 + }, + { + "candidate_nll": 0.7256224155426025, + "code_tokens": 35, + "cvar95_kl": 0.16967622935771942, + "delta_nll": 0.02137625217437744, + "max_kl": 0.2402161955833435, + "mean_kl": 0.015219705179333687, + "reference_nll": 0.7042461633682251, + "task_id": 345, + "token_count": 35, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7883664965629578, + "code_tokens": 61, + "cvar95_kl": 0.10833339393138885, + "delta_nll": -0.014501988887786865, + "max_kl": 0.21474015712738037, + "mean_kl": 0.014987628906965256, + "reference_nll": 0.8028684854507446, + "task_id": 346, + "token_count": 61, + "top1_agreement": 0.9672130942344666 + }, + { + "candidate_nll": 1.0584540367126465, + "code_tokens": 55, + "cvar95_kl": 0.37269127368927, + "delta_nll": 0.10373646020889282, + "max_kl": 0.41993966698646545, + "mean_kl": 0.04622839018702507, + "reference_nll": 0.9547175765037537, + "task_id": 347, + "token_count": 55, + "top1_agreement": 0.9090909361839294 + }, + { + "candidate_nll": 1.5620064735412598, + "code_tokens": 108, + "cvar95_kl": 0.32694026827812195, + "delta_nll": 0.055039286613464355, + "max_kl": 0.6413977742195129, + "mean_kl": 0.0443376861512661, + "reference_nll": 1.5069671869277954, + "task_id": 348, + "token_count": 108, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.2950985431671143, + "code_tokens": 58, + "cvar95_kl": 0.0453801266849041, + "delta_nll": 0.007792472839355469, + "max_kl": 0.05167590081691742, + "mean_kl": 0.01025429554283619, + "reference_nll": 1.2873060703277588, + "task_id": 349, + "token_count": 58, + "top1_agreement": 0.9137930870056152 + }, + { + "candidate_nll": 0.8452746272087097, + "code_tokens": 91, + "cvar95_kl": 0.19592022895812988, + "delta_nll": 0.01781940460205078, + "max_kl": 0.33996641635894775, + "mean_kl": 0.025951500982046127, + "reference_nll": 0.8274552226066589, + "task_id": 350, + "token_count": 91, + "top1_agreement": 0.9450549483299255 + }, + { + "candidate_nll": 0.7650046348571777, + "code_tokens": 105, + "cvar95_kl": 0.12187197804450989, + "delta_nll": 0.0012651681900024414, + "max_kl": 0.1969258189201355, + "mean_kl": 0.02312031202018261, + "reference_nll": 0.7637394666671753, + "task_id": 351, + "token_count": 105, + "top1_agreement": 0.961904764175415 + }, + { + "candidate_nll": 0.6635512709617615, + "code_tokens": 49, + "cvar95_kl": 0.06867866963148117, + "delta_nll": 0.016298651695251465, + "max_kl": 0.08183785527944565, + "mean_kl": 0.01197011023759842, + "reference_nll": 0.64725261926651, + "task_id": 352, + "token_count": 49, + "top1_agreement": 0.9795918464660645 + }, + { + "candidate_nll": 1.449521541595459, + "code_tokens": 27, + "cvar95_kl": 0.07368543744087219, + "delta_nll": -0.012641429901123047, + "max_kl": 0.09271962940692902, + "mean_kl": 0.011212228797376156, + "reference_nll": 1.462162971496582, + "task_id": 353, + "token_count": 27, + "top1_agreement": 0.9629629850387573 + }, + { + "candidate_nll": 1.0336425304412842, + "code_tokens": 25, + "cvar95_kl": 0.0426093190908432, + "delta_nll": -0.006734132766723633, + "max_kl": 0.04410243406891823, + "mean_kl": 0.007871788926422596, + "reference_nll": 1.0403766632080078, + "task_id": 354, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2883840799331665, + "code_tokens": 98, + "cvar95_kl": 0.21403411030769348, + "delta_nll": -0.006670355796813965, + "max_kl": 0.2629324495792389, + "mean_kl": 0.029719708487391472, + "reference_nll": 1.2950544357299805, + "task_id": 355, + "token_count": 98, + "top1_agreement": 0.9387755393981934 + }, + { + "candidate_nll": 0.6873620748519897, + "code_tokens": 23, + "cvar95_kl": 0.16906067728996277, + "delta_nll": 0.03758037090301514, + "max_kl": 0.1847136914730072, + "mean_kl": 0.020032957196235657, + "reference_nll": 0.6497817039489746, + "task_id": 356, + "token_count": 23, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.9132437705993652, + "code_tokens": 31, + "cvar95_kl": 0.30116376280784607, + "delta_nll": 0.04787003993988037, + "max_kl": 0.5450748205184937, + "mean_kl": 0.02548260986804962, + "reference_nll": 1.8653737306594849, + "task_id": 357, + "token_count": 31, + "top1_agreement": 0.9677419066429138 + }, + { + "candidate_nll": 0.7662094235420227, + "code_tokens": 36, + "cvar95_kl": 0.09610622376203537, + "delta_nll": 0.018116295337677002, + "max_kl": 0.10044874250888824, + "mean_kl": 0.013398345559835434, + "reference_nll": 0.7480931282043457, + "task_id": 358, + "token_count": 36, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.493288278579712, + "code_tokens": 38, + "cvar95_kl": 0.06188933923840523, + "delta_nll": 0.059610724449157715, + "max_kl": 0.07984708994626999, + "mean_kl": 0.014115938916802406, + "reference_nll": 1.4336775541305542, + "task_id": 359, + "token_count": 38, + "top1_agreement": 0.9736841917037964 + }, + { + "candidate_nll": 1.6256788969039917, + "code_tokens": 25, + "cvar95_kl": 0.13042597472667694, + "delta_nll": 0.01958906650543213, + "max_kl": 0.1417156457901001, + "mean_kl": 0.025679897516965866, + "reference_nll": 1.6060898303985596, + "task_id": 360, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8828887939453125, + "code_tokens": 26, + "cvar95_kl": 0.04449836537241936, + "delta_nll": 0.003439664840698242, + "max_kl": 0.060522399842739105, + "mean_kl": 0.009025627747178078, + "reference_nll": 0.8794491291046143, + "task_id": 361, + "token_count": 26, + "top1_agreement": 0.9615384340286255 + }, + { + "candidate_nll": 0.7649604082107544, + "code_tokens": 62, + "cvar95_kl": 0.08534520864486694, + "delta_nll": 0.0225183367729187, + "max_kl": 0.15457487106323242, + "mean_kl": 0.012285039760172367, + "reference_nll": 0.7424420714378357, + "task_id": 362, + "token_count": 62, + "top1_agreement": 0.9677419066429138 + }, + { + "candidate_nll": 1.7195329666137695, + "code_tokens": 36, + "cvar95_kl": 0.08205614984035492, + "delta_nll": -0.018986105918884277, + "max_kl": 0.09916187822818756, + "mean_kl": 0.015349745750427246, + "reference_nll": 1.7385190725326538, + "task_id": 363, + "token_count": 36, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 1.353416919708252, + "code_tokens": 119, + "cvar95_kl": 0.5580270886421204, + "delta_nll": 0.022827982902526855, + "max_kl": 1.9180057048797607, + "mean_kl": 0.04773281142115593, + "reference_nll": 1.330588936805725, + "task_id": 364, + "token_count": 119, + "top1_agreement": 0.9327731132507324 + }, + { + "candidate_nll": 0.2993519604206085, + "code_tokens": 37, + "cvar95_kl": 0.039086200296878815, + "delta_nll": -0.013355612754821777, + "max_kl": 0.039905060082674026, + "mean_kl": 0.004022568464279175, + "reference_nll": 0.3127075731754303, + "task_id": 365, + "token_count": 37, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.791767418384552, + "code_tokens": 28, + "cvar95_kl": 0.03727693855762482, + "delta_nll": 0.009033918380737305, + "max_kl": 0.05657028406858444, + "mean_kl": 0.006294121500104666, + "reference_nll": 0.7827335000038147, + "task_id": 366, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.589308500289917, + "code_tokens": 143, + "cvar95_kl": 0.14229589700698853, + "delta_nll": 0.022961676120758057, + "max_kl": 0.21147610247135162, + "mean_kl": 0.019217727705836296, + "reference_nll": 0.5663468241691589, + "task_id": 367, + "token_count": 143, + "top1_agreement": 0.9580419659614563 + }, + { + "candidate_nll": 1.759843349456787, + "code_tokens": 29, + "cvar95_kl": 0.10141370445489883, + "delta_nll": -0.022462964057922363, + "max_kl": 0.10386858880519867, + "mean_kl": 0.019215067848563194, + "reference_nll": 1.7823063135147095, + "task_id": 368, + "token_count": 29, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 1.0677158832550049, + "code_tokens": 28, + "cvar95_kl": 0.05450204014778137, + "delta_nll": -0.004772543907165527, + "max_kl": 0.06476400792598724, + "mean_kl": 0.012451707385480404, + "reference_nll": 1.0724884271621704, + "task_id": 369, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 1.279981255531311, + "code_tokens": 30, + "cvar95_kl": 0.06510163843631744, + "delta_nll": 0.015943527221679688, + "max_kl": 0.09215538948774338, + "mean_kl": 0.01294393464922905, + "reference_nll": 1.2640377283096313, + "task_id": 370, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 0.5000909566879272, + "code_tokens": 85, + "cvar95_kl": 0.27216628193855286, + "delta_nll": 0.07828086614608765, + "max_kl": 0.5431474447250366, + "mean_kl": 0.027180511504411697, + "reference_nll": 0.4218100905418396, + "task_id": 371, + "token_count": 85, + "top1_agreement": 0.9529411792755127 + }, + { + "candidate_nll": 1.0120993852615356, + "code_tokens": 44, + "cvar95_kl": 0.30642008781433105, + "delta_nll": 0.034855782985687256, + "max_kl": 0.4907491207122803, + "mean_kl": 0.030185284093022346, + "reference_nll": 0.9772436022758484, + "task_id": 372, + "token_count": 44, + "top1_agreement": 0.9545454382896423 + }, + { + "candidate_nll": 0.8296409845352173, + "code_tokens": 19, + "cvar95_kl": 0.01662239246070385, + "delta_nll": 0.025538742542266846, + "max_kl": 0.01662239246070385, + "mean_kl": 0.0036729678977280855, + "reference_nll": 0.8041022419929504, + "task_id": 373, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7459011077880859, + "code_tokens": 113, + "cvar95_kl": 0.26824596524238586, + "delta_nll": 0.008478224277496338, + "max_kl": 0.4233720302581787, + "mean_kl": 0.032212935388088226, + "reference_nll": 0.7374228835105896, + "task_id": 374, + "token_count": 113, + "top1_agreement": 0.982300877571106 + }, + { + "candidate_nll": 1.5034258365631104, + "code_tokens": 40, + "cvar95_kl": 0.12359834462404251, + "delta_nll": 0.028231382369995117, + "max_kl": 0.13159124553203583, + "mean_kl": 0.025472169741988182, + "reference_nll": 1.4751944541931152, + "task_id": 375, + "token_count": 40, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 1.6152609586715698, + "code_tokens": 51, + "cvar95_kl": 0.2168998271226883, + "delta_nll": 0.008600950241088867, + "max_kl": 0.24238993227481842, + "mean_kl": 0.02901824750006199, + "reference_nll": 1.606660008430481, + "task_id": 376, + "token_count": 51, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 1.6528877019882202, + "code_tokens": 55, + "cvar95_kl": 0.2462296038866043, + "delta_nll": 0.07185518741607666, + "max_kl": 0.33251094818115234, + "mean_kl": 0.025987640023231506, + "reference_nll": 1.5810325145721436, + "task_id": 377, + "token_count": 55, + "top1_agreement": 0.9272727370262146 + }, + { + "candidate_nll": 1.1624196767807007, + "code_tokens": 27, + "cvar95_kl": 0.0669163316488266, + "delta_nll": 0.03235137462615967, + "max_kl": 0.09114660322666168, + "mean_kl": 0.01090506836771965, + "reference_nll": 1.130068302154541, + "task_id": 378, + "token_count": 27, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8933786153793335, + "code_tokens": 32, + "cvar95_kl": 0.07897178828716278, + "delta_nll": 0.05357241630554199, + "max_kl": 0.09632521867752075, + "mean_kl": 0.01158739160746336, + "reference_nll": 0.8398061990737915, + "task_id": 379, + "token_count": 32, + "top1_agreement": 0.96875 + }, + { + "candidate_nll": 0.6535678505897522, + "code_tokens": 64, + "cvar95_kl": 0.05663813278079033, + "delta_nll": 0.0010297894477844238, + "max_kl": 0.07583219558000565, + "mean_kl": 0.011218101717531681, + "reference_nll": 0.6525380611419678, + "task_id": 380, + "token_count": 64, + "top1_agreement": 0.96875 + }, + { + "candidate_nll": 1.064474105834961, + "code_tokens": 35, + "cvar95_kl": 0.04539664834737778, + "delta_nll": 0.014819860458374023, + "max_kl": 0.04852963238954544, + "mean_kl": 0.007564667146652937, + "reference_nll": 1.049654245376587, + "task_id": 381, + "token_count": 35, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5031339526176453, + "code_tokens": 152, + "cvar95_kl": 0.28159278631210327, + "delta_nll": 0.05128636956214905, + "max_kl": 0.4535447061061859, + "mean_kl": 0.03207201883196831, + "reference_nll": 0.4518475830554962, + "task_id": 382, + "token_count": 152, + "top1_agreement": 0.9276315569877625 + }, + { + "candidate_nll": 1.2915598154067993, + "code_tokens": 77, + "cvar95_kl": 0.07869604974985123, + "delta_nll": 0.030225515365600586, + "max_kl": 0.10668745636940002, + "mean_kl": 0.013667777180671692, + "reference_nll": 1.2613343000411987, + "task_id": 383, + "token_count": 77, + "top1_agreement": 0.9610389471054077 + }, + { + "candidate_nll": 0.6013168692588806, + "code_tokens": 77, + "cvar95_kl": 0.20085495710372925, + "delta_nll": 0.03303581476211548, + "max_kl": 0.34251508116722107, + "mean_kl": 0.025617128238081932, + "reference_nll": 0.5682810544967651, + "task_id": 384, + "token_count": 77, + "top1_agreement": 0.9870129823684692 + }, + { + "candidate_nll": 0.5524380207061768, + "code_tokens": 71, + "cvar95_kl": 0.05695692077279091, + "delta_nll": -0.009650766849517822, + "max_kl": 0.0851060152053833, + "mean_kl": 0.008187651634216309, + "reference_nll": 0.5620887875556946, + "task_id": 385, + "token_count": 71, + "top1_agreement": 0.98591548204422 + }, + { + "candidate_nll": 0.8625392317771912, + "code_tokens": 112, + "cvar95_kl": 0.17196093499660492, + "delta_nll": -0.015840113162994385, + "max_kl": 0.30524876713752747, + "mean_kl": 0.022420452907681465, + "reference_nll": 0.8783793449401855, + "task_id": 386, + "token_count": 112, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 0.7487248778343201, + "code_tokens": 111, + "cvar95_kl": 0.18515963852405548, + "delta_nll": 0.028821170330047607, + "max_kl": 0.29252779483795166, + "mean_kl": 0.019462736323475838, + "reference_nll": 0.7199037075042725, + "task_id": 387, + "token_count": 111, + "top1_agreement": 0.954954981803894 + }, + { + "candidate_nll": 0.7912490963935852, + "code_tokens": 62, + "cvar95_kl": 0.0766354575753212, + "delta_nll": 0.0033528804779052734, + "max_kl": 0.11929142475128174, + "mean_kl": 0.01223558746278286, + "reference_nll": 0.7878962159156799, + "task_id": 388, + "token_count": 62, + "top1_agreement": 0.9838709831237793 + }, + { + "candidate_nll": 0.6615013480186462, + "code_tokens": 52, + "cvar95_kl": 0.15594740211963654, + "delta_nll": 0.05092132091522217, + "max_kl": 0.18398654460906982, + "mean_kl": 0.016978276893496513, + "reference_nll": 0.6105800271034241, + "task_id": 389, + "token_count": 52, + "top1_agreement": 0.9615384340286255 + }, + { + "candidate_nll": 1.9351726770401, + "code_tokens": 24, + "cvar95_kl": 0.05543317645788193, + "delta_nll": 0.03994607925415039, + "max_kl": 0.06325768679380417, + "mean_kl": 0.01573164202272892, + "reference_nll": 1.8952265977859497, + "task_id": 390, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.8907670378684998, + "code_tokens": 48, + "cvar95_kl": 0.0701715350151062, + "delta_nll": -0.0010029077529907227, + "max_kl": 0.07999762892723083, + "mean_kl": 0.013937552459537983, + "reference_nll": 0.8917699456214905, + "task_id": 391, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 1.1023372411727905, + "code_tokens": 98, + "cvar95_kl": 0.422923743724823, + "delta_nll": 0.05204427242279053, + "max_kl": 1.2044174671173096, + "mean_kl": 0.03686472401022911, + "reference_nll": 1.05029296875, + "task_id": 392, + "token_count": 98, + "top1_agreement": 0.9489796161651611 + }, + { + "candidate_nll": 1.024994969367981, + "code_tokens": 48, + "cvar95_kl": 0.12956997752189636, + "delta_nll": 0.03230935335159302, + "max_kl": 0.1435510367155075, + "mean_kl": 0.017899086698889732, + "reference_nll": 0.9926856160163879, + "task_id": 393, + "token_count": 48, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9701754450798035, + "code_tokens": 56, + "cvar95_kl": 0.09107991307973862, + "delta_nll": 0.024180054664611816, + "max_kl": 0.1538565307855606, + "mean_kl": 0.014384918846189976, + "reference_nll": 0.9459953904151917, + "task_id": 394, + "token_count": 56, + "top1_agreement": 0.9821428656578064 + }, + { + "candidate_nll": 0.6258216500282288, + "code_tokens": 88, + "cvar95_kl": 0.10527922958135605, + "delta_nll": 0.024544835090637207, + "max_kl": 0.2543560862541199, + "mean_kl": 0.012746840715408325, + "reference_nll": 0.6012768149375916, + "task_id": 395, + "token_count": 88, + "top1_agreement": 0.9886363744735718 + }, + { + "candidate_nll": 1.6432030200958252, + "code_tokens": 50, + "cvar95_kl": 0.08093447983264923, + "delta_nll": 0.009672760963439941, + "max_kl": 0.1042337641119957, + "mean_kl": 0.015018048696219921, + "reference_nll": 1.6335302591323853, + "task_id": 396, + "token_count": 50, + "top1_agreement": 0.9399999976158142 + }, + { + "candidate_nll": 0.4747089147567749, + "code_tokens": 85, + "cvar95_kl": 0.09135095775127411, + "delta_nll": 0.03772178292274475, + "max_kl": 0.14321166276931763, + "mean_kl": 0.011858598329126835, + "reference_nll": 0.43698713183403015, + "task_id": 397, + "token_count": 85, + "top1_agreement": 0.9882352948188782 + }, + { + "candidate_nll": 1.0748419761657715, + "code_tokens": 27, + "cvar95_kl": 0.20562946796417236, + "delta_nll": 0.0006288290023803711, + "max_kl": 0.2801329493522644, + "mean_kl": 0.021176137030124664, + "reference_nll": 1.0742131471633911, + "task_id": 398, + "token_count": 27, + "top1_agreement": 0.9629629850387573 + }, + { + "candidate_nll": 1.10923171043396, + "code_tokens": 48, + "cvar95_kl": 0.043779462575912476, + "delta_nll": -0.008386731147766113, + "max_kl": 0.05599815025925636, + "mean_kl": 0.006833361461758614, + "reference_nll": 1.117618441581726, + "task_id": 399, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 1.6806906461715698, + "code_tokens": 29, + "cvar95_kl": 0.09506280720233917, + "delta_nll": 0.03284335136413574, + "max_kl": 0.11023616045713425, + "mean_kl": 0.014915095642209053, + "reference_nll": 1.647847294807434, + "task_id": 400, + "token_count": 29, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 0.9267867803573608, + "code_tokens": 63, + "cvar95_kl": 0.12057723104953766, + "delta_nll": -0.011428594589233398, + "max_kl": 0.13842305541038513, + "mean_kl": 0.019357677549123764, + "reference_nll": 0.9382153749465942, + "task_id": 401, + "token_count": 63, + "top1_agreement": 0.9365079402923584 + }, + { + "candidate_nll": 0.7074007391929626, + "code_tokens": 89, + "cvar95_kl": 0.34924203157424927, + "delta_nll": 0.03234207630157471, + "max_kl": 0.5531567335128784, + "mean_kl": 0.03665266931056976, + "reference_nll": 0.6750586628913879, + "task_id": 402, + "token_count": 89, + "top1_agreement": 0.9213483333587646 + }, + { + "candidate_nll": 1.3831775188446045, + "code_tokens": 133, + "cvar95_kl": 0.23752744495868683, + "delta_nll": 0.03889918327331543, + "max_kl": 0.48672419786453247, + "mean_kl": 0.041971735656261444, + "reference_nll": 1.344278335571289, + "task_id": 403, + "token_count": 133, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 0.9621319770812988, + "code_tokens": 25, + "cvar95_kl": 0.02308012545108795, + "delta_nll": 0.013595223426818848, + "max_kl": 0.032042309641838074, + "mean_kl": 0.00443142419680953, + "reference_nll": 0.94853675365448, + "task_id": 404, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 1.0576459169387817, + "code_tokens": 33, + "cvar95_kl": 0.033895932137966156, + "delta_nll": 0.020345568656921387, + "max_kl": 0.041690096259117126, + "mean_kl": 0.006905803922563791, + "reference_nll": 1.0373003482818604, + "task_id": 405, + "token_count": 33, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 0.7989373207092285, + "code_tokens": 93, + "cvar95_kl": 0.05490586906671524, + "delta_nll": 0.04216235876083374, + "max_kl": 0.06338140368461609, + "mean_kl": 0.009969198144972324, + "reference_nll": 0.7567749619483948, + "task_id": 406, + "token_count": 93, + "top1_agreement": 0.9784946441650391 + }, + { + "candidate_nll": 1.0354381799697876, + "code_tokens": 101, + "cvar95_kl": 0.199149027466774, + "delta_nll": 0.06703859567642212, + "max_kl": 0.24845950305461884, + "mean_kl": 0.03779398649930954, + "reference_nll": 0.9683995842933655, + "task_id": 407, + "token_count": 101, + "top1_agreement": 0.9504950642585754 + }, + { + "candidate_nll": 0.6493760943412781, + "code_tokens": 152, + "cvar95_kl": 0.34511128067970276, + "delta_nll": 0.08461284637451172, + "max_kl": 0.7118408679962158, + "mean_kl": 0.039437342435121536, + "reference_nll": 0.5647632479667664, + "task_id": 408, + "token_count": 152, + "top1_agreement": 0.9539473652839661 + }, + { + "candidate_nll": 1.372037410736084, + "code_tokens": 33, + "cvar95_kl": 0.13796420395374298, + "delta_nll": -0.028834104537963867, + "max_kl": 0.15659023821353912, + "mean_kl": 0.021029746159911156, + "reference_nll": 1.4008715152740479, + "task_id": 409, + "token_count": 33, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 1.3267860412597656, + "code_tokens": 29, + "cvar95_kl": 0.053871266543865204, + "delta_nll": 0.04032313823699951, + "max_kl": 0.06536777317523956, + "mean_kl": 0.011939626187086105, + "reference_nll": 1.2864629030227661, + "task_id": 410, + "token_count": 29, + "top1_agreement": 0.931034505367279 + }, + { + "candidate_nll": 1.4100799560546875, + "code_tokens": 27, + "cvar95_kl": 0.09136359393596649, + "delta_nll": -0.047989845275878906, + "max_kl": 0.12292865663766861, + "mean_kl": 0.013250056654214859, + "reference_nll": 1.4580698013305664, + "task_id": 411, + "token_count": 27, + "top1_agreement": 0.8888888955116272 + }, + { + "candidate_nll": 0.3465244472026825, + "code_tokens": 33, + "cvar95_kl": 0.08240586519241333, + "delta_nll": 0.036484330892562866, + "max_kl": 0.13825030624866486, + "mean_kl": 0.008573217317461967, + "reference_nll": 0.31004011631011963, + "task_id": 412, + "token_count": 33, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7493338584899902, + "code_tokens": 27, + "cvar95_kl": 0.04621867090463638, + "delta_nll": -0.04706442356109619, + "max_kl": 0.06486675143241882, + "mean_kl": 0.006624146830290556, + "reference_nll": 0.7963982820510864, + "task_id": 413, + "token_count": 27, + "top1_agreement": 0.9259259104728699 + }, + { + "candidate_nll": 0.7413839101791382, + "code_tokens": 84, + "cvar95_kl": 0.1502968817949295, + "delta_nll": 0.03749042749404907, + "max_kl": 0.2795763313770294, + "mean_kl": 0.017890559509396553, + "reference_nll": 0.7038934826850891, + "task_id": 414, + "token_count": 84, + "top1_agreement": 0.9404761791229248 + }, + { + "candidate_nll": 1.0625836849212646, + "code_tokens": 103, + "cvar95_kl": 0.15374208986759186, + "delta_nll": 0.017113804817199707, + "max_kl": 0.31822943687438965, + "mean_kl": 0.022190583869814873, + "reference_nll": 1.045469880104065, + "task_id": 415, + "token_count": 103, + "top1_agreement": 0.9514563083648682 + }, + { + "candidate_nll": 0.9791324138641357, + "code_tokens": 88, + "cvar95_kl": 0.25087299942970276, + "delta_nll": 0.0627850890159607, + "max_kl": 0.3618548810482025, + "mean_kl": 0.040619973093271255, + "reference_nll": 0.916347324848175, + "task_id": 416, + "token_count": 88, + "top1_agreement": 0.9545454382896423 + }, + { + "candidate_nll": 1.314563512802124, + "code_tokens": 62, + "cvar95_kl": 0.13094450533390045, + "delta_nll": 0.005632758140563965, + "max_kl": 0.1478251814842224, + "mean_kl": 0.020497025921940804, + "reference_nll": 1.30893075466156, + "task_id": 417, + "token_count": 62, + "top1_agreement": 0.9838709831237793 + }, + { + "candidate_nll": 1.4929040670394897, + "code_tokens": 24, + "cvar95_kl": 0.11574216187000275, + "delta_nll": -0.04380476474761963, + "max_kl": 0.14014270901679993, + "mean_kl": 0.015025750733911991, + "reference_nll": 1.5367088317871094, + "task_id": 418, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 1.349303960800171, + "code_tokens": 34, + "cvar95_kl": 0.05510561913251877, + "delta_nll": 0.010851860046386719, + "max_kl": 0.05950147658586502, + "mean_kl": 0.008356660604476929, + "reference_nll": 1.3384521007537842, + "task_id": 419, + "token_count": 34, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 0.7278001308441162, + "code_tokens": 43, + "cvar95_kl": 0.07342124730348587, + "delta_nll": 0.008054733276367188, + "max_kl": 0.09533413499593735, + "mean_kl": 0.012286564335227013, + "reference_nll": 0.719745397567749, + "task_id": 420, + "token_count": 43, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.5068901777267456, + "code_tokens": 54, + "cvar95_kl": 0.15584784746170044, + "delta_nll": 0.017584919929504395, + "max_kl": 0.1880244016647339, + "mean_kl": 0.028502067551016808, + "reference_nll": 1.4893052577972412, + "task_id": 421, + "token_count": 54, + "top1_agreement": 0.9259259104728699 + }, + { + "candidate_nll": 0.7509819865226746, + "code_tokens": 50, + "cvar95_kl": 0.07063458114862442, + "delta_nll": 0.021096527576446533, + "max_kl": 0.10904515534639359, + "mean_kl": 0.008501277305185795, + "reference_nll": 0.729885458946228, + "task_id": 422, + "token_count": 50, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.824878990650177, + "code_tokens": 235, + "cvar95_kl": 0.6002566814422607, + "delta_nll": 0.0673685073852539, + "max_kl": 1.2297152280807495, + "mean_kl": 0.0612056702375412, + "reference_nll": 0.7575104832649231, + "task_id": 423, + "token_count": 235, + "top1_agreement": 0.9276595711708069 + }, + { + "candidate_nll": 2.001413583755493, + "code_tokens": 33, + "cvar95_kl": 0.13605687022209167, + "delta_nll": -0.02724146842956543, + "max_kl": 0.15042546391487122, + "mean_kl": 0.020412232726812363, + "reference_nll": 2.0286550521850586, + "task_id": 424, + "token_count": 33, + "top1_agreement": 0.939393937587738 + }, + { + "candidate_nll": 1.0411940813064575, + "code_tokens": 45, + "cvar95_kl": 0.07268987596035004, + "delta_nll": 0.05469167232513428, + "max_kl": 0.09944944828748703, + "mean_kl": 0.01068007294088602, + "reference_nll": 0.9865024089813232, + "task_id": 425, + "token_count": 45, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.5914716720581055, + "code_tokens": 28, + "cvar95_kl": 0.022881601005792618, + "delta_nll": -0.031488239765167236, + "max_kl": 0.02506117708981037, + "mean_kl": 0.005920010153204203, + "reference_nll": 0.6229599118232727, + "task_id": 426, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.704237163066864, + "code_tokens": 49, + "cvar95_kl": 0.05167461559176445, + "delta_nll": 0.02597278356552124, + "max_kl": 0.06918805837631226, + "mean_kl": 0.00966575462371111, + "reference_nll": 0.6782643795013428, + "task_id": 427, + "token_count": 49, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.3034667372703552, + "code_tokens": 107, + "cvar95_kl": 0.2436646968126297, + "delta_nll": 0.05405113101005554, + "max_kl": 0.5793058276176453, + "mean_kl": 0.026638265699148178, + "reference_nll": 0.24941560626029968, + "task_id": 428, + "token_count": 107, + "top1_agreement": 0.9906542301177979 + }, + { + "candidate_nll": 1.231398105621338, + "code_tokens": 48, + "cvar95_kl": 0.1559012085199356, + "delta_nll": 0.010488033294677734, + "max_kl": 0.30371880531311035, + "mean_kl": 0.01734270341694355, + "reference_nll": 1.2209100723266602, + "task_id": 429, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 1.8664627075195312, + "code_tokens": 40, + "cvar95_kl": 0.31352293491363525, + "delta_nll": 0.026921868324279785, + "max_kl": 0.37432050704956055, + "mean_kl": 0.04723726958036423, + "reference_nll": 1.8395408391952515, + "task_id": 430, + "token_count": 40, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 0.4768107831478119, + "code_tokens": 46, + "cvar95_kl": 0.11361858993768692, + "delta_nll": 0.0073271095752716064, + "max_kl": 0.16207297146320343, + "mean_kl": 0.01349517423659563, + "reference_nll": 0.4694836735725403, + "task_id": 431, + "token_count": 46, + "top1_agreement": 0.95652174949646 + }, + { + "candidate_nll": 1.0894922018051147, + "code_tokens": 31, + "cvar95_kl": 0.09538447111845016, + "delta_nll": 0.04704248905181885, + "max_kl": 0.11169710755348206, + "mean_kl": 0.021600106731057167, + "reference_nll": 1.042449712753296, + "task_id": 432, + "token_count": 31, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.43920353055000305, + "code_tokens": 58, + "cvar95_kl": 0.04361216351389885, + "delta_nll": 0.02271905541419983, + "max_kl": 0.060956016182899475, + "mean_kl": 0.005568974651396275, + "reference_nll": 0.4164844751358032, + "task_id": 433, + "token_count": 58, + "top1_agreement": 0.9482758641242981 + }, + { + "candidate_nll": 1.1330468654632568, + "code_tokens": 49, + "cvar95_kl": 0.05100981518626213, + "delta_nll": 0.01527702808380127, + "max_kl": 0.07299221307039261, + "mean_kl": 0.00913967564702034, + "reference_nll": 1.1177698373794556, + "task_id": 434, + "token_count": 49, + "top1_agreement": 0.9795918464660645 + }, + { + "candidate_nll": 0.9536568522453308, + "code_tokens": 18, + "cvar95_kl": 0.022316552698612213, + "delta_nll": 0.00706326961517334, + "max_kl": 0.022316552698612213, + "mean_kl": 0.0035301940515637398, + "reference_nll": 0.9465935826301575, + "task_id": 435, + "token_count": 18, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1768170595169067, + "code_tokens": 28, + "cvar95_kl": 0.0430050827562809, + "delta_nll": 0.01740550994873047, + "max_kl": 0.05284154787659645, + "mean_kl": 0.00821190420538187, + "reference_nll": 1.1594115495681763, + "task_id": 436, + "token_count": 28, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7771786451339722, + "code_tokens": 57, + "cvar95_kl": 0.15104912221431732, + "delta_nll": 0.03511190414428711, + "max_kl": 0.25534528493881226, + "mean_kl": 0.018327562138438225, + "reference_nll": 0.7420667409896851, + "task_id": 437, + "token_count": 57, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 0.6877825260162354, + "code_tokens": 88, + "cvar95_kl": 0.13252298533916473, + "delta_nll": -0.012327373027801514, + "max_kl": 0.21508657932281494, + "mean_kl": 0.015555155463516712, + "reference_nll": 0.7001098990440369, + "task_id": 438, + "token_count": 88, + "top1_agreement": 0.9772727489471436 + }, + { + "candidate_nll": 1.2044841051101685, + "code_tokens": 23, + "cvar95_kl": 0.025185663253068924, + "delta_nll": 0.0056972503662109375, + "max_kl": 0.027176523581147194, + "mean_kl": 0.0072504254058003426, + "reference_nll": 1.1987868547439575, + "task_id": 439, + "token_count": 23, + "top1_agreement": 0.95652174949646 + }, + { + "candidate_nll": 1.2415874004364014, + "code_tokens": 40, + "cvar95_kl": 0.10134722292423248, + "delta_nll": -0.017869949340820312, + "max_kl": 0.10220809280872345, + "mean_kl": 0.01714177429676056, + "reference_nll": 1.2594573497772217, + "task_id": 440, + "token_count": 40, + "top1_agreement": 0.925000011920929 + }, + { + "candidate_nll": 1.1963706016540527, + "code_tokens": 20, + "cvar95_kl": 0.027778862044215202, + "delta_nll": 0.035155534744262695, + "max_kl": 0.027778862044215202, + "mean_kl": 0.006575617007911205, + "reference_nll": 1.16121506690979, + "task_id": 441, + "token_count": 20, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.004531741142273, + "code_tokens": 63, + "cvar95_kl": 0.10316728800535202, + "delta_nll": -0.011253833770751953, + "max_kl": 0.1762593686580658, + "mean_kl": 0.01415962353348732, + "reference_nll": 1.015785574913025, + "task_id": 442, + "token_count": 63, + "top1_agreement": 0.9841269850730896 + }, + { + "candidate_nll": 0.984671950340271, + "code_tokens": 39, + "cvar95_kl": 0.050316669046878815, + "delta_nll": 0.031990647315979004, + "max_kl": 0.06336773931980133, + "mean_kl": 0.008897070772945881, + "reference_nll": 0.952681303024292, + "task_id": 443, + "token_count": 39, + "top1_agreement": 0.9487179517745972 + }, + { + "candidate_nll": 1.3959965705871582, + "code_tokens": 50, + "cvar95_kl": 0.1869727522134781, + "delta_nll": -0.015868067741394043, + "max_kl": 0.3494020402431488, + "mean_kl": 0.02093510329723358, + "reference_nll": 1.4118646383285522, + "task_id": 444, + "token_count": 50, + "top1_agreement": 0.9399999976158142 + }, + { + "candidate_nll": 0.9565740823745728, + "code_tokens": 63, + "cvar95_kl": 0.14460085332393646, + "delta_nll": -0.01090317964553833, + "max_kl": 0.20678384602069855, + "mean_kl": 0.019380493089556694, + "reference_nll": 0.9674772620201111, + "task_id": 445, + "token_count": 63, + "top1_agreement": 0.9682539701461792 + }, + { + "candidate_nll": 0.9631728529930115, + "code_tokens": 46, + "cvar95_kl": 0.07042079418897629, + "delta_nll": -0.009389162063598633, + "max_kl": 0.11634950339794159, + "mean_kl": 0.008682248182594776, + "reference_nll": 0.9725620150566101, + "task_id": 446, + "token_count": 46, + "top1_agreement": 0.97826087474823 + }, + { + "candidate_nll": 0.7040809392929077, + "code_tokens": 25, + "cvar95_kl": 0.01806756667792797, + "delta_nll": 0.02518641948699951, + "max_kl": 0.018341299146413803, + "mean_kl": 0.003948192112147808, + "reference_nll": 0.6788945198059082, + "task_id": 447, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.8307716250419617, + "code_tokens": 111, + "cvar95_kl": 0.20295751094818115, + "delta_nll": 0.016505718231201172, + "max_kl": 0.2827592194080353, + "mean_kl": 0.02860953100025654, + "reference_nll": 0.8142659068107605, + "task_id": 448, + "token_count": 111, + "top1_agreement": 0.9729729890823364 + }, + { + "candidate_nll": 0.7987746596336365, + "code_tokens": 71, + "cvar95_kl": 0.17072945833206177, + "delta_nll": 0.0702168345451355, + "max_kl": 0.3220522403717041, + "mean_kl": 0.025422334671020508, + "reference_nll": 0.728557825088501, + "task_id": 449, + "token_count": 71, + "top1_agreement": 0.9577465057373047 + }, + { + "candidate_nll": 1.3708029985427856, + "code_tokens": 28, + "cvar95_kl": 0.06496630609035492, + "delta_nll": 7.939338684082031e-05, + "max_kl": 0.06876038014888763, + "mean_kl": 0.012101695872843266, + "reference_nll": 1.3707236051559448, + "task_id": 450, + "token_count": 28, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 1.205378770828247, + "code_tokens": 25, + "cvar95_kl": 0.04758976399898529, + "delta_nll": 0.0049228668212890625, + "max_kl": 0.05442549288272858, + "mean_kl": 0.008710004389286041, + "reference_nll": 1.200455904006958, + "task_id": 451, + "token_count": 25, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 1.3065283298492432, + "code_tokens": 40, + "cvar95_kl": 0.06758678704500198, + "delta_nll": 0.0037565231323242188, + "max_kl": 0.10142870992422104, + "mean_kl": 0.010259933769702911, + "reference_nll": 1.302771806716919, + "task_id": 452, + "token_count": 40, + "top1_agreement": 0.9750000238418579 + }, + { + "candidate_nll": 1.0784885883331299, + "code_tokens": 177, + "cvar95_kl": 0.7137001752853394, + "delta_nll": 0.03404438495635986, + "max_kl": 1.6582825183868408, + "mean_kl": 0.06332435458898544, + "reference_nll": 1.04444420337677, + "task_id": 453, + "token_count": 177, + "top1_agreement": 0.9604519605636597 + }, + { + "candidate_nll": 1.1014056205749512, + "code_tokens": 51, + "cvar95_kl": 0.058417707681655884, + "delta_nll": 0.010227084159851074, + "max_kl": 0.0759146511554718, + "mean_kl": 0.01010934729129076, + "reference_nll": 1.0911785364151, + "task_id": 454, + "token_count": 51, + "top1_agreement": 0.9607843160629272 + }, + { + "candidate_nll": 0.4840967357158661, + "code_tokens": 69, + "cvar95_kl": 0.2349165380001068, + "delta_nll": 0.02963632345199585, + "max_kl": 0.5340167880058289, + "mean_kl": 0.019256427884101868, + "reference_nll": 0.45446041226387024, + "task_id": 455, + "token_count": 69, + "top1_agreement": 0.9710144996643066 + }, + { + "candidate_nll": 0.7290761470794678, + "code_tokens": 26, + "cvar95_kl": 0.06364938616752625, + "delta_nll": -0.019183814525604248, + "max_kl": 0.07809752970933914, + "mean_kl": 0.00926834624260664, + "reference_nll": 0.748259961605072, + "task_id": 456, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.5043166875839233, + "code_tokens": 24, + "cvar95_kl": 0.04900293052196503, + "delta_nll": -0.014022588729858398, + "max_kl": 0.050397202372550964, + "mean_kl": 0.013988635502755642, + "reference_nll": 1.5183392763137817, + "task_id": 457, + "token_count": 24, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.1355589628219604, + "code_tokens": 15, + "cvar95_kl": 0.016511835157871246, + "delta_nll": -0.01491689682006836, + "max_kl": 0.016511835157871246, + "mean_kl": 0.0033816960640251637, + "reference_nll": 1.1504758596420288, + "task_id": 458, + "token_count": 15, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 1.172558307647705, + "code_tokens": 43, + "cvar95_kl": 0.06352805346250534, + "delta_nll": 0.023209333419799805, + "max_kl": 0.07637634873390198, + "mean_kl": 0.014446056447923183, + "reference_nll": 1.1493489742279053, + "task_id": 459, + "token_count": 43, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1015124320983887, + "code_tokens": 18, + "cvar95_kl": 0.020868539810180664, + "delta_nll": 0.009186744689941406, + "max_kl": 0.020868539810180664, + "mean_kl": 0.005316751077771187, + "reference_nll": 1.0923256874084473, + "task_id": 460, + "token_count": 18, + "top1_agreement": 0.9444444179534912 + }, + { + "candidate_nll": 0.6854031085968018, + "code_tokens": 49, + "cvar95_kl": 0.14524470269680023, + "delta_nll": -0.042590439319610596, + "max_kl": 0.29038795828819275, + "mean_kl": 0.012727811932563782, + "reference_nll": 0.7279935479164124, + "task_id": 461, + "token_count": 49, + "top1_agreement": 0.9591836929321289 + }, + { + "candidate_nll": 0.9129638671875, + "code_tokens": 59, + "cvar95_kl": 0.6638474464416504, + "delta_nll": 0.04520779848098755, + "max_kl": 0.9832404255867004, + "mean_kl": 0.05962372571229935, + "reference_nll": 0.8677560687065125, + "task_id": 462, + "token_count": 59, + "top1_agreement": 0.9152542352676392 + }, + { + "candidate_nll": 0.585524320602417, + "code_tokens": 221, + "cvar95_kl": 0.37261953949928284, + "delta_nll": 0.0002581477165222168, + "max_kl": 0.7849997878074646, + "mean_kl": 0.04811101779341698, + "reference_nll": 0.5852661728858948, + "task_id": 463, + "token_count": 221, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 1.2039875984191895, + "code_tokens": 25, + "cvar95_kl": 0.04335497319698334, + "delta_nll": 0.002114534378051758, + "max_kl": 0.04574044048786163, + "mean_kl": 0.009165328927338123, + "reference_nll": 1.2018730640411377, + "task_id": 464, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7584768533706665, + "code_tokens": 36, + "cvar95_kl": 0.027563516050577164, + "delta_nll": 0.03455626964569092, + "max_kl": 0.03031344898045063, + "mean_kl": 0.00764690013602376, + "reference_nll": 0.7239205837249756, + "task_id": 465, + "token_count": 36, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.717377781867981, + "code_tokens": 158, + "cvar95_kl": 0.42555877566337585, + "delta_nll": 0.07323294878005981, + "max_kl": 0.9941749572753906, + "mean_kl": 0.04040837287902832, + "reference_nll": 0.6441448330879211, + "task_id": 466, + "token_count": 158, + "top1_agreement": 0.949367105960846 + }, + { + "candidate_nll": 1.308505892753601, + "code_tokens": 92, + "cvar95_kl": 0.23846109211444855, + "delta_nll": -0.007029175758361816, + "max_kl": 0.3382641077041626, + "mean_kl": 0.03174864128232002, + "reference_nll": 1.315535068511963, + "task_id": 467, + "token_count": 92, + "top1_agreement": 0.9130434989929199 + }, + { + "candidate_nll": 0.9745535254478455, + "code_tokens": 102, + "cvar95_kl": 0.17308418452739716, + "delta_nll": 0.060510873794555664, + "max_kl": 0.3671827018260956, + "mean_kl": 0.02793225273489952, + "reference_nll": 0.9140426516532898, + "task_id": 468, + "token_count": 102, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 0.6459335684776306, + "code_tokens": 171, + "cvar95_kl": 0.4827604293823242, + "delta_nll": 0.035963475704193115, + "max_kl": 0.942743182182312, + "mean_kl": 0.04532768949866295, + "reference_nll": 0.6099700927734375, + "task_id": 469, + "token_count": 171, + "top1_agreement": 0.9532163739204407 + }, + { + "candidate_nll": 1.1729586124420166, + "code_tokens": 40, + "cvar95_kl": 0.1798824816942215, + "delta_nll": -0.031905531883239746, + "max_kl": 0.2184639424085617, + "mean_kl": 0.017238488420844078, + "reference_nll": 1.2048641443252563, + "task_id": 470, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.250765085220337, + "code_tokens": 48, + "cvar95_kl": 0.06154729425907135, + "delta_nll": 0.014751195907592773, + "max_kl": 0.0640554204583168, + "mean_kl": 0.013638262636959553, + "reference_nll": 1.2360138893127441, + "task_id": 471, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 1.59060800075531, + "code_tokens": 24, + "cvar95_kl": 0.10479508340358734, + "delta_nll": -0.014312386512756348, + "max_kl": 0.10830754041671753, + "mean_kl": 0.018556347116827965, + "reference_nll": 1.6049203872680664, + "task_id": 472, + "token_count": 24, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.9859647154808044, + "code_tokens": 48, + "cvar95_kl": 0.08888667821884155, + "delta_nll": -0.0011394619941711426, + "max_kl": 0.11321575194597244, + "mean_kl": 0.014232267625629902, + "reference_nll": 0.9871041774749756, + "task_id": 473, + "token_count": 48, + "top1_agreement": 0.9791666865348816 + }, + { + "candidate_nll": 1.383477807044983, + "code_tokens": 25, + "cvar95_kl": 0.0694597065448761, + "delta_nll": -0.003405332565307617, + "max_kl": 0.0974477082490921, + "mean_kl": 0.011316094547510147, + "reference_nll": 1.3868831396102905, + "task_id": 474, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.49570894241333, + "code_tokens": 30, + "cvar95_kl": 0.21150292456150055, + "delta_nll": -0.003035902976989746, + "max_kl": 0.23595860600471497, + "mean_kl": 0.026464898139238358, + "reference_nll": 1.4987448453903198, + "task_id": 475, + "token_count": 30, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.4447154998779297, + "code_tokens": 19, + "cvar95_kl": 0.03659699857234955, + "delta_nll": 0.017897605895996094, + "max_kl": 0.03659699857234955, + "mean_kl": 0.005064981058239937, + "reference_nll": 1.4268178939819336, + "task_id": 476, + "token_count": 19, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.135135293006897, + "code_tokens": 12, + "cvar95_kl": 0.029555844143033028, + "delta_nll": 0.05606281757354736, + "max_kl": 0.029555844143033028, + "mean_kl": 0.011669459752738476, + "reference_nll": 1.0790724754333496, + "task_id": 477, + "token_count": 12, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 1.3537266254425049, + "code_tokens": 37, + "cvar95_kl": 0.31013914942741394, + "delta_nll": 0.013416886329650879, + "max_kl": 0.3211129903793335, + "mean_kl": 0.03070978820323944, + "reference_nll": 1.340309739112854, + "task_id": 478, + "token_count": 37, + "top1_agreement": 0.9729729890823364 + }, + { + "candidate_nll": 1.2267135381698608, + "code_tokens": 33, + "cvar95_kl": 0.11491800099611282, + "delta_nll": 0.021283745765686035, + "max_kl": 0.16394609212875366, + "mean_kl": 0.0180739164352417, + "reference_nll": 1.2054297924041748, + "task_id": 479, + "token_count": 33, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 0.8754968643188477, + "code_tokens": 94, + "cvar95_kl": 0.10086624324321747, + "delta_nll": 0.02338707447052002, + "max_kl": 0.14188213646411896, + "mean_kl": 0.01716691441833973, + "reference_nll": 0.8521097898483276, + "task_id": 480, + "token_count": 94, + "top1_agreement": 0.978723406791687 + }, + { + "candidate_nll": 0.7159632444381714, + "code_tokens": 90, + "cvar95_kl": 0.30320173501968384, + "delta_nll": 0.09119832515716553, + "max_kl": 0.6596633195877075, + "mean_kl": 0.03411167114973068, + "reference_nll": 0.6247649192810059, + "task_id": 481, + "token_count": 90, + "top1_agreement": 0.9111111164093018 + }, + { + "candidate_nll": 1.4283697605133057, + "code_tokens": 47, + "cvar95_kl": 0.09160161763429642, + "delta_nll": -0.0193941593170166, + "max_kl": 0.17787083983421326, + "mean_kl": 0.011072036810219288, + "reference_nll": 1.4477639198303223, + "task_id": 482, + "token_count": 47, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8885927200317383, + "code_tokens": 61, + "cvar95_kl": 0.05456550791859627, + "delta_nll": -0.03524142503738403, + "max_kl": 0.09876620769500732, + "mean_kl": 0.010700013488531113, + "reference_nll": 0.9238341450691223, + "task_id": 483, + "token_count": 61, + "top1_agreement": 0.9836065769195557 + }, + { + "candidate_nll": 1.209304690361023, + "code_tokens": 39, + "cvar95_kl": 0.0643913596868515, + "delta_nll": -0.014210104942321777, + "max_kl": 0.07246784120798111, + "mean_kl": 0.009054924361407757, + "reference_nll": 1.2235147953033447, + "task_id": 484, + "token_count": 39, + "top1_agreement": 0.9743589758872986 + }, + { + "candidate_nll": 0.9627336263656616, + "code_tokens": 152, + "cvar95_kl": 0.3158322274684906, + "delta_nll": 0.030571818351745605, + "max_kl": 0.6445234417915344, + "mean_kl": 0.036978960037231445, + "reference_nll": 0.932161808013916, + "task_id": 485, + "token_count": 152, + "top1_agreement": 0.9342105388641357 + }, + { + "candidate_nll": 1.0412721633911133, + "code_tokens": 100, + "cvar95_kl": 0.27707260847091675, + "delta_nll": 0.07228976488113403, + "max_kl": 0.5407271981239319, + "mean_kl": 0.032359085977077484, + "reference_nll": 0.9689823985099792, + "task_id": 486, + "token_count": 100, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.6469086408615112, + "code_tokens": 89, + "cvar95_kl": 0.09540291875600815, + "delta_nll": 0.025980830192565918, + "max_kl": 0.23202835023403168, + "mean_kl": 0.012065545655786991, + "reference_nll": 0.6209278106689453, + "task_id": 487, + "token_count": 89, + "top1_agreement": 0.9775280952453613 + }, + { + "candidate_nll": 1.2830270528793335, + "code_tokens": 40, + "cvar95_kl": 0.07157190144062042, + "delta_nll": -0.02797412872314453, + "max_kl": 0.0817950889468193, + "mean_kl": 0.019496969878673553, + "reference_nll": 1.311001181602478, + "task_id": 488, + "token_count": 40, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7482709884643555, + "code_tokens": 76, + "cvar95_kl": 0.2708171010017395, + "delta_nll": 0.046703994274139404, + "max_kl": 0.6911482214927673, + "mean_kl": 0.027385178953409195, + "reference_nll": 0.7015669941902161, + "task_id": 489, + "token_count": 76, + "top1_agreement": 0.9868420958518982 + }, + { + "candidate_nll": 1.2618752717971802, + "code_tokens": 56, + "cvar95_kl": 0.08406724780797958, + "delta_nll": 0.012887954711914062, + "max_kl": 0.1315460503101349, + "mean_kl": 0.014775125309824944, + "reference_nll": 1.2489873170852661, + "task_id": 490, + "token_count": 56, + "top1_agreement": 0.9642857313156128 + }, + { + "candidate_nll": 1.5164787769317627, + "code_tokens": 34, + "cvar95_kl": 0.06681719422340393, + "delta_nll": 0.027048468589782715, + "max_kl": 0.0801108330488205, + "mean_kl": 0.013115234673023224, + "reference_nll": 1.48943030834198, + "task_id": 491, + "token_count": 34, + "top1_agreement": 0.970588207244873 + }, + { + "candidate_nll": 0.693871021270752, + "code_tokens": 96, + "cvar95_kl": 0.13144978880882263, + "delta_nll": 0.015759527683258057, + "max_kl": 0.2185310423374176, + "mean_kl": 0.014104914851486683, + "reference_nll": 0.6781114935874939, + "task_id": 492, + "token_count": 96, + "top1_agreement": 0.9895833134651184 + }, + { + "candidate_nll": 0.7470608949661255, + "code_tokens": 451, + "cvar95_kl": 0.48705556988716125, + "delta_nll": 0.048365771770477295, + "max_kl": 1.3948087692260742, + "mean_kl": 0.04957452416419983, + "reference_nll": 0.6986951231956482, + "task_id": 493, + "token_count": 451, + "top1_agreement": 0.9423503279685974 + }, + { + "candidate_nll": 1.6578117609024048, + "code_tokens": 37, + "cvar95_kl": 0.08977089822292328, + "delta_nll": 0.04550206661224365, + "max_kl": 0.12209009379148483, + "mean_kl": 0.014246249571442604, + "reference_nll": 1.6123096942901611, + "task_id": 494, + "token_count": 37, + "top1_agreement": 0.9459459185600281 + }, + { + "candidate_nll": 1.2139036655426025, + "code_tokens": 42, + "cvar95_kl": 0.03702859953045845, + "delta_nll": -0.027694225311279297, + "max_kl": 0.048859067261219025, + "mean_kl": 0.009880652651190758, + "reference_nll": 1.2415978908538818, + "task_id": 495, + "token_count": 42, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.0272395610809326, + "code_tokens": 33, + "cvar95_kl": 0.05445656552910805, + "delta_nll": 0.010483622550964355, + "max_kl": 0.0635639876127243, + "mean_kl": 0.01233659591525793, + "reference_nll": 1.0167559385299683, + "task_id": 496, + "token_count": 33, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 0.974816620349884, + "code_tokens": 43, + "cvar95_kl": 0.20148463547229767, + "delta_nll": 0.048764944076538086, + "max_kl": 0.22077667713165283, + "mean_kl": 0.03526098281145096, + "reference_nll": 0.926051676273346, + "task_id": 497, + "token_count": 43, + "top1_agreement": 0.9534883499145508 + }, + { + "candidate_nll": 0.7556564211845398, + "code_tokens": 73, + "cvar95_kl": 0.2679443955421448, + "delta_nll": 0.01921999454498291, + "max_kl": 0.43183472752571106, + "mean_kl": 0.026784859597682953, + "reference_nll": 0.7364364266395569, + "task_id": 498, + "token_count": 73, + "top1_agreement": 0.9452054500579834 + }, + { + "candidate_nll": 0.976017951965332, + "code_tokens": 15, + "cvar95_kl": 0.08704472333192825, + "delta_nll": 0.06354176998138428, + "max_kl": 0.08704472333192825, + "mean_kl": 0.012904077768325806, + "reference_nll": 0.9124761819839478, + "task_id": 499, + "token_count": 15, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2683161497116089, + "code_tokens": 34, + "cvar95_kl": 0.0562664270401001, + "delta_nll": 0.00923621654510498, + "max_kl": 0.08258020132780075, + "mean_kl": 0.006387277040630579, + "reference_nll": 1.259079933166504, + "task_id": 500, + "token_count": 34, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1545133590698242, + "code_tokens": 136, + "cvar95_kl": 0.20561525225639343, + "delta_nll": 0.04603016376495361, + "max_kl": 0.30097803473472595, + "mean_kl": 0.028467528522014618, + "reference_nll": 1.1084831953048706, + "task_id": 501, + "token_count": 136, + "top1_agreement": 0.9632353186607361 + }, + { + "candidate_nll": 1.250040888786316, + "code_tokens": 17, + "cvar95_kl": 0.05284865200519562, + "delta_nll": 0.0038847923278808594, + "max_kl": 0.05284865200519562, + "mean_kl": 0.01225394569337368, + "reference_nll": 1.246156096458435, + "task_id": 502, + "token_count": 17, + "top1_agreement": 0.9411764740943909 + }, + { + "candidate_nll": 0.7802497744560242, + "code_tokens": 33, + "cvar95_kl": 0.0808057114481926, + "delta_nll": -0.012804925441741943, + "max_kl": 0.12308210879564285, + "mean_kl": 0.009628756903111935, + "reference_nll": 0.7930546998977661, + "task_id": 503, + "token_count": 33, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.8536463379859924, + "code_tokens": 38, + "cvar95_kl": 0.1811784952878952, + "delta_nll": 0.011199533939361572, + "max_kl": 0.23403283953666687, + "mean_kl": 0.013523564673960209, + "reference_nll": 0.8424468040466309, + "task_id": 504, + "token_count": 38, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 0.423294335603714, + "code_tokens": 61, + "cvar95_kl": 0.10471564531326294, + "delta_nll": 0.025769710540771484, + "max_kl": 0.18075010180473328, + "mean_kl": 0.014746279455721378, + "reference_nll": 0.3975246250629425, + "task_id": 505, + "token_count": 61, + "top1_agreement": 0.9672130942344666 + }, + { + "candidate_nll": 0.770177960395813, + "code_tokens": 143, + "cvar95_kl": 0.24456648528575897, + "delta_nll": 0.0344584584236145, + "max_kl": 0.4301949441432953, + "mean_kl": 0.033687617629766464, + "reference_nll": 0.7357195019721985, + "task_id": 506, + "token_count": 143, + "top1_agreement": 0.9720279574394226 + }, + { + "candidate_nll": 1.1219351291656494, + "code_tokens": 41, + "cvar95_kl": 0.07029931992292404, + "delta_nll": 0.03286921977996826, + "max_kl": 0.12154436111450195, + "mean_kl": 0.010534724220633507, + "reference_nll": 1.0890659093856812, + "task_id": 507, + "token_count": 41, + "top1_agreement": 0.9756097793579102 + }, + { + "candidate_nll": 0.3722248375415802, + "code_tokens": 67, + "cvar95_kl": 0.06160646677017212, + "delta_nll": 0.007543087005615234, + "max_kl": 0.0704554095864296, + "mean_kl": 0.008399199694395065, + "reference_nll": 0.36468175053596497, + "task_id": 508, + "token_count": 67, + "top1_agreement": 0.9850746393203735 + }, + { + "candidate_nll": 1.3727877140045166, + "code_tokens": 76, + "cvar95_kl": 0.12699520587921143, + "delta_nll": -0.017472267150878906, + "max_kl": 0.14101627469062805, + "mean_kl": 0.023906322196125984, + "reference_nll": 1.3902599811553955, + "task_id": 509, + "token_count": 76, + "top1_agreement": 0.9342105388641357 + }, + { + "candidate_nll": 0.6619356870651245, + "code_tokens": 137, + "cvar95_kl": 0.554984450340271, + "delta_nll": 0.026083052158355713, + "max_kl": 0.8842194080352783, + "mean_kl": 0.05225493758916855, + "reference_nll": 0.6358526349067688, + "task_id": 510, + "token_count": 137, + "top1_agreement": 0.9124087691307068 + } + ], + "policy": { + "default_bits": 8, + "name": "uniform_int8_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 4792320, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 4792320, + "resident_compression_ratio": 3.9384615384615387 + }, + "task_delta_nll_distribution": { + "maximum": 0.11862868070602417, + "median": 0.016885578632354736, + "minimum": -0.06821763515472412, + "q25": 6.486475467681885e-05, + "q75": 0.034057170152664185 + }, + "task_macro": { + "candidate_nll": 1.0312559577226639, + "delta_nll": 0.017209371507167816, + "mean_kl": 0.018916484776418656, + "reference_nll": 1.0140465862154961, + "task_count": 500, + "top1_agreement": 0.9646199452877045 + }, + "token_weighted": { + "candidate_nll": 0.9411832094192505, + "cvar95_kl": 0.2425336241722107, + "delta_nll": 0.02434563636779785, + "max_kl": 1.9180057048797607, + "mean_kl": 0.025100789964199066, + "reference_nll": 0.9168375730514526, + "token_count": 30244, + "top1_agreement": 0.958537220954895 + } + } + }, + "claim_scope": { + "confirmation_touched": true, + "generated_code_executed": false, + "phase": "confirmation", + "protocol_eligible": true, + "speed_claim_allowed": false, + "teacher_forced_fidelity_only": true, + "whole_model_memory_claim_allowed": false + }, + "continuation_decision": { + "all_gates_pass": true, + "confirmation_permitted": false, + "gates": { + "all_values_finite": true, + "equal_byte_bootstrap_interval_above_zero": true, + "exact_resident_bytes": true, + "primary_cvar95_token_kl_lower_than_uniform_int4": true, + "primary_macro_delta_nll_reduction_at_least_15_percent": true, + "primary_mean_token_kl_lower_than_uniform_int4": true, + "primary_top1_not_lower_than_uniform_int4": true + } + }, + "contrasts": { + "primary_vs_mean_random_equal_byte": { + "paired_bootstrap": { + "bootstrap_samples": 10000, + "confidence": 0.95, + "confidence_interval": [ + 1.9802198561151823, + 2.086135943630834 + ], + "mean_improvement": 2.03316749749581, + "paired_examples": 500, + "seed": 2339 + } + }, + "primary_vs_uniform_int4": { + "paired_bootstrap": { + "bootstrap_samples": 10000, + "confidence": 0.95, + "confidence_interval": [ + 2.0922485922992227, + 2.199865858581662 + ], + "mean_improvement": 2.146029668748379, + "paired_examples": 500, + "seed": 2339 + }, + "primary_macro_delta_nll": 0.8037128749489785, + "relative_reduction": 0.727531178384957, + "uniform_macro_delta_nll": 2.949742543697357 + } + }, + "environment": { + "command": [ + "python.exe", + "scripts\\evaluate_mbpp.py", + "--phase", + "confirmation", + "--calibration-artifact", + "evidence\\mbpp-v02-calibration.json", + "--prepared-manifest", + "evidence\\mbpp-v02-confirmation-manifest.json", + "--confirmation-lock", + "recurquant:unlock-mbpp-confirmation:rq-v0.2", + "--local-files-only", + "--output", + "artifacts\\mbpp-v02-confirmation.json" + ], + "cuda_runtime": "12.8", + "datasets": "4.8.5", + "device": "cuda", + "device_name": "NVIDIA GeForce RTX 5070 Laptop GPU", + "model_dtype": "torch.bfloat16", + "nvidia_driver_versions": [ + "592.15" + ], + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "torch": "2.11.0+cu128", + "tracked_worktree_clean": true, + "transformers": "5.14.1" + }, + "schedule": { + "candidate_generated_tokens_fed_back": false, + "candidate_order": [ + "uniform_int4_nearest", + "uniform_int8_nearest", + "read_risk_l0_nearest", + "mse_selected_nearest", + "random_l18_nearest", + "random_l4_nearest", + "random_l13_nearest", + "read_risk_l0_stochastic_seed_2339", + "read_risk_l0_stochastic_seed_2340", + "read_risk_l0_stochastic_seed_2341" + ], + "checkpoint_path": "artifacts\\mbpp-v02-confirmation.checkpoint.json", + "elapsed_wall_seconds_not_a_latency_benchmark": 24620.9765337, + "final_checkpoint_sha256": "df0040cc9cebdbc442992e75d19f9090456f9b249da062c095840e731b6c4609", + "group_size": 128, + "phase": "confirmation", + "resumed_task_count": 370, + "row_count": 500, + "run_signature_sha256": "5d15268224357bb078315ef2c2b6e710a7eb8a2734df2527f9637b382951c78a", + "scored_first_code_token_from_prefill": true, + "seed": 2339 + }, + "source": { + "calibration_artifact_path": "evidence\\mbpp-v02-calibration.json", + "calibration_evidence_sha256": "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee", + "dataset_manifest": { + "config": "full", + "dataset_id": "google-research-datasets/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "confirmation", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "row_count": 500, + "rows": [ + { + "sha256": "2f6a29231bbebfdb0b17c2e267f26f9b35cfd6e6724771647ff1bb77821c40ca", + "task_id": 11 + }, + { + "sha256": "69701847454dfe9133e911d6e636c93515010268a4e768ddb80452c3d13b6614", + "task_id": 12 + }, + { + "sha256": "363abf0efa8bed1a2643c6f4a0912c40686d7721a7a24026fe5dcb2301dbd492", + "task_id": 13 + }, + { + "sha256": "5ebb47a4e9ea4f41cff8bdd7e41783b40429299b3d764f568bfa4dd31ae5fa24", + "task_id": 14 + }, + { + "sha256": "018ccd1c6fb7ec04498dfd5758eb1b8daa39b7ec39e49cc2020178f3bf69ee33", + "task_id": 15 + }, + { + "sha256": "8e4122f4410ea296503267f6ab8dd17f75ae966a3312022560075810a596c1ee", + "task_id": 16 + }, + { + "sha256": "b96c8dfb7ee4fae1195ad89a7f95d6e872988541238679783c855e829e7383fc", + "task_id": 17 + }, + { + "sha256": "705af8c177b2f31958e0633f222ace66b96f41b5922c4176b3822d9c9d22fb06", + "task_id": 18 + }, + { + "sha256": "fdbbfc6f6ec0567d2a5d14f5e3c1bea1af31517de6fb21ff0512c1c1c9448218", + "task_id": 19 + }, + { + "sha256": "be366443f7b156b46f910e0b178411a943894cdea3d4528903f3e23fb9846cdc", + "task_id": 20 + }, + { + "sha256": "c8c319ead869d4b9dba5d362042b55f8d8e7ba0a96dfdae33824b46a6a82e9e2", + "task_id": 21 + }, + { + "sha256": "9ff2683e64c8a4148d7ad62110756a1e3d54297ca285c313c223e596b9260fff", + "task_id": 22 + }, + { + "sha256": "6ab376afe986e3bf251ffd655081a82ce0745c0f6cabe8180d6bf9c0204e6d52", + "task_id": 23 + }, + { + "sha256": "8787c6a4ad8a8ce82cfe3f1983aae531a6a9134e85a981290865dc41996277e9", + "task_id": 24 + }, + { + "sha256": "66a9e6dad977f0a86a21d391656edde9f8558c2125c161c57bbb2d936560c839", + "task_id": 25 + }, + { + "sha256": "81c94a6c19f80640a985bd88e07e3be904962964e57e954fef3be94d683e3401", + "task_id": 26 + }, + { + "sha256": "212c739c9b40f26cc81ac28928921662cb6fe027df124f0c380e31251cd8f636", + "task_id": 27 + }, + { + "sha256": "5e29a25185d0953eb0b5062ac3a40a0fd14a1f94532cea97e892280f5412e6f1", + "task_id": 28 + }, + { + "sha256": "2bc9923a1e44dc11dc70732ad4da7a852f03d737eee531b92b499413466fb6a8", + "task_id": 29 + }, + { + "sha256": "4d763680cadb6bdabd5932a74fbaabb8b992e924abc9e080466cb3ee61f2305b", + "task_id": 30 + }, + { + "sha256": "215c397ffc2e58c32569eba2fb2972fe4303a83213d679f25cd99fa66949267d", + "task_id": 31 + }, + { + "sha256": "cd45e5f183ef7cdfa051dc6750b6a34677b3e0160ef4eff1f364a9f20e1da93f", + "task_id": 32 + }, + { + "sha256": "02760c978d7d50dde29c53b8f140c264ecd87231dfdb7209afdd68b6be4cf8bd", + "task_id": 33 + }, + { + "sha256": "dccdef71e53d0c026eade07be98bae9e95678ee67bc551d4e9bd0e779616eb7a", + "task_id": 34 + }, + { + "sha256": "aa4724f597c4b7636cde1ace9d5807c24bde42f8df576a6fc46d86effb1eada4", + "task_id": 35 + }, + { + "sha256": "7d141a9e2334529294d140fa3ecb1572bb5bce5a77b755f9659451e08c43ac23", + "task_id": 36 + }, + { + "sha256": "7fa06d17c47d679a01a9aa55d9a37e43397e5b5c3d76f906dc53b5fc1ae0746c", + "task_id": 37 + }, + { + "sha256": "009de5516476736101e9e9b904a1d335290578c978e4b932bbd8ae4475fb7082", + "task_id": 38 + }, + { + "sha256": "5aaf4c13f871082a2196f39d96126ea13d7b472823a1bf5de325735e83aef23e", + "task_id": 39 + }, + { + "sha256": "682bc367abdb33a52378f0007d0cb9e71140268bbb01cebfb3d6b36eeb360a35", + "task_id": 40 + }, + { + "sha256": "178b361b3fa5a13f7a3c1cebccbab59e583fd22f1d8fdbab74846347bc29f4d6", + "task_id": 41 + }, + { + "sha256": "0a25a29a2ae9a0f5e59686918169356e953d76d8ccf3121ca4787f808f33539d", + "task_id": 42 + }, + { + "sha256": "206ef439e2f71af51d06bdfa1fe38be444d5d4b2dee58b5db4cf26f663311720", + "task_id": 43 + }, + { + "sha256": "f9ccea714fa76b4ba14787d2cb5997fb1e02f4467cf393076eb799c45e0dc863", + "task_id": 44 + }, + { + "sha256": "b1f2a1bac9e4adecbe6b9e4bef031f4cc1e73721847ddec28a4c309212036395", + "task_id": 45 + }, + { + "sha256": "8d7e5e8686b6dfa0f706a3b6d4cc1a49a273514469bd6c43f3f13234e670b021", + "task_id": 46 + }, + { + "sha256": "870418aa2ef21be37b06635fb9e848864c057ae70a937ddd37709abbfbff0077", + "task_id": 47 + }, + { + "sha256": "907304b908f9017806de07794bf75c35585e902d791cbef3ccf57bac2d632c70", + "task_id": 48 + }, + { + "sha256": "2a31c05527a2774c21f9168760205bf8743cf3f5445d20766d3b0b731351d49f", + "task_id": 49 + }, + { + "sha256": "a9552c1ffb0db7fd510527b21de25ade98178d8a9b46f2553afc1b5540b32105", + "task_id": 50 + }, + { + "sha256": "5f0589cebbc69aafdac77e431f895c2ffdf6c7243caf4fbad9b6d189cd1adb14", + "task_id": 51 + }, + { + "sha256": "c92f7b1e1e9627db3d20d82187d1b6df4faae30b324c32a16a82e8b4e1b97e6c", + "task_id": 52 + }, + { + "sha256": "c3b0a8667f8158d646833fa00f05157b35437f850f7e8077050f5bd3003c592d", + "task_id": 53 + }, + { + "sha256": "4a05ed70d38d8b295de6219759399afd42854f8a9566aef386551e00666d14af", + "task_id": 54 + }, + { + "sha256": "0b23f9a4cb09a1f4cc67b9603dbade3ad6dc4f216d06b83e4f06a688c7c514f5", + "task_id": 55 + }, + { + "sha256": "c7d06955b7fd4893bf0555aadbda7aedfaaa38077981964911b91b5b717b06df", + "task_id": 56 + }, + { + "sha256": "2ee4f615555caad541348003db089ed872d7147436e107e3ba4d7fa837bfdca6", + "task_id": 57 + }, + { + "sha256": "e4b5144effde366f2c024520ab1ec1445f66e7ef96342b21768a9b4e5b7c2e19", + "task_id": 58 + }, + { + "sha256": "bf58ee5a45cfd877586297d1557e3d1b56ac567e951687d211d25fc739e5626a", + "task_id": 59 + }, + { + "sha256": "7b27e236b88d2ce24290ed3fc9a53cbb1d7a27fe855d5260a8b9257328887df8", + "task_id": 60 + }, + { + "sha256": "33e134893d3b545343aeab19299e4e6ca19830c425a0a2e9cc41ecd2cfcd6f16", + "task_id": 61 + }, + { + "sha256": "86e84db560a6f6ea38fad389d80f73a073f9521bb876fbae6c7d2ec4bb45755d", + "task_id": 62 + }, + { + "sha256": "9e2f485080777b58b4070f034769331fd080af2d0fd47a0ba0d01e649e8a3a79", + "task_id": 63 + }, + { + "sha256": "472c3560ce1f1ae28fe42f276d5ef28331679a2ef811152bd16260b51a7e08e2", + "task_id": 64 + }, + { + "sha256": "5ed90b65985a25fc82a206389e3637c2880b448dbdc941e364a47a53e86c5b9e", + "task_id": 65 + }, + { + "sha256": "d0e029585fdba93009db39daa3c95368cac8e76c2a09876963ab3eeac19c4fb6", + "task_id": 66 + }, + { + "sha256": "5a5baef60b6a65436227783ad3b4c8d9aae54ff6b04e2c00a02e33c07351e520", + "task_id": 67 + }, + { + "sha256": "a6af1498663b624ec9c5b9c1ab94087c50b6646cc4f21d3a13ab6189aaae28fb", + "task_id": 68 + }, + { + "sha256": "dbec07be6235f84875a337f1919d906eed2665b752413a7cf99cdb48c39c7404", + "task_id": 69 + }, + { + "sha256": "163d6586c717ac7c18d4fe7912f016d6df888ac2e4a5a5a9f9236527b83c3fdd", + "task_id": 70 + }, + { + "sha256": "cae64ed67ab5d6ee3f5e1d087675761ebcb6376e33220d002dab381afff651f3", + "task_id": 71 + }, + { + "sha256": "995a6cca8fc82d0461c3fe2f3664a6539eae8ef1a7e9d69f40fb92380966f062", + "task_id": 72 + }, + { + "sha256": "14ece3e6418c7b26888aa5a38e78457962fb5c1f5a90b6a7a903801512ab7a85", + "task_id": 73 + }, + { + "sha256": "bc83ca7d863ec7432d8446776d0e933464798d7478b1db96d16f0d85d53a6a0e", + "task_id": 74 + }, + { + "sha256": "7e1e7b7ad6e35e1ad9ce612946ad96d47853b0b831ccef00d8d31be29ca11b07", + "task_id": 75 + }, + { + "sha256": "3f925078ef0fae17ffc99fdd1bcbd78bcf27477e6f2207ed4e31a1f5d88a2548", + "task_id": 76 + }, + { + "sha256": "6ab23be621addc74ae3988f9fad8094f429e762cf5d1041ccdeb4731c5aa6c0b", + "task_id": 77 + }, + { + "sha256": "6d753274f7969d468eb521c0ce58a659ec3c0d03786ec53fe60b11131bdf4dd7", + "task_id": 78 + }, + { + "sha256": "4d3e158b3e71580e3a2a2a7fd51b7fe7beca87de6abb1bcde3304a7c73151d98", + "task_id": 79 + }, + { + "sha256": "5272d05dad6bd25ac4d3ad58beba38ae46c94d4b39b1b8705f6da343d6f7e09a", + "task_id": 80 + }, + { + "sha256": "6c570df4ab02dc78eccec01881546b1673c78120d06cd728f98806f4c4a548ff", + "task_id": 81 + }, + { + "sha256": "c2505b8a66671da0a6cb3260c64f2866567ea65180da1033d709c0e10e87ce9d", + "task_id": 82 + }, + { + "sha256": "048ba5c05339029399a264a3aeb716fd4273765ee8b542a0920676967c8b9066", + "task_id": 83 + }, + { + "sha256": "dd9fc6ecbc3597996e311b9f1f003dd103c554037c7b8be76e954abc3f10e75a", + "task_id": 84 + }, + { + "sha256": "cb6b21693cf0819e22a49b3ae15a9d1adb22e9b36ebd7d115392b03c4c9fe78b", + "task_id": 85 + }, + { + "sha256": "f8ff537cbe7d5b7bb67cfcdd988b38fa226b7cb57da7d44681254de9a65e503a", + "task_id": 86 + }, + { + "sha256": "25a4ec259efdcdc017d62698a5d60585b71ef9b02477f1968bcfc35dad86cc6f", + "task_id": 87 + }, + { + "sha256": "771ed5605e7c5e5d9213c2ff044b83a640f323152aad3ef603b52ceaf019f8f3", + "task_id": 88 + }, + { + "sha256": "02e8bf268416fb14180c7b8a8acbeaa4af888cac0eaaff3cdbf7b855e5512357", + "task_id": 89 + }, + { + "sha256": "b98e46a78cf547b27f14587b81ddc8f93303fb0e64e6cec7c568e5092f605325", + "task_id": 90 + }, + { + "sha256": "ab49ac7c8231642d472966e59d95fb39c51782c70e4519f011310cce247b507e", + "task_id": 91 + }, + { + "sha256": "02ea226ff2c90e62967ab5413734dc1a587075f6a12e198f83a0f76dcd628484", + "task_id": 92 + }, + { + "sha256": "4131ab62c6c84221299aaafcfaa5c8fc2d0375ed351b14bf322c8ad642441c7e", + "task_id": 93 + }, + { + "sha256": "d42f50b469810b0e575b7abb8cc8e172249fdefdf99bb763ee9c2102459bf0b1", + "task_id": 94 + }, + { + "sha256": "26572a8a0cf47a7b96b4bd392213e9a66b75a0eb619a7e0eb77aa21a8c5fd833", + "task_id": 95 + }, + { + "sha256": "704746feaf484fbf86ef2a395edc7a7b561b2f08fe27f1bdf6d55bd232d05f92", + "task_id": 96 + }, + { + "sha256": "4d265ae7b7585759134b446e588fded424f5f0350808dab7c5b380d045c3a401", + "task_id": 97 + }, + { + "sha256": "15436d55206e5095462f78bd610aa5d6c1f130625ea63a358ef52cb46a37baab", + "task_id": 98 + }, + { + "sha256": "e60395697035bd82a20b53f424aaf68e19c80a66c2426484fbc204bee08e9689", + "task_id": 99 + }, + { + "sha256": "ab0ff6ce8c83b157c3d2dec65bac320934c9eb69a87dfa12a7f1042d00a5104e", + "task_id": 100 + }, + { + "sha256": "13fca81b40435a74b8b6790d67c74ff0bc2854531ecf825f50edc77d213ec90e", + "task_id": 101 + }, + { + "sha256": "53d3b51aaa3e61c3a9d767bd002396428ce211eb29da068f812f26cf90a71f3e", + "task_id": 102 + }, + { + "sha256": "27ef5636f3e1f33fc642ae45b1535b15e949e8497c1673e597938fa0c34425d9", + "task_id": 103 + }, + { + "sha256": "2924e5101914d9c3a3123c08a0f4ee42067f1722d19d5d9c61a1bb7266265efe", + "task_id": 104 + }, + { + "sha256": "823f28eec0ba779713ce1a000215d105a3040958e59a93357a6da436b461144f", + "task_id": 105 + }, + { + "sha256": "74301b21f36e8486110c706581f472523601c79f4767c4eca7a23aa4a91f7676", + "task_id": 106 + }, + { + "sha256": "380a03031053e153a203e3af252c5863fb268625d45945141a0a304cf775cf4e", + "task_id": 107 + }, + { + "sha256": "d5531ed21205a5d60352855a47b1c627a90a60d9641fc8dc88c3838cef4b7d53", + "task_id": 108 + }, + { + "sha256": "3bf618870f43656c259bb30a7f5d8631282a03c7f519ecee0c64f59b8b2ce136", + "task_id": 109 + }, + { + "sha256": "e6dbaa4a22ed5bdd7ab864942b8aa0c0044ffdd94fc96d396cda6e8006fa45bd", + "task_id": 110 + }, + { + "sha256": "4baf6b8a0444929743dcf3488310bdb4d177fb92e76e535df9730d01cfc0276b", + "task_id": 111 + }, + { + "sha256": "80e3fef8e5d172bdf4d7770fc3ffd8fe02931317096f75b98fb3f7e17cca02e8", + "task_id": 112 + }, + { + "sha256": "f4481235f45a83bdfe00e359b7500562efc26ef38d2e1fa1af2d72998bee63c8", + "task_id": 113 + }, + { + "sha256": "783c5601d9451aa103c5a217e6b965248c2d6f879c41b74ed2b8bb5c6e3ef606", + "task_id": 114 + }, + { + "sha256": "d5421e5c55296231524f67533b697a81d9ca26ccbcc14172a2a3d6d587ffce4b", + "task_id": 115 + }, + { + "sha256": "24e88b1e95597ec6efea83399873a8a72825f19df9a4484c755db9a7c7937eea", + "task_id": 116 + }, + { + "sha256": "13cb9c1e9768119c3dfbad381d1bbdcae8ff027957f8272c75e8e14fc1b36d03", + "task_id": 117 + }, + { + "sha256": "c02627535f4b4c9fdbd8519dd35a26b2354bd445fe90cdad8e8057de5cf0e786", + "task_id": 118 + }, + { + "sha256": "c137e29604fe5876b06630c740aed84d2f807031141687c7b2910a7fa538dc54", + "task_id": 119 + }, + { + "sha256": "9031f102b1230be91e8916ead0e9b156393f7ddcdb70b86aedc3ec9621d3195c", + "task_id": 120 + }, + { + "sha256": "d8b1a4b63046753b46f8d9c949e146687dbd5ac95777c13efdec136323b5355a", + "task_id": 121 + }, + { + "sha256": "c6a68d8757ba5642fc1b462fd1e20ed539d1cca315d63cf8365da62d5412c481", + "task_id": 122 + }, + { + "sha256": "120959347d4ef625b5cc7e7f937fa3fd8b80fa2ff5571279d6ad61fa61eeecbe", + "task_id": 123 + }, + { + "sha256": "f740b3dbbd4c05f286d8071ae99d4e5919acb73f33fc86fe98a3af63496e1a5a", + "task_id": 124 + }, + { + "sha256": "0943210c451ca802e2e3efb463932dca33c936d0d0defbf6545d3f5321d1855e", + "task_id": 125 + }, + { + "sha256": "05a9d552d96e355a7a16c1e4aba2921b4c21edc40f32078aaa8b3a04cc397577", + "task_id": 126 + }, + { + "sha256": "2d2ae75a8a2826a37c80d1030539df94332b535e26f797ed49a3effc3300135e", + "task_id": 127 + }, + { + "sha256": "8c27f7c9a36a3066a7eaeb7109fac13b3142f953ca144770d28ae81495e39d6e", + "task_id": 128 + }, + { + "sha256": "1a844a84218a805c097ef35531319fd808b8f237a6e149805161158a7a7a26b6", + "task_id": 129 + }, + { + "sha256": "999821329bec7721e2a7e36e4ff8146c8e35c0dd3e960de622eabdf90bca0bdb", + "task_id": 130 + }, + { + "sha256": "24d2e2b8a3f9bdf82cc43b49b856bddaf76d05581443513751a2e7231ffd3e78", + "task_id": 131 + }, + { + "sha256": "be99aaba1de6c8108beb5d29a9d228474efc278c5b457e27d6d4b83345358abe", + "task_id": 132 + }, + { + "sha256": "7bd8e14799247cada71c1ed9ea0e893724ae9f113a95a533cf125a892dc30a05", + "task_id": 133 + }, + { + "sha256": "3821e32efa6fb8d5daa3976a4b38b22622bd9c076468ff69877aa79898e91dc9", + "task_id": 134 + }, + { + "sha256": "311477e634bc30e3c38b0d3cd841e7ce46621663fad87b5562d857f229158bc1", + "task_id": 135 + }, + { + "sha256": "ff64235c93476f0e989635d4498b7548c176cfca8a99bc76e8570de2b3fa7e93", + "task_id": 136 + }, + { + "sha256": "9442f82e59db4825384b253025fb5e8e2125a1970d7d7ad16c4bae67e7e8baf6", + "task_id": 137 + }, + { + "sha256": "83ee57e3168feec39507e03d212f840d9abcf3b921c671fca8d1548ce725b6e1", + "task_id": 138 + }, + { + "sha256": "fc5ea864e3ee5f1243c1b2890d89387dd0c5785b8830dd1fb0c3a5b4295c9271", + "task_id": 139 + }, + { + "sha256": "2493fc54053b527aef5cae741556d59c71f4c080c5fd7ccdcfd043df66aa148f", + "task_id": 140 + }, + { + "sha256": "f816c18b373eac8c9cbb867878f08a9de3988e141093ff51a189cfb4a47fbddc", + "task_id": 141 + }, + { + "sha256": "7e9583f5c048b1bc583747db8ee33019fdf2bec68feeedc9d1b0229cf1eeafbe", + "task_id": 142 + }, + { + "sha256": "8f15463fa74d9a6401c928627304dea026e68ed220cd1ab2a367672f5807e6b1", + "task_id": 143 + }, + { + "sha256": "47aa4a7f81473acb006ce6655c740a01cd2595cc6c4953a40b2025c70f51912c", + "task_id": 144 + }, + { + "sha256": "28f022689390fa11f26734e58f3414efff6cbfc70f8f882c1bbe5f46e9d84b56", + "task_id": 145 + }, + { + "sha256": "c49fd3f990c0bd76dbe7743a03e7f1e7480d3629fe86662a9b85ea61cbe457ed", + "task_id": 146 + }, + { + "sha256": "e062544f369a935a41eecb5903781645eb88074292e964972f16d443b0d6e2ca", + "task_id": 147 + }, + { + "sha256": "11c0ab80bc80cac210b3735ee7e73d38846f2148d559ef528a2e296ef44dfa83", + "task_id": 148 + }, + { + "sha256": "c237acdd9bb305cae18bac0bfcde60262573e07439c17f6258a63974e96a1e78", + "task_id": 149 + }, + { + "sha256": "55b7f2aa839596e044c5c692e22cbd32f684cd11ba0fad7910fb3fcd7b4e37c0", + "task_id": 150 + }, + { + "sha256": "c5b1cc0e09d58f9ae5f926f1c97708c3d67f7d6d0a8fc9a1ee0d75fa5dd6556d", + "task_id": 151 + }, + { + "sha256": "4a6f20274f7f8623b19918c5e1ec2f32764ebb13c29a90b0756980ca624ecaf9", + "task_id": 152 + }, + { + "sha256": "d7489cef07092c64f7a36224d0b463b3a7414c9a635399c653740ed9a82c2512", + "task_id": 153 + }, + { + "sha256": "7a09a77ebef8461c3df5ca0e0b94bdf8193b21661b8c7288b640582105776a4f", + "task_id": 154 + }, + { + "sha256": "050beae947159ab1382fbdde49f5c257e0c4e32127c710f3b2c10e9c02d4de94", + "task_id": 155 + }, + { + "sha256": "476ebc7da4c853e029a3f2b5d7291818f6ffe643f13db4125f2a0d9642111369", + "task_id": 156 + }, + { + "sha256": "2b3a442fe56306279c7308c6d5f17c0f7cf0d8201a702fea94fa6b27d6b6f6a7", + "task_id": 157 + }, + { + "sha256": "0741a465476a7d71c6482d4dd5f74edf629f2d7e2c020ba50337ef409bafcd7c", + "task_id": 158 + }, + { + "sha256": "2000f1d47fef625b730a76ded04fd24dfcf2de212f9464c2ad3081ba3adedada", + "task_id": 159 + }, + { + "sha256": "54cd63229b66bc905c46cb12f231a0a7ea808ce86c353cb2f73f3b3d33843f9a", + "task_id": 160 + }, + { + "sha256": "21cb75acf3c93f065303983a4a818bb3c319fbe5a6f03234c0ac261217535718", + "task_id": 161 + }, + { + "sha256": "e9b3378fc810d27fc3ea9808c2cfd663df99f47b41b0f8461c05d964c58e3487", + "task_id": 162 + }, + { + "sha256": "2678238a3cfddd410a5beb44ad9567ae7023542fe135b78aec309da00d01b4bc", + "task_id": 163 + }, + { + "sha256": "4a93f106b7f6b76d78355179b60ab6fc1270661a7da19be801a794b11c5ba4d8", + "task_id": 164 + }, + { + "sha256": "29bcc66380a822cb34a4204390989eea5d1d6de7c05be38f825bc186f0963f62", + "task_id": 165 + }, + { + "sha256": "c55f5b34a8a2f8359e0db294ff4eb6dc150c8bbc0f05fbd0e58f2bf213f8970a", + "task_id": 166 + }, + { + "sha256": "e3c5cfd263b198cce950f1bd5dfc1de998cad87d4ea39709a41ddcb590f927d3", + "task_id": 167 + }, + { + "sha256": "8c753db16553a82c3e893012632cbd5a82ceb0348cd2b620c241e4277fc8434d", + "task_id": 168 + }, + { + "sha256": "51103c0568757a15aca3288c8690ddba84d5ad4b8a5d4a173b3d1498dba95968", + "task_id": 169 + }, + { + "sha256": "5a7130e56261023a7a9af9647f8353bfb370a7b806ea5fb68434f55f0fb2491c", + "task_id": 170 + }, + { + "sha256": "bf803d4376ec1be41854a125a47cc5bf8ac49a52b74df6f5e4f0a67df8a8a59c", + "task_id": 171 + }, + { + "sha256": "54c0888dd2c8c2aa99c21e526153d1cd4353981c2ced2c96250d69bdeaf095e5", + "task_id": 172 + }, + { + "sha256": "973c9f7c8512078d946c5ee1c2971cc01f4da88862ffd48c2ffa6929f121f5bd", + "task_id": 173 + }, + { + "sha256": "df6608a5d3f60cdb79c16b56241cd4bbac523d593dd0ddb898b64627c132a808", + "task_id": 174 + }, + { + "sha256": "c44b6fa384ad71da45894353369f1aeea685aee9a950824f7edb0d09a8e112fc", + "task_id": 175 + }, + { + "sha256": "170d0225314287b10082e0cc4618858ff5b518a2c5a2bd2016882d4060874593", + "task_id": 176 + }, + { + "sha256": "1ad7fbb80912869942b7c1a6d909a1fd0d7b106f94dc1b92d85a6e57f9d926c9", + "task_id": 177 + }, + { + "sha256": "8fd023bb68accffa9114c792d76a4aae2a872e496cea74810d5beeb02d533e1f", + "task_id": 178 + }, + { + "sha256": "f63d6519ea4be2358a70497b66d1b10442ff892e5d93c853c237e6edf2eda88c", + "task_id": 179 + }, + { + "sha256": "b90a0fec32e4f57161074a84ad7804c6be794ada3c27b2d00bc9e76ae3accfad", + "task_id": 180 + }, + { + "sha256": "a6241a1b151f178fda03690b01cb31effca05fc8be21675eb98ca7dec566c156", + "task_id": 181 + }, + { + "sha256": "0cf1a5a203bf22c21f174c5f9c3f10b78cff0e00e893cab3c2d37d10ecf37273", + "task_id": 182 + }, + { + "sha256": "6c045b07b241b62f0116414e0f2de5f747ce2664706bd53c985a447eb07a1add", + "task_id": 183 + }, + { + "sha256": "fe8432431767e168a054cb9f7579a4117851eb1987c3829256cd5324b7d0abb3", + "task_id": 184 + }, + { + "sha256": "d3b49c071fd026a4fb97766d0f2b5110c702e7cd1c748a2250b60b8cb19c3a2b", + "task_id": 185 + }, + { + "sha256": "f9c72af9d9283176427f75ad58d6cd3f388bc7fc41704e9ca6e1a393c5985dbd", + "task_id": 186 + }, + { + "sha256": "3bf24e849f6e0d54f150ad902ff274f9556761d0adcb32a15d3ce0f396121ee3", + "task_id": 187 + }, + { + "sha256": "ca2004404c39601a10e21abf078fe1461bfca104acbab91eac0c6fe541bfcb69", + "task_id": 188 + }, + { + "sha256": "9c04c3013905867455d36aa1d2fb472fdb6b9dd47bbca70049fa21682a7175b8", + "task_id": 189 + }, + { + "sha256": "2e45424e9b432f19b9a51f30db80fb2edfa6bc1b129b5b2dacab038abc6cbd9a", + "task_id": 190 + }, + { + "sha256": "85adc139072b74675a71fbaf07c4066a772fa25bc44672d46c3b810a3896f539", + "task_id": 191 + }, + { + "sha256": "9d023e6c419c566332cace172fb5514db165ab6e011792faac2cbcd4bbe05833", + "task_id": 192 + }, + { + "sha256": "8553628cfabe9185cb6b6e09f77b72f6d90b7f225246881ccfd1ccc1e9b288cf", + "task_id": 193 + }, + { + "sha256": "a2734f1c04fd7e6dfe5d33b3717bfbc7c5de9dc21816a4c4abef3694096e5520", + "task_id": 194 + }, + { + "sha256": "e8e12d64fb8e2e32c3302027fde3e77976401283f2938fb314ba191541dee142", + "task_id": 195 + }, + { + "sha256": "2364b4a4b6c7bc97cb76c840bfc077f13d44f79813d8366a2b215b54ddd30037", + "task_id": 196 + }, + { + "sha256": "5eb78a0b48e90693efd1cd7f32851e57d032efbd6e05db3f6e43e6591124a726", + "task_id": 197 + }, + { + "sha256": "8fbe6d489ae07c7606f6b837e463e92df271ff0e4f3e765dfa91561ef4853269", + "task_id": 198 + }, + { + "sha256": "38ef3f4ca2766b9a4259981e7a5598b733f86a00f2fa9ff78a84ee130b8a428b", + "task_id": 199 + }, + { + "sha256": "70ecac3fba135b8f75f627c59920c2bdef412d85a3570d5cd1c93e67efbe90ef", + "task_id": 200 + }, + { + "sha256": "fd61c5f1bc5a385d917a1f983e1e5a330b55fb12d5933459119980a4904526c1", + "task_id": 201 + }, + { + "sha256": "f89d460da6b0853684386bfca30a4b42448878e8f9636de3770500d0b74fbf1c", + "task_id": 202 + }, + { + "sha256": "80f11f6b5057b2e466db28155459822e0ffa253daa879e9c13ee4628a7685814", + "task_id": 203 + }, + { + "sha256": "7cc78f32fde8466cafc6526297f450feb431db494e62d8a97a48d0b8bb238979", + "task_id": 204 + }, + { + "sha256": "71a7ad804a167d337188cb02b93f3401322e35823289b1d1b3c372a3167379c5", + "task_id": 205 + }, + { + "sha256": "5238b2a3719afbfc752042fe5802df5a2a34ecd84c0190f4e33238db9a9ddd0a", + "task_id": 206 + }, + { + "sha256": "43c8a66107dacec77b40ac8fe1c4b27350b4f5a742b37ac950f285bcc5933982", + "task_id": 207 + }, + { + "sha256": "0d8b71f7c07efa5b15240e33902b5ad7eac144ca40efe93812ccfe124df91741", + "task_id": 208 + }, + { + "sha256": "6de8256560ff8299ea962a76b585ebea5661821512d1a65d7d8cef42d5838437", + "task_id": 209 + }, + { + "sha256": "2bd9bb0e36b7cd08830d193235b67a933ec08895dc2aeb60b3fd88f48526ca4f", + "task_id": 210 + }, + { + "sha256": "3843729048980d804ad35bd60a6526415c4a951e15459ac60324f4aab3f9b6b1", + "task_id": 211 + }, + { + "sha256": "cfd41fddf21f2d186644f2ba72b29acca18b502a49f8b3f9bd188c4243d97093", + "task_id": 212 + }, + { + "sha256": "dbb88bb03d414fc310e2a59e7abf3e70382f4552ecd0b62d7ec8b7893f787fa6", + "task_id": 213 + }, + { + "sha256": "3c4941b00096f46643bce962a62672bb37a59d6e1e309c606ed890e67f128b7d", + "task_id": 214 + }, + { + "sha256": "327ee93fd78735f1a6511c84e74921fb96777828ff8ed765e515c716e525d761", + "task_id": 215 + }, + { + "sha256": "524227a9717f5911af7a477f3669e4ecef72c20c48f3d32de9b8c2a91f423b97", + "task_id": 216 + }, + { + "sha256": "cabea26272b503e37aed663d1581153f7924c4575306163cac9b9a4f19edadb3", + "task_id": 217 + }, + { + "sha256": "6517220c5a05a0cf7720b8a6a3184da3a997439dc0a5d12d02fd51106a42e053", + "task_id": 218 + }, + { + "sha256": "14e685dfc0764f4e060ce081721570448a4cc05648ff974d00f22bc8940fe324", + "task_id": 219 + }, + { + "sha256": "82a360c9a073bfbf0059d99a5d93d1764d99bc9c9899010b8ba3794e573d9680", + "task_id": 220 + }, + { + "sha256": "dbdd12ff871833c0de1f1d5b1709aee344009fde6dd1bc812b73e1e9283dedcf", + "task_id": 221 + }, + { + "sha256": "690346901931e24a3fabae155ab36455a69e95e07980fd8dc88f2f979f61c978", + "task_id": 222 + }, + { + "sha256": "8708a014dfd0b446c90b258c77775c8fdeb032fbdcbb1ac74455a04bb4c1f504", + "task_id": 223 + }, + { + "sha256": "e5b185f6361b38031c3524e28324c5fb3a572018bedfadf748d19226a06afe6b", + "task_id": 224 + }, + { + "sha256": "f703efe5275f4a094d86256b8868306650706216405077726570522bcf3c2f83", + "task_id": 225 + }, + { + "sha256": "246d1121aed5f93e5da420c60c98df579736c59a8fa005e5f16fc582248714a2", + "task_id": 226 + }, + { + "sha256": "2463cebfe43ad6bdeb076292da2d6eea5e76052aefa60150e06bf33a3af51f17", + "task_id": 227 + }, + { + "sha256": "edd7c15994fdc6efef3334ebba3b2b57185e82f10983df8119a7ebbf89db61e1", + "task_id": 228 + }, + { + "sha256": "e610488735f3f63f6065ce6e7b3151ff2e7492d5606f4c015aec822666db99b3", + "task_id": 229 + }, + { + "sha256": "b9c3b7788b79e161ec4e1985a6fb75d15863e84067865708ce955cc0abd63239", + "task_id": 230 + }, + { + "sha256": "2e2148af44e0a8f59a15f3cc826038985963f85930524df2b9422a6fc0a7e39b", + "task_id": 231 + }, + { + "sha256": "1d28fae3bb1335d765a48bca116ff7460f1ebdb9be535bbd16ed186dc4bd529a", + "task_id": 232 + }, + { + "sha256": "2e7b8df9ccd380e69558721ee2ee142603009dc43f390a49f5ea4582b2f3fda8", + "task_id": 233 + }, + { + "sha256": "f2559e68b39ccb8c5c421b3b5a38a413221834e3b9869aa2cbf64660c7404828", + "task_id": 234 + }, + { + "sha256": "8f6ba9aa1762ddd7e9f83ccadf883d8f73dd84e4eb162a9a880d7dfe6d71cc83", + "task_id": 235 + }, + { + "sha256": "c91fa6515e6861035b85ad8fd8fd7318ae06a35e1fd8013925fa7369585f2503", + "task_id": 236 + }, + { + "sha256": "550002fa0cd08ab6e51efdf52891b8d51ea205ffe26caac5f75a6a24912f37bc", + "task_id": 237 + }, + { + "sha256": "bf55d5e07d2feefb499afe94dd6caf1f6be6bc2516c3fe1cacc2181f1918309b", + "task_id": 238 + }, + { + "sha256": "2a7f878119acf25d53d17fb549eac1899fef52ad2c9b0b60768a0e718d084bf1", + "task_id": 239 + }, + { + "sha256": "f93c9da1a5de9648988dde7f7a9ee71d2ac295768f4b8d51df5da78d3c2d8437", + "task_id": 240 + }, + { + "sha256": "c18ec2070de3cafde537543d5b16293d8212b4d38e36ccf2ce43750706792747", + "task_id": 241 + }, + { + "sha256": "d827a98c206e55b21cb348af83262379bdfe21c8cc65a058ed772481a406c4b1", + "task_id": 242 + }, + { + "sha256": "e656e616ec4aa4d7f30ef3d9ed925a5ca7743043ae96df47537342a05f5ab473", + "task_id": 243 + }, + { + "sha256": "7691ba2cf1ebcc1a1b0f2124fd7b21b830d0a2e008f454f07e5230ad1f5ebd1b", + "task_id": 244 + }, + { + "sha256": "20ef0092979a437ad23865f6eb294ada45c156cc78bc8a256f6bb5c15de13691", + "task_id": 245 + }, + { + "sha256": "5b8fe1ed0503d1d96b6276a5ab726c91d5711e7971a5a429c6090cd5b5ab9163", + "task_id": 246 + }, + { + "sha256": "3d1caef92c89cbc99842120ed162e911c3d179b7ae1c3f30e24072b5aac8e7d8", + "task_id": 247 + }, + { + "sha256": "147e7afc930ea89ff1143aa1cc6540bff8100bebe03d60510971d998e7ec96a9", + "task_id": 248 + }, + { + "sha256": "fc2181ff89e89b8c1246a56d638e76a6ad294568f1abc504b1c7a8744c038200", + "task_id": 249 + }, + { + "sha256": "15d753c2d9a994f5a5e631dea8fa1c725725baee2aadbd4330487cd8b9b1850a", + "task_id": 250 + }, + { + "sha256": "ebd4fc8cb01abb17bde8acdceca42b5163837611a7ee33129174c467f20adcfd", + "task_id": 251 + }, + { + "sha256": "19b799513e56cd92490857a42b3a105836c51cd66da7c66a8eee60cd68a9f79e", + "task_id": 252 + }, + { + "sha256": "0f58e14c5ab619e1295761a91ac1f2259eef0a0b5c083437daa886b284cc7a63", + "task_id": 253 + }, + { + "sha256": "319181b67d9e98bf63885418561f9a991cbb12729fe4e212ef83699dc921cd4e", + "task_id": 254 + }, + { + "sha256": "9125c82eb74146a4cb85c693e800fbd93b4a3527c83e001ed6bc35e39467f2e7", + "task_id": 255 + }, + { + "sha256": "b4aea978719f9cb1475f221854af913288c28c2fecb2b79875d88d8500b8cbd9", + "task_id": 256 + }, + { + "sha256": "fb0e85415d4a83f4d55315b84d5dd022ad54adaf7b3a5c40e983cb52807ee2e3", + "task_id": 257 + }, + { + "sha256": "25f98f17f1c15116fe7646d8270fb8afdf7887d7e3c4e6c2cfb0da7005d6c717", + "task_id": 258 + }, + { + "sha256": "b7dd5b7de8aa2b07015c9b0caa2d7fe99201b05f3f23521b28a635e0e0a8d995", + "task_id": 259 + }, + { + "sha256": "96f0dd3617f4d1e9777ce7ef83adde00c7815739328b6aa86069b19dde896194", + "task_id": 260 + }, + { + "sha256": "802c0b39f263187802fd2c0b92e8d97ebece7ba7d21a386c242c258807cba3e9", + "task_id": 261 + }, + { + "sha256": "3f346fc5605a2d7d23e96673d5da90c44d1e84ecdeb5ada0c5bd3ba3897658e1", + "task_id": 262 + }, + { + "sha256": "1e7358836748876bc2670a0fa1b35c4d052768be12b47137ad27f6a54c31678f", + "task_id": 263 + }, + { + "sha256": "bed4d32d90966acfe2f2213526611e2ca7f34ebcc722a8a597f02bf6299ac915", + "task_id": 264 + }, + { + "sha256": "b569fda57ca6578d8ea0e8eed4b42f69496401491e8a852e3451d4c5bc2b5363", + "task_id": 265 + }, + { + "sha256": "16b0159a777674d34d73bd163e4ee24bf3dc71ace62eab528f80419afd706086", + "task_id": 266 + }, + { + "sha256": "94f0f0e9a97f3aa87e7a7ff98309cdc36a4a62f66f18379e2aa1143a947139c9", + "task_id": 267 + }, + { + "sha256": "7d40ffde20e0204fb1ca47ded9a5effb4d50b907e25065bff7e4ba1438df0fca", + "task_id": 268 + }, + { + "sha256": "b16f4f5ef64cdd19e5eb481ee1a66aeada1981774c79ee031acc1ba1395c5fb2", + "task_id": 269 + }, + { + "sha256": "1a3077a7c720fabd63eb4ff1bf361b34460561c2bd3e34f8766954bd21c76196", + "task_id": 270 + }, + { + "sha256": "f3e44b4e61a94ae392ad260bef41f3a02538825713cb3afa148024e958eba59f", + "task_id": 271 + }, + { + "sha256": "806519fddcd7a32defd20e48d6a0a3be33b61f7095013b29fe783885f6bf2703", + "task_id": 272 + }, + { + "sha256": "cce63f2198749f630e00a4fc7335e1b30de019090ea8908e3a643687b1e44323", + "task_id": 273 + }, + { + "sha256": "6e6b04d34dc592d25d63262dfec633a73205b7476ef1c84e093e2f3b063b2fad", + "task_id": 274 + }, + { + "sha256": "576831528ac961df9df2e12e2d5d22934b36b9cb51e4867e28d10b18d01c1ad8", + "task_id": 275 + }, + { + "sha256": "4ad4ccee8051eaf28b2af36b9ede35f74b404016e95d3424b482c0c0908b12d5", + "task_id": 276 + }, + { + "sha256": "d524f85f1ab6d99e7c617433346c738fef31d87c4f18f14a756e079f16186d9d", + "task_id": 277 + }, + { + "sha256": "822b142b1294dd9bf50e31ef00548b691bd57a5ce10ea23c39899ad12abb9e96", + "task_id": 278 + }, + { + "sha256": "8bfdc3cfb3f40a7e0dd37ff14ea1d9cc4ec100dd92b86d941ea30df203270476", + "task_id": 279 + }, + { + "sha256": "3703231f72aa5bc7bdaa8b8b18fa7888e6d13f8a63df01eaf389fa314d1a441e", + "task_id": 280 + }, + { + "sha256": "60c0524fb232761fe64317053466c70f694edfa77082ea73b9a49b135dab8cb2", + "task_id": 281 + }, + { + "sha256": "d2241d1a85127c1659c23f44041983b19ee5cbcb95a5edb69c36c0bb96b8d4c3", + "task_id": 282 + }, + { + "sha256": "5490874303af7a93a33964466fdedc374e05f7dfab331d4c6e56fcc65e51645a", + "task_id": 283 + }, + { + "sha256": "0c10c9d1d4ae8e1d29f82b18dfe0236dab44ecef6e5d8efa3317016269fe5e78", + "task_id": 284 + }, + { + "sha256": "13a8732809db094d657c5a3a00749c51ca10c8db8ea685b1c94cc34b33171193", + "task_id": 285 + }, + { + "sha256": "72af9eafb8501ec4f30b44727150964a569f93eb641b9e91c32d11f3e8ad73dc", + "task_id": 286 + }, + { + "sha256": "b47f48253aa1fbe2ffbd95e8a56846acc6a4790337ca4ca4ff67dbdca6ab1b1b", + "task_id": 287 + }, + { + "sha256": "15f880a7c4af4fa97e8b93df0a0df6cccc81bc93c12e252787e9781e71ffd8d1", + "task_id": 288 + }, + { + "sha256": "afa075279a3217cfce2462447504a298347c489c0f867008390e02908132a9be", + "task_id": 289 + }, + { + "sha256": "2060112b1440f412693512863466676c9ef1927748038e24e0530881aaa69e4f", + "task_id": 290 + }, + { + "sha256": "0835fbfe45ae0bec588efaf8e677a4713266ff7f33f5411983e00f21968b4fb5", + "task_id": 291 + }, + { + "sha256": "9c4497472121f46364681d46b51f65fa3a7e43521276d44fdd4184e8ce573bcf", + "task_id": 292 + }, + { + "sha256": "eb5394c2209d93efbc45a1127bc14626ccea628c01d3726840c73f623140a68d", + "task_id": 293 + }, + { + "sha256": "f6018aaf61c665641c8431592203e6515d1195aaf03084b62a3002bc8faba1c0", + "task_id": 294 + }, + { + "sha256": "bedfe0bbd89ce95272cf7700a98eac14cc1f5c2f6c8c7459d6644a06aa2e5cd5", + "task_id": 295 + }, + { + "sha256": "7deebed09b0a00080b7aadd114efdb7657ce3baecc019faec749be5c2d3b2de2", + "task_id": 296 + }, + { + "sha256": "e334bd45b69d5cfab68658b7fe59b2919b15e3b68c211eba9dbd102adcee8e0d", + "task_id": 297 + }, + { + "sha256": "e4536d3708847cdd3d0e298961a4830cd7f86e2f499a1062998d9e98789f36a8", + "task_id": 298 + }, + { + "sha256": "8d25b6ef08a8181222780ea1532dc38561062a14da45202a82eb0bd8ee37ff2e", + "task_id": 299 + }, + { + "sha256": "21ec0578ed01492eff92d7356d316f6ad27869a479c1f3149c60358c7c908bc0", + "task_id": 300 + }, + { + "sha256": "abcb9938ebceccdd541a15886dd56ba549c2b5ee386617dcbddfdb6a9bafe21d", + "task_id": 301 + }, + { + "sha256": "3fbf0e50f66ad0fd77b3242d9a0fc02108701491b86cf066dc053037181c4dd5", + "task_id": 302 + }, + { + "sha256": "d66028b612c489097eb709368c5b1864d0d7ba837867dac091105473542d1932", + "task_id": 303 + }, + { + "sha256": "588a90ee693ca462a2a30bf4afe3d02e94e215b776b6d932806a82f17b20a4f3", + "task_id": 304 + }, + { + "sha256": "599d437f9462248fc5d60f70537f9d7683b82dcde18422652a04e7c6e8620568", + "task_id": 305 + }, + { + "sha256": "4ec6aed035b183f25a548c65e5b26ce36e2f881620eb28f8dc8e922be6a25112", + "task_id": 306 + }, + { + "sha256": "74678316d4cbf5b251099bc648ccf8600bfad13108bcaadedb1e87f7391e8d15", + "task_id": 307 + }, + { + "sha256": "028339f380ad488f5567761626cac374e89f51f35dafc06c5f895345759eb07b", + "task_id": 308 + }, + { + "sha256": "80245ee3803cdf01b99d7ca0d0d45f7c5098d86df2dc435d6b9c6174e257845d", + "task_id": 309 + }, + { + "sha256": "ff880d9097ac27a2fd7eecbd39dd0e340c60ebe44d6027e247dd62dffc46d585", + "task_id": 310 + }, + { + "sha256": "8c346f5684e6b5944b02b2f3654a2928cdb50a7d33e4199e490cc0da388c6fb3", + "task_id": 311 + }, + { + "sha256": "58358d84fd7305a04e0f12a77fc5d46089c3e883a35fa2d3b5fb3f9644abb1c5", + "task_id": 312 + }, + { + "sha256": "43790774f6b1b218acb5e2cb4538f217b18995ddbaa1d707b8f0ee5c4e310d2e", + "task_id": 313 + }, + { + "sha256": "2bb31a2c3964fc250d32e5cd7747693954ba5984a43a2698e511fa8e953322ca", + "task_id": 314 + }, + { + "sha256": "f1643afd2b1e000fee72880930817911457318719ea04677edc79af740848be1", + "task_id": 315 + }, + { + "sha256": "c2f38ae4b96890e38e2d0bfe611db2da6bb93434447533c4d64cc7522196cc7f", + "task_id": 316 + }, + { + "sha256": "2ab1afc0469480799d6158f98e0ee06763b57dd84fa601d88b6ee706f412be60", + "task_id": 317 + }, + { + "sha256": "fdaf4c762208f99a4a540aa92c20905a33a01b997ad3625e3806c723a15fc285", + "task_id": 318 + }, + { + "sha256": "9ef0f43909b7de780b0ffdb21c92d33e51864e863b0fccc19c2abb7e072c12b8", + "task_id": 319 + }, + { + "sha256": "fe7cd8c6925f0ac78916814a8c8c4804912d423d41806580c96ac9b97027740a", + "task_id": 320 + }, + { + "sha256": "0198e293bf2b11a8c08ae3d8e76dc18bfa522d371f88dbbbdaad0b756b7efac1", + "task_id": 321 + }, + { + "sha256": "9619394380f45f5665e32aabacbe96d7aa1b80a8363e4021ee061b919678eff0", + "task_id": 322 + }, + { + "sha256": "2ff2380786352d4e1dac1e7803d08859f1cdb72f447266ee4c901219e79faa8c", + "task_id": 323 + }, + { + "sha256": "4e475ac3b9a4bb472188bf02ce53fc3662bcb43d90978493c36f6302cbd6ddc9", + "task_id": 324 + }, + { + "sha256": "f853790752583f64e086fa9167d179cb017bc71165b02a5d4057805f37d1a4fc", + "task_id": 325 + }, + { + "sha256": "5ec6c2b31ecaed6b80a6b7d7ca25ddc183094ed2971b045b731a13868eb9bb56", + "task_id": 326 + }, + { + "sha256": "40dd8688924b4f5d02d081e819b9e3c273b5ba9c58506f817eac51cfba08054b", + "task_id": 327 + }, + { + "sha256": "5d1fb48f0ca9f4175aac5c8d5439ce49ef46f22cd23cd1e24b2218d2a52943d5", + "task_id": 328 + }, + { + "sha256": "ed7f77aead2d8462672bad73e6e34bdc51b25f3977e7447b51d0374e1d371a06", + "task_id": 329 + }, + { + "sha256": "6945f99dfd906c96a80b742a3fe71a33fc8ae3e92dfbbcde5211c89d533f29fa", + "task_id": 330 + }, + { + "sha256": "e09fe0cc4eb796ab0174c4dd1a4e20f986dfe3766c230b8a1f2e9c6526739df5", + "task_id": 331 + }, + { + "sha256": "872a0d97802e3edffb25689f2d9ddeec1f67beb62ed243b0b8cb6e59020277bc", + "task_id": 332 + }, + { + "sha256": "c807de13315dd573f06c0cd4ec1d6712c61d85f5a321b2d5ca92dc9575b706d5", + "task_id": 333 + }, + { + "sha256": "4c0be99a89e67cecd3c3681ce1b06ad89d69c9dcb80ceb1902e601eadc88f1fa", + "task_id": 334 + }, + { + "sha256": "b906e34a5a242e20e424540611ced997fa24388dae5f07549613c69dbd836a0e", + "task_id": 335 + }, + { + "sha256": "5aa1a0b5a0f6a623b3993b590a5f4f4e2e6fd0d53d11860496e4b8377415e719", + "task_id": 336 + }, + { + "sha256": "c84e776c68c5cb666f778e5f676116f0190d9049d510de25c4eac7a9caf8a0f0", + "task_id": 337 + }, + { + "sha256": "cf0a0deccd52af8605f4df1ef7c10307443ad90458852fe5e24642086cd2645c", + "task_id": 338 + }, + { + "sha256": "37db2c5de7780bf16b7e638b6c8ae3be007f0dbad9c58dc709818b7bce3b727d", + "task_id": 339 + }, + { + "sha256": "ebe35ebda7a0e46c9a96ed2f2a1bcee80bae946c075d53720431d77d21752121", + "task_id": 340 + }, + { + "sha256": "ca0764e6c60a9b68db2b9563a408398a9b2d807f0f54e9ded016c60079788f6f", + "task_id": 341 + }, + { + "sha256": "77dfa50cafb3690ca8156c8b0070b0da8982be0c3ecfa125f404c01588480bf8", + "task_id": 342 + }, + { + "sha256": "78a922bfac4c984433ae881316475f9fa6cd475647a632793e0e9d71ba543d22", + "task_id": 343 + }, + { + "sha256": "00b8566589e206d13beedffbe6c2eeb4f567f29c2787302d13699ce4793ca810", + "task_id": 344 + }, + { + "sha256": "913fa20da1f77c6cca78fa2f10425206f132af2fde9faf7b6449bec8fd8db024", + "task_id": 345 + }, + { + "sha256": "8a93ffc38e9de1708863464313a18914b233cef42d84973231263c6371e287b0", + "task_id": 346 + }, + { + "sha256": "03707dd85fe7378082ea92eb0443e2d5fbc7128215a23d4eddb3e86542b746d5", + "task_id": 347 + }, + { + "sha256": "d141b5e2afaa5926bd1d607be3b4bdd1eaee1b4ff0632f45cee48909f43191d3", + "task_id": 348 + }, + { + "sha256": "4433bb86b5b45ce189af3ff4689f2fbd20ab2033dac61b4e06fe7d79840f32b8", + "task_id": 349 + }, + { + "sha256": "15e99544384c556e20f477bbd4544dac723f6445bd74ea34be1e6fa45335e465", + "task_id": 350 + }, + { + "sha256": "25eb09f36f9af217505397e3d8c2f4541caeaf8b14613b41b00b51d4415a0b50", + "task_id": 351 + }, + { + "sha256": "bb3ea61faf6d2daeb717914d6ab85279681b799f7fdad53c1a105476ad97b37d", + "task_id": 352 + }, + { + "sha256": "5c261a73d5152778f8029bde8f02ae280c997774be6f57245cb99dd24cbf0318", + "task_id": 353 + }, + { + "sha256": "49aa39ce941a3521e4c88a04e89c403859b034730db2c21d4dbeaf3c3183572d", + "task_id": 354 + }, + { + "sha256": "0a81815f31a8c0502c4455437f79af1c3b5b28f4bd99c7321adbae925f5f5ea2", + "task_id": 355 + }, + { + "sha256": "a106a9f410cc8f6a345a0639f06e8b6c83188ee6a21e79792d95e5419c3a04f1", + "task_id": 356 + }, + { + "sha256": "a7574ed518bb42a9f894897873e164586787e96451e5d78a7076bcb63e23189b", + "task_id": 357 + }, + { + "sha256": "edd2e0079d54976ddcfe9b20b6b5919906cddf6e94f5500c4886995db78276d2", + "task_id": 358 + }, + { + "sha256": "d157c11ac76aa251062ee6b1fd108dac7868843530d82e7f167a526d27e893b2", + "task_id": 359 + }, + { + "sha256": "9aab2e8bfdd4f3b137b4448eed06259384ba45692a9c058c37abda10061ddfac", + "task_id": 360 + }, + { + "sha256": "7ad6cb2cc83b07965ec36dad914f07ea4e2310712edd124e319b77e3ef76287c", + "task_id": 361 + }, + { + "sha256": "cfe68ec821203a89385791481376e73690c9619b1efb1c9989d04a1180dca411", + "task_id": 362 + }, + { + "sha256": "1d9c11fc7b028311854ecc3974da899e2660b31d4fd1fcff2c97cf1ee3a0cd2b", + "task_id": 363 + }, + { + "sha256": "942402c8d6ab5c8dc057df573756222f7745878e239c0d0ae2af0266d179b341", + "task_id": 364 + }, + { + "sha256": "7f1872b2a89fc624016b5dfc33a04b02241bd12a611a9f278bae7fa860320a88", + "task_id": 365 + }, + { + "sha256": "210e8e3f73c48d3357f58926c645857d6058a68ce1b75fb853e90066f799efba", + "task_id": 366 + }, + { + "sha256": "be7456d2ce42dd05c8061f226b98ebf8408c45ce1f89f188ea8f772eadfba369", + "task_id": 367 + }, + { + "sha256": "97d36952d01115c3b56fb45b6648ad273129e18391a586c5c2e8e2e48670eb8d", + "task_id": 368 + }, + { + "sha256": "8405b6d90b1688225a891f34dfbba0581dab88ec939968d3514c0af832681135", + "task_id": 369 + }, + { + "sha256": "741ff253a563fd3a46736ae470a5e5636f4058861e7b44187f3fee8609d0b2c7", + "task_id": 370 + }, + { + "sha256": "dbb6bf44b681816812d24e1c85ef8586bed38cdc72bebcf135191851313f8751", + "task_id": 371 + }, + { + "sha256": "d77de30dfb38d286a7c0dc7dec657b8deb688b2b3ea4584c66c7a4b3fccaf09c", + "task_id": 372 + }, + { + "sha256": "f89131c0c70c00526ced382a6d6c58ea41dd8d892a851aca6ee89d9572a132f5", + "task_id": 373 + }, + { + "sha256": "352c2126fb8c014a14d94a19905ade9bb210c1555c5ac1cc8094ee5375715d98", + "task_id": 374 + }, + { + "sha256": "e485cf48124dbeecd2f3a01150d5dde008cdf88ed1efcbd975c3c1698b72f87c", + "task_id": 375 + }, + { + "sha256": "c721c557341deb2cf72c38c7217793c15736b55e8450c1e11be69b1ba8c6c666", + "task_id": 376 + }, + { + "sha256": "dd43cd1c8d6f20da3fffe9a9ea873c65193e9dc997a7c7c3e611cb00473cc025", + "task_id": 377 + }, + { + "sha256": "12d431116661022dd07545451d9d1386445ec10993213984b29477b03f7d62d1", + "task_id": 378 + }, + { + "sha256": "8f1c3a8f42992424e6aa70fef206a804d9b78d03d8c2948631e6b7239a525bcd", + "task_id": 379 + }, + { + "sha256": "efa85785210bba60bc0f521b47e1d9e675e05aba2cb5f21cf6c81a27e18f80ba", + "task_id": 380 + }, + { + "sha256": "c12b111a98846bec34098e03974c9f2c0c9c1212f6e4514875a4f6bc86289233", + "task_id": 381 + }, + { + "sha256": "dc9182879b66972afe02eeabd15113ab7e2f1e1568e0e68db6451773b218d64b", + "task_id": 382 + }, + { + "sha256": "9551f88c9d8b8d41d71ce0ebd47ee6643bcd2409a8369059c58f3953be694837", + "task_id": 383 + }, + { + "sha256": "628d47d0b2a72f70e18edfdf85ad30fe9a428f15b128ed0efc0389ee102cb327", + "task_id": 384 + }, + { + "sha256": "a6d73e35b8d8a66c7924c1dd38ba57a884667055995d626e51007e038b2a832b", + "task_id": 385 + }, + { + "sha256": "e7121637308f5b2a366b793ed7253037801f5bacc9a3b40e7e2e13eb2dd16e6a", + "task_id": 386 + }, + { + "sha256": "54c18d01847c67ae6e9c691c36eeabfdfc6b78f38eaba25ea59bf2a22f18edf5", + "task_id": 387 + }, + { + "sha256": "250d665208d3771cb6f8a8ddf6d5ad6ddfb20317aea19280715c67e782cc6607", + "task_id": 388 + }, + { + "sha256": "ca0f67414b9f8ce4ef2bf351bc2be79b22dfdcb8399c33ff67f13dea9fe5f9da", + "task_id": 389 + }, + { + "sha256": "f803403e005d94ec891028012aed6c4d87c27f5f23dc366e981317659ba1ccba", + "task_id": 390 + }, + { + "sha256": "bc7a033d6f5a7c9a155252d76064d0611a365af7d3d0c98e9ec948e066f231d4", + "task_id": 391 + }, + { + "sha256": "98ea9707d826c34a41fe877ea35947409c416d2d22913c6b4e2b287433648cda", + "task_id": 392 + }, + { + "sha256": "ff435b6557f27e35771b1a15006d786d52c347cabdf3027be46ff6ab116008c4", + "task_id": 393 + }, + { + "sha256": "09d6585833400d2e122558b6d80a57c90b9c1b9767f1bef1e6a6ba98dacb7ccc", + "task_id": 394 + }, + { + "sha256": "9c5bb4295a2e879e18d52d71bb0b951f87a7c7a4590c436246a852d75eba868e", + "task_id": 395 + }, + { + "sha256": "644f1bf7b575c915371a3103337ad232f56b06c3b79ccb2ac0384cf10d940657", + "task_id": 396 + }, + { + "sha256": "8e4f8ab9760c117a1824c6bb7db20bcfd353ab20e3d595a495c95bd46b357b33", + "task_id": 397 + }, + { + "sha256": "d340c5cb4c7685a2ebb9e01c8d6d6fa10c7a965a4f378b1d661b33d0331b109a", + "task_id": 398 + }, + { + "sha256": "cb7d4d03da7adf20c73fc7206d0be2e65713a82c9a47bd655a26ae94d95f55a8", + "task_id": 399 + }, + { + "sha256": "ae81c38b2c00d00b2b7040e14eaaf70636fa6a20b92572ae859b972de982a3b2", + "task_id": 400 + }, + { + "sha256": "577603435dcf0022bd5d05ef0235ca3afd73f08a8d599a01f35bbd8e551c7aaf", + "task_id": 401 + }, + { + "sha256": "0e64f3ea7246c08db578b365ac79e3d773cbac74f0780752e3d259ba4ebee3dc", + "task_id": 402 + }, + { + "sha256": "7e6412995a40210f133f023a8cec4b50100c3f36862106e7b36965805058b441", + "task_id": 403 + }, + { + "sha256": "1e1f2fde9648b40cae968eeca68df7910b8cd86a593ac4a62c740c81b1922b26", + "task_id": 404 + }, + { + "sha256": "70071e06087cec4cf88f1cc16de6929fadfe9b33889144aa936c8d3fb3075a5a", + "task_id": 405 + }, + { + "sha256": "6bbc422f02bf1e74e2496ea78055528f7f1822348f99af8e1b0871139c3c4f78", + "task_id": 406 + }, + { + "sha256": "fdda6befb7380acb6c3b6a49e6ea960e6ddf444fec1747516ba431d6e77a96bd", + "task_id": 407 + }, + { + "sha256": "756569fec5d3f8ce6135e4a734867bf0716c8d658b2ce6f6e2d57c63d24c6058", + "task_id": 408 + }, + { + "sha256": "6b5e83867ba953ea836aea2871337cfef6355c144684192167adc590b10b3b97", + "task_id": 409 + }, + { + "sha256": "59c5c07c575bd15f09a7d57b8beccb7b1090aa344b18ee4db42deb9989e96ec1", + "task_id": 410 + }, + { + "sha256": "0179c29cbd42c2046ac7dbc0c68f56ad5e15d4340a7e5a95ee1c691903536792", + "task_id": 411 + }, + { + "sha256": "4e30a719595c2b8a7280b83f40522b2e5078da5aa456f1adb221f167d48731d4", + "task_id": 412 + }, + { + "sha256": "55f1e43d53a66428eaaf18eb02a27a0285197a8286fae13d415feb5e8957c51f", + "task_id": 413 + }, + { + "sha256": "aff705795c68ce211052e44e3bfd57214b565ba33db47f94aadfab179c5b47f5", + "task_id": 414 + }, + { + "sha256": "1158cda6a946280aa807ff5a9759f2d8cb1baee193c2d46739d6e1f4f2eefbeb", + "task_id": 415 + }, + { + "sha256": "5b7ff6ed0eed5ea54ce95eb5741e002e2d203e85371729f71a45f48c7c7f2e02", + "task_id": 416 + }, + { + "sha256": "60e4a46f78a80395c234a293bd1e8d8bd06ebcfcd601aefea4d27f4e439e6f76", + "task_id": 417 + }, + { + "sha256": "d50a6919f4a5e0910ade0c1b959c6c391fdacb4a9151b02d84e41e8802763f16", + "task_id": 418 + }, + { + "sha256": "79a575df50f4d9b220729b0a6fe0fd3cfc855509ef4fb4698314bf52aa1784da", + "task_id": 419 + }, + { + "sha256": "7f13a7500a59d696ae8904ad214000ddfea1b186bdfb42022d4e3be17456d847", + "task_id": 420 + }, + { + "sha256": "d5e210428d0578963941379cb2242d9aafb46fa4d3965602f45e6323b6f9b63a", + "task_id": 421 + }, + { + "sha256": "0eba342cef414d69277ee052d916efef69268d38b1733e936e005f7c299ca07e", + "task_id": 422 + }, + { + "sha256": "3fb84b23ea74883614a27f2de80b7b578d9badb2a0e9912b97f246e391473fe7", + "task_id": 423 + }, + { + "sha256": "2e87915df4b259a1f1a88a0869cea8d521845e3f5e690294fe80d73c26f3d063", + "task_id": 424 + }, + { + "sha256": "8f2fbbc3a4cc308fb9351ad9ebfc91f7c8000a9250161c8b46525bf016ec62cf", + "task_id": 425 + }, + { + "sha256": "4a39015b60cf1837f16fe9b41ed76c980db660173ee69f114feb9022fdc2c401", + "task_id": 426 + }, + { + "sha256": "572ccd8d94c819abcc887a5127a5ec7d2818b6e9a8bcceba9988af9df06e1164", + "task_id": 427 + }, + { + "sha256": "82fea8cf0fdacc58cecb1c057bf0b72cb89774ffdf2234f896ac00734451317d", + "task_id": 428 + }, + { + "sha256": "76c5d64da0e111299a737c30bfcfc5b2f312b5153afc32c286bed167f44e375b", + "task_id": 429 + }, + { + "sha256": "e0e267a2ffd8f4b22deed3e17fdf132b12b57e323570ac65b50d4f3460de6e00", + "task_id": 430 + }, + { + "sha256": "29dda656e6906c514bd0a24c86e9906e91e5e6252efed721fb9fb1f30201a494", + "task_id": 431 + }, + { + "sha256": "0ccfadde33c2cd7cc9c7297cc9da7afe3c33174cf8374c9eb04f3017ba978d2e", + "task_id": 432 + }, + { + "sha256": "91b57007b7ee1c1aa97f8d14fa958cd3942ef326c7dc9c62fd597e2dca55dbcd", + "task_id": 433 + }, + { + "sha256": "d52408b1b8082bb757ab1a713f88c131a03879c264756130ff4bf03a80833c12", + "task_id": 434 + }, + { + "sha256": "5ae2bc49b5d418a5b73f2c2a2172f3ff83b60736fc4e81a5d9fc8cd43b6f097f", + "task_id": 435 + }, + { + "sha256": "90e16b1e649208a0a42b9521df66a77d7ff2b8bbec2fb1a698058c04df2500c4", + "task_id": 436 + }, + { + "sha256": "c7cc2d93c25efc0e25f9784d015df7a5e06f0c1334bec875c2af42fe7fc63f6f", + "task_id": 437 + }, + { + "sha256": "0df5effaa4842e28c0ac03d3e06c50af4f5bfcb71ff68e1ef50d359f89a49afe", + "task_id": 438 + }, + { + "sha256": "4aa5251f2c99769e18869d45ae93d80bc4f7f8068e413b605ea67b2b41da08ff", + "task_id": 439 + }, + { + "sha256": "42f060f45f76843bbc09ff282aab9b1aee642763d4f8b3688d3eaf3f423788ee", + "task_id": 440 + }, + { + "sha256": "b964e8555b10a0d7308b677019b44aaf1e14879a4bcbf8deba64a2029956f2d1", + "task_id": 441 + }, + { + "sha256": "6a0f47baef1ae5e919708989d28e2f350009bc69420421e14476a45e580bd04c", + "task_id": 442 + }, + { + "sha256": "32274088c83531f0957d90f1ef2e9c078ae1c03f8bd315d60e125aebbee1aeeb", + "task_id": 443 + }, + { + "sha256": "b64e4e008307c3509c5a60f5994a158dfab19d6d0ad7d7e1d034305651da2637", + "task_id": 444 + }, + { + "sha256": "f782f60c38b93e5dfbb4912c7212a5c32ef3c57919f76990118d358ae704e67c", + "task_id": 445 + }, + { + "sha256": "c3d2acf33428d45671dc365e0f4b7f41ea5c50b82cf6033417e8c58360c088c5", + "task_id": 446 + }, + { + "sha256": "8f4bfd5f3f3c0fde6d216d5722574fe46346a4cade4836b27276d3adfff6b32d", + "task_id": 447 + }, + { + "sha256": "bcba6b1d1d8f953e142c3fc58d63845980f3ee05840fd0cc44f2aac77bee1366", + "task_id": 448 + }, + { + "sha256": "6001737c658d40251410d05ae17a76596b0b5a473f475f3f9bce2dde728ef110", + "task_id": 449 + }, + { + "sha256": "2f262a1fe0aa292d1c05c6c7a54b11497eae618bc99c212d2597f7653fc540c9", + "task_id": 450 + }, + { + "sha256": "35e752af37ccaeee3318768767c41a4d98c3100de8b3d05191bde3ebe39596f5", + "task_id": 451 + }, + { + "sha256": "9794bdf965e51e9775d2953c06e87ca82a61b280009eff3ed7dd0c1b28bddddb", + "task_id": 452 + }, + { + "sha256": "9d7e39f13669d309a8628459e32d913fb39788cf3cc7baffdd208caea37fa85c", + "task_id": 453 + }, + { + "sha256": "66928d9b49e34c9854d364e49ed4b0bbfec402e851cb28980b8b63704e30d059", + "task_id": 454 + }, + { + "sha256": "7eb4f7077232b544b25930c0f174008271b47397497655149f68fdd7740d9b50", + "task_id": 455 + }, + { + "sha256": "a52a10792cc945ec9e85b393707ce5a90f7cb09f6badf59c6dd440c79d3a6ae5", + "task_id": 456 + }, + { + "sha256": "15c11f878ed7340009751e41cb45116318009559105b11595a3e00b5ff25688e", + "task_id": 457 + }, + { + "sha256": "bc396deb81854e793ac1587a32c0b16b24ec40a97393b4384680a954d726fd9c", + "task_id": 458 + }, + { + "sha256": "05d7b8a2f7bcbc1e17e62759d12d37b35ccbf613fe441634d6e676240107e24d", + "task_id": 459 + }, + { + "sha256": "0b54dd55c706306ae8e621501912717e681c08de7eda1e0cefdcf830ca554937", + "task_id": 460 + }, + { + "sha256": "738535c0d1745f9a1ff76633dd8682058b020c56f50e80e1f919efadef1707c9", + "task_id": 461 + }, + { + "sha256": "57b8abd98652d60594a67a4d0777cb031c62c7eb6b19c1f86f956511946f0f76", + "task_id": 462 + }, + { + "sha256": "cb1fb2e12395a713434e064edafc76c981dc68f7f9fee112e6959036039b459c", + "task_id": 463 + }, + { + "sha256": "984987d382d3cb45d6cf5790ce57e13c9ea0f59fff56df420545e0ab0834b0c9", + "task_id": 464 + }, + { + "sha256": "794be688ff200c8c76627cf673620f58499723f3b9b196d7a6b4128ad95f41d3", + "task_id": 465 + }, + { + "sha256": "d2579a273aeff89a7fa13d785457817f7421060f4ce653fb71d5501ee2d0d4fa", + "task_id": 466 + }, + { + "sha256": "24c85b3c08e271ea0db463dd1d844cafae596388961a55328fbab8be851c843c", + "task_id": 467 + }, + { + "sha256": "7042d4cc071d0acb5562591a94b54ded66ef31eb8f79bad3d48ff25f4cffc641", + "task_id": 468 + }, + { + "sha256": "12ccda53757d733485a9b0154fb3c93ffaaa4c54d88931e7166fbc935c89ba13", + "task_id": 469 + }, + { + "sha256": "edbcd0153b737021feb8896a0275d0459d90a07e3b3457c762e5a714107f6078", + "task_id": 470 + }, + { + "sha256": "4c811034d89bfcaf14b54b0bddf449cab4db37b3c66f197b8f498a5b68062beb", + "task_id": 471 + }, + { + "sha256": "b47647824b3ce2a37fe28dd071fa31e881a2f66d7f54f43bd95c6b6a5b61eada", + "task_id": 472 + }, + { + "sha256": "6f036ae099de2abac390a0c7cbbe7c0934c92e3c2888f652c6facbaafae17b69", + "task_id": 473 + }, + { + "sha256": "afc5bb6b9220189f56fe9a703602093420ff79d98ef812ffc1eb25506f26b414", + "task_id": 474 + }, + { + "sha256": "085441b4c11acd255ee4e7f3456405593012a2a494d742233f3b14f1b0e78337", + "task_id": 475 + }, + { + "sha256": "804ec9d1ffe208cbe60cfc729abab380c502c5126017dcf376f7c8781e7b852f", + "task_id": 476 + }, + { + "sha256": "be92e18e21c9526de06052f9d9b2a0ce26a97e2fea74bca165bb77adeb38f238", + "task_id": 477 + }, + { + "sha256": "1a3abf4b1aab9f0e25b34a5c39480b54b66e6ced2231ecb79d4247b6a801b29b", + "task_id": 478 + }, + { + "sha256": "d7614c25225fc25c1b0a7436854ef495a3906fa49f4f3d48798e4e804da04646", + "task_id": 479 + }, + { + "sha256": "906e6ef7b33aa2335fdfa4d4ec2a5b15ca00138d60580998e9f6be5c7153ec22", + "task_id": 480 + }, + { + "sha256": "eda124daaeb803b2df1f4026d47d51ad3dd086869265dc7df82bce9391dcde0d", + "task_id": 481 + }, + { + "sha256": "3f5bce00a9180f1cc74ca5ba3ea74e23d21c0d50f4e63e7029c46bfce3bee24e", + "task_id": 482 + }, + { + "sha256": "3b08049b726c50b7b6140e3e29aebc7ffbde4b3a7377181b3937b3e72197b657", + "task_id": 483 + }, + { + "sha256": "f020b03336fdabc3e84987d913b07c2b51d427b865f3cff92095c08b62b5fac9", + "task_id": 484 + }, + { + "sha256": "a20341c859c43d2f5d35236b81d4c8c1b4275b2b24223a2721c64f7fb1458d87", + "task_id": 485 + }, + { + "sha256": "9e851c71920a7f8895737f873dd1edfe858f2e5ef70bc27bbf5fc4a3bedaebbb", + "task_id": 486 + }, + { + "sha256": "1671e7eecd9a87f428209990045f310f94a93deb18c72670b9b8fdf3ef2390d6", + "task_id": 487 + }, + { + "sha256": "b1196a28fc4ae41522543435bb921b5c78d5ecbf29582eef08c9b153e1a65070", + "task_id": 488 + }, + { + "sha256": "6a609aa375933ec2b32f40860a04dd500d92be3e525f0f603786a1f88b8ff7f8", + "task_id": 489 + }, + { + "sha256": "764315278a99dd3ef7ff7e6c00afdef2b9a3a24a7d3bae43825dde160e72a61e", + "task_id": 490 + }, + { + "sha256": "59fe5074d624e7a1b3a538f78afef34d7247e86f60c0478cb53f4faaa2950704", + "task_id": 491 + }, + { + "sha256": "c6e907a3d108eaa8a0d4f7381e5210ee74f030f05ae9f81f9826d0fc7d589906", + "task_id": 492 + }, + { + "sha256": "84cb0e1d47122c054e5a084c77ef0962d06ba45025b8918d9eb7e802ec68e08a", + "task_id": 493 + }, + { + "sha256": "dde8e7712b499c0b4cc14bfe1708a98825c6c14ebf979664a7eedef2e07c4d8c", + "task_id": 494 + }, + { + "sha256": "aab4144e484a8525e196ef8ddefec5b927f72c9a089c189f2fd4a845de9eb450", + "task_id": 495 + }, + { + "sha256": "189642f6c1b43939304530fda78a9b92e8980d27e1589cb024c456c20a8445c8", + "task_id": 496 + }, + { + "sha256": "0ec7d8f5629efa5720389ca52dce888e5d5cf76f66626c290b98ae96b7314d23", + "task_id": 497 + }, + { + "sha256": "4aafd44f85d8fa3ba6a35237b4a8b0664ebf4d70b858d675b4c28552da6fa9be", + "task_id": 498 + }, + { + "sha256": "bbe93b89c36edee65aa8734055d3860ae37d5bb8f1610f4c35d024ddc4ea0cbd", + "task_id": 499 + }, + { + "sha256": "f184ca62b8ff80a9fa5229f24948dd1f0323ce551d6f1fb2be52c7befca787a7", + "task_id": 500 + }, + { + "sha256": "1e1f67f6795affec921655e9c5ce1f1a0a88e5cbf5246ad0bad6f225696b4d72", + "task_id": 501 + }, + { + "sha256": "7412ea1c47189b902527f543e3d7f5b545f23cc3f9c7d42a30d84e965526c2d5", + "task_id": 502 + }, + { + "sha256": "e7cc091a492314b808f6af95689f6e5ff5d5337494bf0a5c825c414d90f5198f", + "task_id": 503 + }, + { + "sha256": "8b04ed8ffb03266a82cf671c0a31ed72c82076a1e2c09f50455482e80f4b67e4", + "task_id": 504 + }, + { + "sha256": "0e7d4d0b672b93a0fdbbd5afac27e2ac7eed5c27e967fe79b74a5c5425fef0fd", + "task_id": 505 + }, + { + "sha256": "24d81a9357afd3cada0e73f2bef736c397261dc2099465c5553ca5b041df3493", + "task_id": 506 + }, + { + "sha256": "f98aa30f2e58adf274be27279e70a5912952fd736bbf4383e5a0fa4f1a2d13e1", + "task_id": 507 + }, + { + "sha256": "231d6d8ef2e3da18cdac979b4c28ed8d7f3dcaa27094af862101de56fbceee00", + "task_id": 508 + }, + { + "sha256": "5dff11bb143c42c4100e35322bf44a10bfa2a1ff0c88c7dc23d0d4c00b52b94a", + "task_id": 509 + }, + { + "sha256": "6a5aadf88bdd30a3448fb69d1e3b40320ca0a7adb8e0c42c1d1745e8b390a691", + "task_id": 510 + } + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": null, + "source_split": "test" + }, + "dataset_manifest_sha256": "060aaff7117dc47af6c01253a912f34b6956241c336bbc7216e73bca8624d2d4", + "model_id": "Qwen/Qwen3.5-0.8B-Base", + "model_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + "prepared_manifest_evidence_sha256": "21a6d18c6a0887b1499d156a3d610d4bfafdd59d3557713485b62038e263b96a", + "repository_commit": "6bd5bed2b61e192526ba8fdbec8232801cbea843", + "token_manifest": [ + { + "code_token_ids_sha256": "ca4074a90d9924be4e36b616edb1fb00ec12a5eaffefb9a5813387c761c50e1e", + "code_tokens": 98, + "prompt_token_ids_sha256": "ed99353e8634f79ee55af2198182202f7bc84427c2cddc8fa4d3b3218a39fb67", + "prompt_tokens": 88, + "task_id": 11, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "7a07bffbee0607ebba8bc719eef65c351d89ea0085ed1c92059ab5834c3d3184", + "code_tokens": 19, + "prompt_token_ids_sha256": "41b434ec1ea4056dcae6131dbaff63088c7c6833fbeb8a77f7a5b79cb51a7e7f", + "prompt_tokens": 215, + "task_id": 12, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "221fcb82068107d797be46f12073f32e9dfc949992ce8866a7607e2cdf599f02", + "code_tokens": 39, + "prompt_token_ids_sha256": "a39d1e784403627fea22a54a99d9e2019cd244fbae7a651745ba75453137f7f9", + "prompt_tokens": 239, + "task_id": 13, + "total_tokens": 278 + }, + { + "code_token_ids_sha256": "44956cfe26f8b9cd8a218f2820997aeaa1f378147a54093cc84ee699320a882b", + "code_tokens": 24, + "prompt_token_ids_sha256": "d01e4094a64e7225e4851ec7ce56a839092be6086c65dcf43aae5a97197c31c1", + "prompt_tokens": 87, + "task_id": 14, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "4a7d3767866cd6d449afa9a55314639dd9911a22c6b582fafd7889ce784b0d51", + "code_tokens": 25, + "prompt_token_ids_sha256": "b26b6fe54ca0665182b853eda401a9f250df526f9ef89c04db06ecbb201e0093", + "prompt_tokens": 113, + "task_id": 15, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "3764692433fb372c33425f9a53ad24eb14e5db843302adfa5b5b0a8a53c951e8", + "code_tokens": 55, + "prompt_token_ids_sha256": "6ecbf53756b49711a63907e2332b7ba1c4f1109dabd7a407de2d664f8bff5e57", + "prompt_tokens": 92, + "task_id": 16, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "0c4a09601bb3445d380f00408d51bb28d3e58451474426fa7e64388426f76934", + "code_tokens": 16, + "prompt_token_ids_sha256": "15b1a6bc09a362631db60b7b45984c6f30e4cce2649aa5dcbc6c62873dcdb694", + "prompt_tokens": 68, + "task_id": 17, + "total_tokens": 84 + }, + { + "code_token_ids_sha256": "c0728f0361691ca53a8576262c93c131ac3c0674ad5e839ea8f0b887e17b804a", + "code_tokens": 199, + "prompt_token_ids_sha256": "689290e79a13cd85182e1e53a4cc5c49911dd6bb0c8a29b3eef06db4cb8034b8", + "prompt_tokens": 102, + "task_id": 18, + "total_tokens": 301 + }, + { + "code_token_ids_sha256": "f66a26c66f1854dc46e3b08a790cb9d0df0ab49128a51e8be04e69ee0c017df2", + "code_tokens": 28, + "prompt_token_ids_sha256": "e9b78ca3e0110cda78ea2034bab7b33dc5f6d48fb563cecb1b6d1e917acf1673", + "prompt_tokens": 101, + "task_id": 19, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "ae9c0dfe9a3c6f29c3b4fbba71aae76fad03543b6e6e12d4e34a26ae50c22ec5", + "code_tokens": 87, + "prompt_token_ids_sha256": "f4849f630e91bb0427fd5ad7e120f8b34b1a1a70aeea09b3a60baf05aa29f92d", + "prompt_tokens": 80, + "task_id": 20, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "20f4393df1dbea6bce755816285ad46507dd58921703dcddbdded90ba91e085a", + "code_tokens": 35, + "prompt_token_ids_sha256": "b91035c6853da4eb360e2a07ccb945b7af361dfc3f62ad66542394454629331a", + "prompt_tokens": 108, + "task_id": 21, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "96c2f4cdf35461af186b19a7eba036772a5470849ceb4c32ebf1b4a2f9ca145e", + "code_tokens": 62, + "prompt_token_ids_sha256": "941b1bd73064348187aa42b1c4b840da4ee1b598de449c02b13c97236f566bf4", + "prompt_tokens": 113, + "task_id": 22, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "2acd5f2c91c0cf905a0c4d65b15711b63a05c1ec3ea03e3b40fa3e447f9c3a95", + "code_tokens": 58, + "prompt_token_ids_sha256": "85816f3811b40926707769c69129140f7d1183071503dbdba043813cd6367983", + "prompt_tokens": 149, + "task_id": 23, + "total_tokens": 207 + }, + { + "code_token_ids_sha256": "f6fdf9cb53b2c54485de4e4bdba6ecd540a2e0a81e11fefc5b6a92be6961d77d", + "code_tokens": 79, + "prompt_token_ids_sha256": "dfb771f01c701c76191c084dad9b1f2907fb2fc4b78c1a2aa7c29c3d8b26fbd9", + "prompt_tokens": 87, + "task_id": 24, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "b1fd9b33ab2d8e2dd7e7931a44b6f92280b1390875aa6f120df42eeb92a08c31", + "code_tokens": 58, + "prompt_token_ids_sha256": "0af9bf0df6730c0bae8a2c97fc6383f3934c8e07e3731d186863dd5b04f675a9", + "prompt_tokens": 104, + "task_id": 25, + "total_tokens": 162 + }, + { + "code_token_ids_sha256": "db7853bd6a3ae3c0e856a916b9b6803f8dc163d4bfb7a0795dfde8fe535291c6", + "code_tokens": 48, + "prompt_token_ids_sha256": "3f29dbeb6aff516a38f8c773d454d279fe014adb5ce2b4cce51ccfb96ad66a71", + "prompt_tokens": 142, + "task_id": 26, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "fd1309c323771b967dad115ece05bd67273d805f680d339f4c23dfdcd9c27dd4", + "code_tokens": 37, + "prompt_token_ids_sha256": "ab1dd51def39597fcf1d5ff1dfe6dbebb650d0eabb7ccad55efb1dc117f22faa", + "prompt_tokens": 116, + "task_id": 27, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "f4cd18271b3fafb0029071d6e5b40458c6a1810cc42aeba39bde3ce36d2927d7", + "code_tokens": 61, + "prompt_token_ids_sha256": "b2cdefbda762a3291840c7a63a182e116a981c65ae85b490c02d7c1a324746ae", + "prompt_tokens": 80, + "task_id": 28, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "8264181d8b34af2d9cbd07da5e4084489d6ab91622bad459dd7bd0da39c73991", + "code_tokens": 77, + "prompt_token_ids_sha256": "0bdd84ec7d91f960f2ee48ce688e7099459977d2ffbe014bdb7bc8c2ef047a90", + "prompt_tokens": 137, + "task_id": 29, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "a7483fdaddb05b76d4a2be4bd59083d949ca756d9798897ca8ee1f227148873a", + "code_tokens": 98, + "prompt_token_ids_sha256": "de38a7613d5d82cc419bc02c0acc47a14a7d0369135adf20825aa1506cc0d81e", + "prompt_tokens": 95, + "task_id": 30, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "ac6d875ff1fad87c9754785301d1d6157743fd9c5f066d864db10465934f6a07", + "code_tokens": 162, + "prompt_token_ids_sha256": "012c336cc3b3b7ce9e3765952be744a1e5eb0f71dee14ff538402c340b0dbd86", + "prompt_tokens": 316, + "task_id": 31, + "total_tokens": 478 + }, + { + "code_token_ids_sha256": "10e3a04899828e4c297b205675e770b686b8972ff197e45a36ed0ae1ec08918e", + "code_tokens": 103, + "prompt_token_ids_sha256": "4a9d2bf3169137f973c1378d5d0e7549fbd97d4e06ad15c5d0dc40729464b122", + "prompt_tokens": 81, + "task_id": 32, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "79bb64818ce913fb005b1128a6381acb757468e48a4a3c12bfe666406b86ff8f", + "code_tokens": 73, + "prompt_token_ids_sha256": "21e72dbb837c0e026946f0989c9a460e3463668805005c4b79c7a7a4f1bd53c2", + "prompt_tokens": 84, + "task_id": 33, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "084005ab9510b7e85c731d8b89475152925bcca967173b05ec5552b539a56599", + "code_tokens": 119, + "prompt_token_ids_sha256": "dcc46d60716f9dbe17e266f0ddab1b550005733b6288092e0c8acac261d1fa64", + "prompt_tokens": 98, + "task_id": 34, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "d4e1f618f7cca6007e0f09abd1ae5a67f53af7276e39ea630cce5e8b65a159b8", + "code_tokens": 17, + "prompt_token_ids_sha256": "3662e296713e8f1edfd7bbaaa8dd668c0dbb5498232b35b3e18ea32d6eabcca6", + "prompt_tokens": 73, + "task_id": 35, + "total_tokens": 90 + }, + { + "code_token_ids_sha256": "200f6db7298be70ecc782e8d0a30761a8d562e2a4ea2e7412f0ff0ebc7644832", + "code_tokens": 56, + "prompt_token_ids_sha256": "f347eb604a7f9da27f6104b7e432fcdab391c2db3b8b3ffb51799443b0e86d3e", + "prompt_tokens": 95, + "task_id": 36, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "b15d7bf5a1c94f21a03001d4aee2cbbb1bf592636ce2ca6041019195a2dc5dfd", + "code_tokens": 57, + "prompt_token_ids_sha256": "aa474b2c164d94f4ce6c82da5e9e4cf4c5bdb9cc66614a584100f3c204d8e857", + "prompt_tokens": 217, + "task_id": 37, + "total_tokens": 274 + }, + { + "code_token_ids_sha256": "a48da0e5a048b4f9d6194945a3d9c526ce0dcf5612cf75ca0ced9435efc8db00", + "code_tokens": 63, + "prompt_token_ids_sha256": "897a4ee4fd5f6e307ea63bb0b3488b3f675c7b93d2c76e1702385e23f36ccc49", + "prompt_tokens": 114, + "task_id": 38, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "a6f227b2f487bd7b3c0a936e4be584ddd5a23e847c718170591325a654a6cab7", + "code_tokens": 200, + "prompt_token_ids_sha256": "210bab400ea5da5536cc28e598d53dd54f20b203f68f7777d1756cee5b059d76", + "prompt_tokens": 93, + "task_id": 39, + "total_tokens": 293 + }, + { + "code_token_ids_sha256": "377f905274a38c1b43b4d4dfa9c12200a1aea0ebe52ee934125093118e1ad4a8", + "code_tokens": 30, + "prompt_token_ids_sha256": "ad2cee03328388d1f57dd321d28242f388500e3cfd1ec100e6f07fb2a0db24c7", + "prompt_tokens": 344, + "task_id": 40, + "total_tokens": 374 + }, + { + "code_token_ids_sha256": "6436487de97d6c255c86b19fdab467c77f3ff77912bef299ad851fae7d6d13da", + "code_tokens": 28, + "prompt_token_ids_sha256": "f64a1b42d9646080299a6166b75c09769a173875fbe0f717ec72d9b85fbe51e4", + "prompt_tokens": 150, + "task_id": 41, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "b7a3f210a3585f4b8f55727852e815e4e3e10aee32dddf485a4f548ee1dd5d18", + "code_tokens": 26, + "prompt_token_ids_sha256": "385daf033e6f9e8fdf41af1f4fbe97c7677c8237d1157e77db0bef410c72b97c", + "prompt_tokens": 107, + "task_id": 42, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "aa53a04cc981295b35d37380f9f955fc351a9a0b50d95b5af42edf9f674183b8", + "code_tokens": 54, + "prompt_token_ids_sha256": "44a392e9d847d83c24c3254e6ac10ca75b1a7668f5f1939afded586d8e4a3e15", + "prompt_tokens": 88, + "task_id": 43, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "44c824bbfc11af6312d49c7677ec43f0de083212acc643ff3ca18ef412c7a5a1", + "code_tokens": 47, + "prompt_token_ids_sha256": "1b1117a2ae53889adcfe9d326a46b4d18ad5849b90f6c7ad31c130c1eaa9a0ce", + "prompt_tokens": 81, + "task_id": 44, + "total_tokens": 128 + }, + { + "code_token_ids_sha256": "d0dfaa1cad80b3ac30ee4992c395ff2d36a13c00cdf161cabb8f9da0675e23ab", + "code_tokens": 93, + "prompt_token_ids_sha256": "4e498a9df7f26dffc85989a37aeeb767cabf24a9aad8138d0a3657c1f1866592", + "prompt_tokens": 100, + "task_id": 45, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "9ab4c157f93c4edd52f0b7dd90abe2a3f2e56e5b6b45c697572a11f50d34ccee", + "code_tokens": 30, + "prompt_token_ids_sha256": "c663f8c87c27556ef65d7c0374375ca2f20dcd62ab0affc905cc157f6e999ae8", + "prompt_tokens": 94, + "task_id": 46, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "a07e2b3d43a3b1b2837b3ae326e16d3c7390ca30ea5e909b2d2ab0ad326794c0", + "code_tokens": 89, + "prompt_token_ids_sha256": "b34cdf5c66d268933013da91cda1877cbdde70c40eaa30cc5ad47ea940fe9d29", + "prompt_tokens": 86, + "task_id": 47, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "fd17e090ed65c9b06e57387112805f960f73fa94bc73464bf1432f966a81c6ef", + "code_tokens": 70, + "prompt_token_ids_sha256": "bfb67484ef69e10dcb9d8682434e5341d8164c379fe58d6534f0ef91254f543d", + "prompt_tokens": 82, + "task_id": 48, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "f97eb04b3722a2502920a008089c2e0dd8b5dd708f8fd779378d32bf92e8e44d", + "code_tokens": 26, + "prompt_token_ids_sha256": "e7d215ee203a8178b569941d75974534a94f0d2adeda524f76b79cc48cc6644d", + "prompt_tokens": 194, + "task_id": 49, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "1b9fc36cc01bd8d59ec0e95e3cb73d2047d3352ace9d4cd9551f9d3a3c2de915", + "code_tokens": 48, + "prompt_token_ids_sha256": "716debad76c783b9d4da0f2eb306c46698b5fa084b1328bb22f987e21cd65505", + "prompt_tokens": 166, + "task_id": 50, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "0d569c061b3db691dad5a7e1fbe219d9b5ff92824583579a335870389cdce499", + "code_tokens": 29, + "prompt_token_ids_sha256": "0197a61bf27df1bb477044885b773b43193f3c8a0f7336fc2e33fa7f10ce0512", + "prompt_tokens": 82, + "task_id": 51, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "be19dbc524f163e3ddddd94839057d7ea46a7d7b4c026d9f970e3555c0824937", + "code_tokens": 17, + "prompt_token_ids_sha256": "1aa0dea339e491a4b40cf40f825b178858faaea433b5c82155ee57f84354144b", + "prompt_tokens": 86, + "task_id": 52, + "total_tokens": 103 + }, + { + "code_token_ids_sha256": "58e786b9131ffeff92479ba3859c080d8257df33dfbccc2ad6fdbfc40433f8d9", + "code_tokens": 37, + "prompt_token_ids_sha256": "5e2f0557ce5033dabf63973ca38d907a27570aae5c5f5cf21ed2a9b4fb7f3ed9", + "prompt_tokens": 86, + "task_id": 53, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "2f2da2e7070d444855c0e1b6274b9ce79e57a8e63179c2dfea699528dfc55bf8", + "code_tokens": 121, + "prompt_token_ids_sha256": "4137ead7c7dd71a7f07e63349c3153d1ea09f0418658597e01e0e2482e0ae2dd", + "prompt_tokens": 176, + "task_id": 54, + "total_tokens": 297 + }, + { + "code_token_ids_sha256": "5328ce52815f42e654c1f443300859c15a34796f32fea3b00f18c88f1b6fdff2", + "code_tokens": 30, + "prompt_token_ids_sha256": "5da18dbc6d4a14366a61c23c9c2017c194c50ec973ddd78124cec6c099400cc5", + "prompt_tokens": 80, + "task_id": 55, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "d33653ab859bfa48ae2e43465b43f6b19d3b0f7634b1e57791d294b7de0db1c5", + "code_tokens": 74, + "prompt_token_ids_sha256": "63f364f6ab6a42463946724a6cb25cd055414ff33d016b0818041b51c02f2570", + "prompt_tokens": 71, + "task_id": 56, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "7f3b8195c3c4302172fec3d44ca1c916374f2a11e453589baae327683243bfc5", + "code_tokens": 53, + "prompt_token_ids_sha256": "33975f33d9ca2399cc6d53c3e0259c19dc29cf270567cce6ba7dbb1885dc496f", + "prompt_tokens": 107, + "task_id": 57, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "1f0f4b0df382dce3e83abb9712e143bd7928cdfb5b5d86fffa79ecd885926f39", + "code_tokens": 21, + "prompt_token_ids_sha256": "edcbc5d459be24f8bd6f406a36d483796018ede1e2f2a5c384f3201a44f18f94", + "prompt_tokens": 84, + "task_id": 58, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "f31e878fce3a1891f8071b7caff1ea84aafc5d58fe59a1ac11b9b1ef33f785ef", + "code_tokens": 20, + "prompt_token_ids_sha256": "0b11a37d25b39c23f1927614979b9cc2822c921dc4988f1a7f1c59cdf30b188a", + "prompt_tokens": 77, + "task_id": 59, + "total_tokens": 97 + }, + { + "code_token_ids_sha256": "a443d585d7a87d6d79211e285b0e01d783ca6222494ceb95fcae55772021c6f2", + "code_tokens": 122, + "prompt_token_ids_sha256": "b6e757c7b148040afdc04f90f979ddb3e1ef77e7a449160c5bb71615a73bc4ad", + "prompt_tokens": 145, + "task_id": 60, + "total_tokens": 267 + }, + { + "code_token_ids_sha256": "6ba5e7ff66a83b07a9991a00cdee94d982a25397674e5432dfd4e3a27dcc4c38", + "code_tokens": 94, + "prompt_token_ids_sha256": "e95ddd07c1e0da2b2caca29823bae5ab11f50f559cf00fd45298af0f69e0151b", + "prompt_tokens": 99, + "task_id": 61, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "ee94442188523f891922c3afbea433be47546deea7e30eb64f0100465148b9c9", + "code_tokens": 12, + "prompt_token_ids_sha256": "9184a820d5228322a4d1d657bc310b1bf6751ef716efd639200f0d9078d15b6a", + "prompt_tokens": 104, + "task_id": 62, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "1db308823f4593774dc50735df950bdfd8682241f70aa1e6bfac23946f4fd78b", + "code_tokens": 38, + "prompt_token_ids_sha256": "815b393dd2fb94de79c3b07904a2368f11039f19fa36cd70d7028405fa1c8eb6", + "prompt_tokens": 151, + "task_id": 63, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "c8a59e076296ce8a47087028a792c70ffa21dce70ed9aa07b3939db6c2a662e4", + "code_tokens": 58, + "prompt_token_ids_sha256": "4116bdacb62fbd2cdbff5dc924507c9c85fbd7a69f27e4c189298925c516e597", + "prompt_tokens": 196, + "task_id": 64, + "total_tokens": 254 + }, + { + "code_token_ids_sha256": "16105f9807e84284ae6417b45a092d735f521a0f6f56b9947f2ae168a78380d6", + "code_tokens": 54, + "prompt_token_ids_sha256": "ec16922ad5c4c8ddc8ffa92d30a377defbfa5bd9f3c550d06ba6ac0261b48955", + "prompt_tokens": 116, + "task_id": 65, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "e1c9c52bed6e427dc9fb811b9fbaff64be024891b78bbf76c7dc50568202d83e", + "code_tokens": 40, + "prompt_token_ids_sha256": "8cbacf868f4620e15dffada8a159bca3ac668cada91f03e2980607626943bb97", + "prompt_tokens": 86, + "task_id": 66, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "e882b415b049dde1d6c5060f61d59d4488c3292c8b2e7bbb4dddea3aa7daf597", + "code_tokens": 115, + "prompt_token_ids_sha256": "3f9edda1dfa2825fd7b067941c5a5d4805615a5ff62deab9405d28098d0b608d", + "prompt_tokens": 128, + "task_id": 67, + "total_tokens": 243 + }, + { + "code_token_ids_sha256": "c70fbcb4bd4cf364cffc3be84c7814f758014aeb8643dbcaf163a6044e8c668d", + "code_tokens": 60, + "prompt_token_ids_sha256": "d50ec4cce1c2b583fdabdeb2e09336affa1837152c28e1d07cdd5692ebba36f3", + "prompt_tokens": 102, + "task_id": 68, + "total_tokens": 162 + }, + { + "code_token_ids_sha256": "f2d0502e46fcc155fccb765c5391c0a73ae7267bab60e360b9e6fbceeff727e0", + "code_tokens": 124, + "prompt_token_ids_sha256": "7e5e54100a17a3fc7bb14fb0de59c6863c32fbc8aa11d5ce99527c5fc47b63a6", + "prompt_tokens": 104, + "task_id": 69, + "total_tokens": 228 + }, + { + "code_token_ids_sha256": "919e961af9f7d90f8becb688e35c761d4ec0bf39a30d45678cc43b0ad33af917", + "code_tokens": 91, + "prompt_token_ids_sha256": "73f541e9ec6c1ba3942b10d9542b355b93b083ba87eedd34e5c671b0a5b32778", + "prompt_tokens": 146, + "task_id": 70, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "2542f94e49df4904fcf3eb34e8ac90b33c087027f3b4e3f6b35d39793469d130", + "code_tokens": 125, + "prompt_token_ids_sha256": "c74181d2aac145606b088dcabc2aa93c593408ee73e60322ab1d7be48a1b9b7f", + "prompt_tokens": 164, + "task_id": 71, + "total_tokens": 289 + }, + { + "code_token_ids_sha256": "3850074acef019a6d2afa5604e982a6069b09681af51d2cfaa76ad771045bf9b", + "code_tokens": 26, + "prompt_token_ids_sha256": "9540fcbf020c8a67e9d9108f12cd85c0935c63a6ccac86664e1502def5b4d03b", + "prompt_tokens": 79, + "task_id": 72, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "9770220e9770ae5712379cffee311f930586560e79b9c94e8915cd444202e611", + "code_tokens": 24, + "prompt_token_ids_sha256": "81a2f7b24bb1fa111ef571f3ba4e741c8abba293d94536e4ede3fe1ba2d4b24f", + "prompt_tokens": 159, + "task_id": 73, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "c000a031e5beb3ed4c5cf24f3a1d3f76b272b280fea56818163ce001dd200a90", + "code_tokens": 180, + "prompt_token_ids_sha256": "ea20709fc745eab3a8f932013aa6ecbe7e5bed7fee7d28e3a2cd7dd3f96b66d6", + "prompt_tokens": 107, + "task_id": 74, + "total_tokens": 287 + }, + { + "code_token_ids_sha256": "874c164a8413e60415fed343cf9cf0bbd4d3b8ffaf17a3a4fc07e5bb5fd45e38", + "code_tokens": 40, + "prompt_token_ids_sha256": "0856e8a0f86c222b1f11695ec7039f5a80673a7adf6b9ba1144a5196f81c55be", + "prompt_tokens": 201, + "task_id": 75, + "total_tokens": 241 + }, + { + "code_token_ids_sha256": "9a5ab9bc318685221ae3a00aece4853a9a2f1b451e0627c8624d4cd5487df8ab", + "code_tokens": 72, + "prompt_token_ids_sha256": "34ffd5a0735716c8aea2eab69629cc50552a2020823d8fb9fd0fa19bab9320ce", + "prompt_tokens": 80, + "task_id": 76, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "9b6b8fef67034cd6fc2896cffcc2b540831f350609d4e331836879fda1b3ff26", + "code_tokens": 20, + "prompt_token_ids_sha256": "c4f9c132c553b040a95f02af9be15b3aad7eeda10a208eb039986986ad25d910", + "prompt_tokens": 85, + "task_id": 77, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "7a2c05dcc15b5d5d4b342f4a4652db0e3410631e8d6756bd784012c72bc85f3f", + "code_tokens": 76, + "prompt_token_ids_sha256": "9f32c67684d87dafd9f0b29fd241cda54729c6ae067b24d5fd48a01f2580cf21", + "prompt_tokens": 89, + "task_id": 78, + "total_tokens": 165 + }, + { + "code_token_ids_sha256": "f8c5ca3dba3ba832e3f68155b6ea45bc226e460d6d4008d5ecaac96849342199", + "code_tokens": 42, + "prompt_token_ids_sha256": "ec9611eab01932858b739d549ac0eefd5f62974f5ef61389ce19ecf053b55577", + "prompt_tokens": 71, + "task_id": 79, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "e439c21eda7f9ea423cf4107a7e860ba7f24538a03464f72a271b86f8eab378a", + "code_tokens": 28, + "prompt_token_ids_sha256": "e9cf18c783b3abf183b0e4ace9eb051fee74723583f71da9ef028e7d3aff23b5", + "prompt_tokens": 83, + "task_id": 80, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "73a78ea6a11845bfd9fc2d848990f3c858f4248c95f126fd06edd7f631cfb08f", + "code_tokens": 58, + "prompt_token_ids_sha256": "d1e840b887a00933a67c4a5774437b8288e0e8d1b8bb2aeed47397accf7ad9c9", + "prompt_tokens": 252, + "task_id": 81, + "total_tokens": 310 + }, + { + "code_token_ids_sha256": "916f0220c27b1c281a41ad390ca7b870755c237f98631f272c7b6e31609c2532", + "code_tokens": 25, + "prompt_token_ids_sha256": "d32748a28cdab0135c671d9beba2b42f96189138dcabda89ed848ae7a268cbe7", + "prompt_tokens": 112, + "task_id": 82, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "a9096fa457b3c7e373c9e7861a5a412f885237141e969f9c58b2f5a7d0eaa276", + "code_tokens": 88, + "prompt_token_ids_sha256": "0ac3e057973a992cb2e7f92d07b63ffce8d93339706ad7c7acc7ed915605e72f", + "prompt_tokens": 79, + "task_id": 83, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "8df335928e123755be904d009a612aeac5f40e49b85562d4aef9f32675dcaf2c", + "code_tokens": 42, + "prompt_token_ids_sha256": "3270e0d5b2b5f0c304ca90ed62e66021f6c97792fe8443d5fbbbfe06e324672f", + "prompt_tokens": 70, + "task_id": 84, + "total_tokens": 112 + }, + { + "code_token_ids_sha256": "c675cf5eda90b7b82bc0514da5ac294f6f65c245552d740c0a6bce4ffd90d03f", + "code_tokens": 24, + "prompt_token_ids_sha256": "6a37b6d7e968320143eb78eaa91c166aafe3591842cce8e5fab66040eef1421a", + "prompt_tokens": 118, + "task_id": 85, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "a73935b567a9ceaab9383a8fe3b4d8eb2e69ebf158959102c641320cc3d507bf", + "code_tokens": 24, + "prompt_token_ids_sha256": "5783709cd4625dc7d491074cf6b415f7bcb750ed24276ecaa42d229d19334abe", + "prompt_tokens": 78, + "task_id": 86, + "total_tokens": 102 + }, + { + "code_token_ids_sha256": "eed89bb525900899baa2242ff332f28ed6862dcfc8d7284a6eb7e87bd83ef3c8", + "code_tokens": 43, + "prompt_token_ids_sha256": "94eb8be73dcc0ccd4b975d48aa83c2a71a92fae79264ce90849ca49f41290525", + "prompt_tokens": 310, + "task_id": 87, + "total_tokens": 353 + }, + { + "code_token_ids_sha256": "11af968b33151ace9b5131d4cdc08f8d8854111927921305703da84a970a0583", + "code_tokens": 24, + "prompt_token_ids_sha256": "8779ddd9f0763372a9bfe9331c60e9b3557fe31e5f3ebd008dcd534db5143f88", + "prompt_tokens": 220, + "task_id": 88, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "e1d18724171881fd8aced45af5c1a08cbd61a11d86cc5ad85187bd4e9c2a400d", + "code_tokens": 14, + "prompt_token_ids_sha256": "cc04d950ca3e9e286cb0b912bab35dd94bf7b742ce93e811af7a836655de0781", + "prompt_tokens": 72, + "task_id": 89, + "total_tokens": 86 + }, + { + "code_token_ids_sha256": "9d8311191c99cbb4c0b5218229daec1143e19cdb2559fb6d5e89fcd9749d9205", + "code_tokens": 41, + "prompt_token_ids_sha256": "e7deb2c9cd0cea443af1ce0ea4c8b25e447014a67efa876a8619df04871a2c2e", + "prompt_tokens": 83, + "task_id": 90, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "4d6e2570c07c0b4fe8ce75e041256f52b55792021cc1881cb8f2d8c68156e942", + "code_tokens": 32, + "prompt_token_ids_sha256": "2cddd75f63a682a8057ee1c480563458d86420a8e40629c3432a0307f3cc3736", + "prompt_tokens": 113, + "task_id": 91, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "3eeb5f3de473dc1e96f6ac50f4f4546cdad752e4b85766173377782ab33ce1b5", + "code_tokens": 53, + "prompt_token_ids_sha256": "e7abb7b690292ee9d260fcb7dce173c01f97d814c0df3e753f6b0b6b20d58c1a", + "prompt_tokens": 85, + "task_id": 92, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "51a9d2b9d6a4c919c7cad1737769e77c0525b60b683a5bd12f15eb961a2ea171", + "code_tokens": 51, + "prompt_token_ids_sha256": "8c399e91f489918832679021affdb6c4dfb1f4ac59e9b0899ac000f45d901567", + "prompt_tokens": 80, + "task_id": 93, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "adf34317830481522a1887bf5b65ffe1f36cf162ccdd640a9498126cf37d400f", + "code_tokens": 36, + "prompt_token_ids_sha256": "f3a9a8c9fc0e1cf71bdbc0f2d18fd12a2dfc353220ced456a1e3fe52512d7db9", + "prompt_tokens": 153, + "task_id": 94, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "54b4abf6744d8d389ccaf8687941e46aeb569c2cdba30eb3788fa5fe936e203d", + "code_tokens": 24, + "prompt_token_ids_sha256": "699d75b92e11ec17e1e042775c8f37c27d836e8d93b0d8a623c814189da3e416", + "prompt_tokens": 103, + "task_id": 95, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "98b72f4568ee3c30cf02cafaede5aa425114aae99f11f38decc83e7825ba8682", + "code_tokens": 39, + "prompt_token_ids_sha256": "f87c8ff4b4e11a7c780f6b65f67b97da4f3f5b3fef2afe56c740dad7b4e3004a", + "prompt_tokens": 71, + "task_id": 96, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "292969477ee44c5d19674f97232ff9038e50a17305f0b9274354be7cd502690f", + "code_tokens": 84, + "prompt_token_ids_sha256": "a2a8cc78a2c3e409084301e283b962b7f7663b83d9ca8b76191ceaeecc74d99a", + "prompt_tokens": 330, + "task_id": 97, + "total_tokens": 414 + }, + { + "code_token_ids_sha256": "9007a4333b815b1827f5dc5cc72f85f78a6ba66917edf3b2abaebd8f574ad698", + "code_tokens": 32, + "prompt_token_ids_sha256": "9e5404764764474f325c5a4654ed2f733a86ade6f8a7f62f657c46bf105fe5d6", + "prompt_tokens": 111, + "task_id": 98, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "ae6dbc9b8c15edc86d4a87b3c0e6a455e88fba0e3ca7f58cd381ed4654b7b9f6", + "code_tokens": 19, + "prompt_token_ids_sha256": "377c17235fd40a9cc91da3f68b3c853aa9b689f5e13d1d2cae48c59f0dec6f3a", + "prompt_tokens": 86, + "task_id": 99, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "e91e0510930051525c5f3b09373c3e12a4bdd5e246b84b8f1ee83144487d340a", + "code_tokens": 50, + "prompt_token_ids_sha256": "9027f1ddca1663ee908227a774d331e0c5b9058e2b5da39708345226a9a6f6ec", + "prompt_tokens": 86, + "task_id": 100, + "total_tokens": 136 + }, + { + "code_token_ids_sha256": "97d799a2ddd249f52f17e5f0dfe3554af676e26f6505e93b92fea83b11f1045f", + "code_tokens": 71, + "prompt_token_ids_sha256": "ca383495b10d22d89821d81ab2ca70caec36803e5c7bcc22cf6673aef5994076", + "prompt_tokens": 123, + "task_id": 101, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "264fc707f93279ecc2c2c32a9e03c7d7fad75b5d44c5be8df38fb818e4bb130c", + "code_tokens": 28, + "prompt_token_ids_sha256": "9b597d762024ffb86715e237f857651d202a6b978c8fe45e3e2e2fbd9d7ad65c", + "prompt_tokens": 81, + "task_id": 102, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "59cb42d2a76c4cee786f9d5deb090cc246504a49034ee544bb509f58d67ba5da", + "code_tokens": 76, + "prompt_token_ids_sha256": "d067807060427c6100fbf5c39d8686b2a14ed89e6cf79bae76d8da40dfc7062d", + "prompt_tokens": 83, + "task_id": 103, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "f4d5a2fdbe065c018862e148c2dba762e4d2992019792236edf2befe31c7c9f9", + "code_tokens": 34, + "prompt_token_ids_sha256": "70166cf7cfc37f3b3f45aa9dccf85b55b3e1abe4fc238661d224aa93620d795e", + "prompt_tokens": 185, + "task_id": 104, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "e641a23de5a23d2580b700a135318d4520bc54fce05b7496b8975803dc93d3c7", + "code_tokens": 11, + "prompt_token_ids_sha256": "41fdd0f7c571355d8524c3e2d212a16186c7458413d99249eaa7541b290d9002", + "prompt_tokens": 72, + "task_id": 105, + "total_tokens": 83 + }, + { + "code_token_ids_sha256": "d5de986de5810799534a50630a394311b1cb315566e44e464a6086dd705aed73", + "code_tokens": 31, + "prompt_token_ids_sha256": "5455224444b6ef5531536d5a7b626ee7b056748abaceddb18067eac6d4b75030", + "prompt_tokens": 154, + "task_id": 106, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "7f51ddda22b70adcf2c00962e05933ee9fe410af5ad541be6e5eedc01672371f", + "code_tokens": 118, + "prompt_token_ids_sha256": "ce39296e861c64d4db594bbb3c85a2f399a1fe32bbae70509728384c85447c2b", + "prompt_tokens": 85, + "task_id": 107, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "fdc8c884a12903eacbd78ca7a3db379117cc926d4c6e174e8707dce026aa46f3", + "code_tokens": 60, + "prompt_token_ids_sha256": "e82a7f107512a0c0f6f48feacbacc388f439efb17a777a1749624fa3764807d5", + "prompt_tokens": 485, + "task_id": 108, + "total_tokens": 545 + }, + { + "code_token_ids_sha256": "c0a1eee1cae5708feab0c624ac73ed90bcf2b874f00dc481296c1f8dedafabc1", + "code_tokens": 47, + "prompt_token_ids_sha256": "f2e758f96a12029f6c309def6b4110816b91a6fb851d0f2d62fed87993dfe510", + "prompt_tokens": 98, + "task_id": 109, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "537234c1c3b93d9026bf0f6d8fbdd14825607c4ac37bd97892cbe986c18f0c6b", + "code_tokens": 84, + "prompt_token_ids_sha256": "d92d4e72a0e83f30b7571e8b2bc17c19d5856bcbf4636b644e528bd2bb908e16", + "prompt_tokens": 288, + "task_id": 110, + "total_tokens": 372 + }, + { + "code_token_ids_sha256": "844ff75eea90279f76b626ad77496218862e5c5e3b07210594e6ff50074e7f6b", + "code_tokens": 27, + "prompt_token_ids_sha256": "36bf67e6d2eafae1ece26731ef973b60efc3ed5585085a2fad5661fc2d87e352", + "prompt_tokens": 248, + "task_id": 111, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "4fc3e9ccf26478cad737fde0881c6b125630fe3dbb0e51ec1c3ee7040f2368f9", + "code_tokens": 19, + "prompt_token_ids_sha256": "a65cf4432d2c0cbeae7bc4ddf1fb01dc68de9445edc3986bbbc7fc05ab6b2cb3", + "prompt_tokens": 72, + "task_id": 112, + "total_tokens": 91 + }, + { + "code_token_ids_sha256": "b7be982b00f7558a8d608825d18470483d2f75457a585d007c24411984154bc5", + "code_tokens": 114, + "prompt_token_ids_sha256": "4b92b23712dcc6d42c6fdcf9a49b1e7c78c6553d2d9ac1cb5f627917c4d0b9f9", + "prompt_tokens": 68, + "task_id": 113, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "2b076988f3d4b45bd5cbc5f0d7cd0e6d8283c4b08b048eb01e77acdce6cb6249", + "code_tokens": 40, + "prompt_token_ids_sha256": "7035d9971c839aab6000daf68b23c1d3e7deb21e9d960c03d30d37dbefaac9d6", + "prompt_tokens": 315, + "task_id": 114, + "total_tokens": 355 + }, + { + "code_token_ids_sha256": "b584f117c73c95b9cd42ccf48e88ea1a8db8224fa02b83266f187a2a4ed1cb70", + "code_tokens": 25, + "prompt_token_ids_sha256": "faef1334581a007b0ef1bcc6ee88fa48b5aefa4c8e670f2ca7e36af2904d81e4", + "prompt_tokens": 76, + "task_id": 115, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "88d80fa2fea0885de933125b857030d0d22258c74b09eb6f0e3a84da65270771", + "code_tokens": 23, + "prompt_token_ids_sha256": "ddca73e02700cd5e2b66607dc1e465c34e925d7108c9a32a42b5ec5ff2cc3bb0", + "prompt_tokens": 86, + "task_id": 116, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "6e85810c50a9fad2e8ae3f05a7a9b7f6194dd87ac3caef5bf443351f7ec6c2c9", + "code_tokens": 77, + "prompt_token_ids_sha256": "796779c06f14f120a1b57af498bb27993619bae06d105bd12ad58d16c565e01d", + "prompt_tokens": 297, + "task_id": 117, + "total_tokens": 374 + }, + { + "code_token_ids_sha256": "3531f86c9ac2069b2df809f5732d106e2cc3e71426ac197caed56fe392c56c22", + "code_tokens": 19, + "prompt_token_ids_sha256": "c1a7932e793602d3b349ae7b17d04d26e9e4925ee890fdab1a38cd2d1c52182f", + "prompt_tokens": 105, + "task_id": 118, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "6f8d7850d23a4da7f271209b269389dcedec6d9fd4ab92d477ec47475f5cdd04", + "code_tokens": 37, + "prompt_token_ids_sha256": "9f8e317ad291c50692bff5613a5d21da55642d0918f82e035c77dd06f40485d5", + "prompt_tokens": 117, + "task_id": 119, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "486629e5c3247ecfa794941e6e7709150137e281177c870fb82b4d9d498a9873", + "code_tokens": 33, + "prompt_token_ids_sha256": "1642a22406b3b950ffa6b5336deea64c49aad64e64c0849ddfabc71839562457", + "prompt_tokens": 143, + "task_id": 120, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "e75c71e50c510cb86fcc27dab32cfc6ee56a2601332ed76d932856675be26709", + "code_tokens": 95, + "prompt_token_ids_sha256": "947619a5a3a5429bd35bf00bafc999280cdf8237f67bf6039dad75bfd7c67877", + "prompt_tokens": 152, + "task_id": 121, + "total_tokens": 247 + }, + { + "code_token_ids_sha256": "ce64ba080056dca5433f061cd5c4df7b7aa779d5ab2b90108b85e6da9ddec4ba", + "code_tokens": 125, + "prompt_token_ids_sha256": "7d6dd1378053d3bc03116ae54c545a3de9c653d438de641751ccb577a4258b71", + "prompt_tokens": 77, + "task_id": 122, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "2978ed2e33a03e1113adc01a4b3f23279974a15bfcf912230b83924a29878342", + "code_tokens": 168, + "prompt_token_ids_sha256": "629117cb305bed6ef535b6aa11dd84f7d4e0ec829779fd9123bf6011d765cd51", + "prompt_tokens": 85, + "task_id": 123, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "a96a0ac08047d37f440ef003cfd47cf2a7067319c8d164d194dd77d0f0fd8502", + "code_tokens": 31, + "prompt_token_ids_sha256": "3842943b68e2181aed711d53a4a9b52eafb41e6c3ac70b39e85da2b8f742a1aa", + "prompt_tokens": 122, + "task_id": 124, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "a7426c7fc130b39623116d16c8d81abc82654c0d6949c0254b77b7da11049b1c", + "code_tokens": 83, + "prompt_token_ids_sha256": "d3ed852055f44583d5b690a874f79df3276e70f322ea0ba8fd01f139b39a9cbe", + "prompt_tokens": 125, + "task_id": 125, + "total_tokens": 208 + }, + { + "code_token_ids_sha256": "5eacd7bdb126bc851fc0ff66212aae8746fd9e11cbbfa9c347b9e086d5ff71cf", + "code_tokens": 50, + "prompt_token_ids_sha256": "71cc29143428760a041cdc13f587d0412a2c1bc02d6af3d73f131d770569a8d5", + "prompt_tokens": 83, + "task_id": 126, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "17953e34e753459fe8d60c700786a0550b98683f74c45cfc5036fdb32ccde99b", + "code_tokens": 69, + "prompt_token_ids_sha256": "3e023bda4119a2d0b3c7a57f0b12a0a6b2db8dae34b52ddaf85530d1cca71c89", + "prompt_tokens": 78, + "task_id": 127, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "f44c28c45126a5cf2ef99ba8923852d534d9b0bd0d5871248418278075f649aa", + "code_tokens": 50, + "prompt_token_ids_sha256": "1da633fd8226e8a9b8587b3763a47b92b46d5dcd9ccf37b79d48636f345b5dc8", + "prompt_tokens": 94, + "task_id": 128, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "9eb624dcf6e38543bcdb0bd182f88cdca66325bcaa4987a3c9e068fd7dae9717", + "code_tokens": 161, + "prompt_token_ids_sha256": "34a97210da86858a0a4a8c003b35a3fa3027fb03ae5d9cac88ddf554e3f16be2", + "prompt_tokens": 164, + "task_id": 129, + "total_tokens": 325 + }, + { + "code_token_ids_sha256": "6616386c5498b49da69f9d54e3485fec331b0d5425d6bee96169bef3f56b8f1b", + "code_tokens": 53, + "prompt_token_ids_sha256": "f258a2e044ba914ac20e34fa8536b247ab956cea825cfe198aeff7a42f66ea28", + "prompt_tokens": 198, + "task_id": 130, + "total_tokens": 251 + }, + { + "code_token_ids_sha256": "d4e6aed3f3b86d9ca64e3278498a2de74aaee170b0883492e83c3d5cb3633eaf", + "code_tokens": 97, + "prompt_token_ids_sha256": "a50dbed60cb8d3b9e69ccd88ccf4e6a6c2cb7d9d71c38a4e38c90aa08df74f7e", + "prompt_tokens": 80, + "task_id": 131, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "e5f570605a252bee0f1cbdeece0da75a13dc82f24b8adc9be1d064263e531335", + "code_tokens": 22, + "prompt_token_ids_sha256": "6f88f1b009611531120ea9174c27408e78d13322183bdd69c81bf5646bcc1e51", + "prompt_tokens": 116, + "task_id": 132, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "baf9073a4b9fbfdc65b87e7371db10321be91977995767760afff173d67e793c", + "code_tokens": 34, + "prompt_token_ids_sha256": "36e2cf063de71c662037eded7c9df41cd1f420f94fff71589ed5f0b76995b80b", + "prompt_tokens": 171, + "task_id": 133, + "total_tokens": 205 + }, + { + "code_token_ids_sha256": "9d9e590898ed5a766797862829e6cba044bd1aaa799b8c1edc21ea255bb4f8ce", + "code_tokens": 81, + "prompt_token_ids_sha256": "1d3724d3f7244584fc7de535c78ddb427b52120b1298a1fb2c4af13a0adbfac0", + "prompt_tokens": 109, + "task_id": 134, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "4bdbc0eeaccc3c56d81c60adbbbd38d4932b90ff750dad9694e8fdf92d1593a9", + "code_tokens": 17, + "prompt_token_ids_sha256": "11396a5bfbd18abcc16c0e2cc3d20b2951bfa91d79c6a7b2ff32519f334f8d3b", + "prompt_tokens": 75, + "task_id": 135, + "total_tokens": 92 + }, + { + "code_token_ids_sha256": "549f9725c32870d7d856ed10693d09a2d0edb4e142576b931ed9735991ec3cf3", + "code_tokens": 181, + "prompt_token_ids_sha256": "9cf4c5577b2c6e75845ebd7fa39c040b6116449ce7f51cd395a12c19139e92f1", + "prompt_tokens": 84, + "task_id": 136, + "total_tokens": 265 + }, + { + "code_token_ids_sha256": "689692118b17be82fa5ac184f217fa647bac166b92ccc198908c049630658e8e", + "code_tokens": 63, + "prompt_token_ids_sha256": "01b16977e551bea64428614c7e832a6f09ee954a0ea9668ebf6f92fbd0a5dcc3", + "prompt_tokens": 174, + "task_id": 137, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "564decc4a34680188fd13c41921d427063e53178df07ff97547126d766544b91", + "code_tokens": 35, + "prompt_token_ids_sha256": "599ab7b99b633de851604ee9859dd8db064b3285e5ab57d70db7b763c0768395", + "prompt_tokens": 98, + "task_id": 138, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "6b7ba9a3013e10b420864b6fda5d58e18a147c0de70a7dab445c530a4f9e4ac9", + "code_tokens": 25, + "prompt_token_ids_sha256": "39f109a083b5acd82fe44a37ff87160de184591734de89d65eaa0c0abeafb713", + "prompt_tokens": 110, + "task_id": 139, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "0df01c94e7cca81c701578f8c7a447b21a3393b75bf763f89d1b341f70548f09", + "code_tokens": 60, + "prompt_token_ids_sha256": "53a43e508589bf4900b64b780bb292e9aca2e8e40ef1279998724859435ce43e", + "prompt_tokens": 190, + "task_id": 140, + "total_tokens": 250 + }, + { + "code_token_ids_sha256": "7da1371e719db577e72e2f0412b53df3c07794a33349ef19c3df9e0e43679c19", + "code_tokens": 86, + "prompt_token_ids_sha256": "3b3431f0fac5d6ce559519662f2b8b37e0e81fb13086cbe25acb9fe5f8602541", + "prompt_tokens": 178, + "task_id": 141, + "total_tokens": 264 + }, + { + "code_token_ids_sha256": "8a5b2f87c5d29af793c4c8420770bcbb62c56cd2e6d94ce5fe02cdf58d378c65", + "code_tokens": 40, + "prompt_token_ids_sha256": "23a6793533e9e3da40682d6f9bd0ef2daf1bd20cc768bccc1ba007ead287530b", + "prompt_tokens": 204, + "task_id": 142, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "b6cbc21701f6e745d7bd860870f9153c711dab6787293f07e8f1ea21a99fead1", + "code_tokens": 27, + "prompt_token_ids_sha256": "071cf1d4ae96c4b61343247eb8371377a3f225bffbb680f98bb359b059c9cb60", + "prompt_tokens": 131, + "task_id": 143, + "total_tokens": 158 + }, + { + "code_token_ids_sha256": "7ee1df7ec87fdd3f7fedba92300d4a80378df9448a0c345585a91eaf15295f17", + "code_tokens": 52, + "prompt_token_ids_sha256": "74ef2fe9d92387280006c68daa96bdc2d24604a1bfb0c6ea0e1d10ce4bb4f508", + "prompt_tokens": 122, + "task_id": 144, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "b468182b74c335010da580a00f4e9985d0809aa0c7ab13b0abd1a5365a28d260", + "code_tokens": 71, + "prompt_token_ids_sha256": "2a382e37b54a56b0bf1df0714336d8757f3e8c295f46a06ca7da1059f8805075", + "prompt_tokens": 107, + "task_id": 145, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "a022ac9b6a93fff76bff6d958b408917d3c86b02f3780818fa45b088226013a9", + "code_tokens": 25, + "prompt_token_ids_sha256": "366019695da8006bb80746fbf3114b871d3de6b8d4369a09946a5aefe7f4d774", + "prompt_tokens": 72, + "task_id": 146, + "total_tokens": 97 + }, + { + "code_token_ids_sha256": "0ed86f0b0e9722b3597b676d45edcd554b1416116dfba80e4856cad582f1a567", + "code_tokens": 103, + "prompt_token_ids_sha256": "da966e29602938b5c2fe16d433f8dcce5227bc034cf04fd06eaabe0cda3acfb2", + "prompt_tokens": 173, + "task_id": 147, + "total_tokens": 276 + }, + { + "code_token_ids_sha256": "ed73b08d5d02ac3f7b5d079d0f7c9767f9d9523d74abb4653fdc1605961a0f2d", + "code_tokens": 116, + "prompt_token_ids_sha256": "8a5a31ee72ca5430b1f05cb4f02421140235bc904b593ac21fd072e2ff3195e9", + "prompt_tokens": 83, + "task_id": 148, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "e57a1827f10e52ab5d9ffd8421e738082d3e36f9e7d9c5ec88df2a0c606c43ae", + "code_tokens": 107, + "prompt_token_ids_sha256": "cae2283487303ae3601903c225d73ad0532d1621fa743a9be40af256e59002b2", + "prompt_tokens": 159, + "task_id": 149, + "total_tokens": 266 + }, + { + "code_token_ids_sha256": "904cbf900ef883a906bba7b85c7e3d29a705da833bc96da0d284df6b0fe8600a", + "code_tokens": 54, + "prompt_token_ids_sha256": "fac22133bdd4ede2bfbc830f401e2e6db582a6fb225231944ec04ac154fa56fa", + "prompt_tokens": 90, + "task_id": 150, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "c7efbfa77840fb4d43b5f22060598be59661b1349f0855820492e75050896340", + "code_tokens": 46, + "prompt_token_ids_sha256": "f249b2550919b736d8a495cff99d28f4d4388d9a51cfbca7f247af2fb68e2dd7", + "prompt_tokens": 88, + "task_id": 151, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "22574600d8fd3baf49f33aa435b31c24a878bc0b08f0ac127f5806985eb5506f", + "code_tokens": 170, + "prompt_token_ids_sha256": "25db7506463ce95eb3fdf9d2daaff54040c8cbf1ac4a2173b864ab882204aaff", + "prompt_tokens": 186, + "task_id": 152, + "total_tokens": 356 + }, + { + "code_token_ids_sha256": "5d016cdd76e1476006301187f4f79082abbf80e89302f44dbadbc81003c4c4f0", + "code_tokens": 45, + "prompt_token_ids_sha256": "682738107e75c17c5e1a0eb1701172e53c0a6f072ebe2c8e234f51a2c37448b8", + "prompt_tokens": 135, + "task_id": 153, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "074504890d25da67e4aea86af259b1ecc221ff949a23ef41a251e4a7e91f96ad", + "code_tokens": 24, + "prompt_token_ids_sha256": "2c9764e9478599482677e38c19dc89beafde096786c93720c12e2786c8142d7d", + "prompt_tokens": 192, + "task_id": 154, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "915c9cf6de4fbf781498970a9d39af9e1ed07bc46d94ee5e1c6c15c9600f5221", + "code_tokens": 78, + "prompt_token_ids_sha256": "9aa9714bf9b4afa4f5e7b13de01ff54046140af029d5299038a632fa0571843e", + "prompt_tokens": 81, + "task_id": 155, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "09b9f34b3d9d44cd86269b6dfcb39d167c10d98e661d9c2af57fa854c0af4dc5", + "code_tokens": 33, + "prompt_token_ids_sha256": "37472796e556fae703e468a8068594040200a3cb26d13e058146f384af6aeaac", + "prompt_tokens": 182, + "task_id": 156, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "12efef85c970e63715ca0d24fe58a7a32ca02f774cd6aa43452fad5b93960758", + "code_tokens": 32, + "prompt_token_ids_sha256": "f9cb5f32a410792999280c775f85d906da92f24ae4bba3aabef9625045d99101", + "prompt_tokens": 229, + "task_id": 157, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "e11293027cc3d8733b6030d5625dae88db0ae1ac95c35328356d315fd34fe74b", + "code_tokens": 77, + "prompt_token_ids_sha256": "52836ad4ff79706c605ba331260bbe8fd142b6f94715c1377050dcbfd0d918ec", + "prompt_tokens": 113, + "task_id": 158, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "8d52e4ee16b599d3d77b9b66fa0a9182b33aa6496afa0dbb73ca1bbb10a2a143", + "code_tokens": 222, + "prompt_token_ids_sha256": "5354dce594727993a6ef03a6fc63efa08a054eb51e740dec615de86f305dfb56", + "prompt_tokens": 78, + "task_id": 159, + "total_tokens": 300 + }, + { + "code_token_ids_sha256": "a8eeb602e626a96416653ac2c4bc28bfa8c2b0a0881b08244cc1d7e78d1220be", + "code_tokens": 86, + "prompt_token_ids_sha256": "44f0871200f30071ddd05325a570e4f2ec5d0798ee4f885f729275d9d10c1849", + "prompt_tokens": 115, + "task_id": 160, + "total_tokens": 201 + }, + { + "code_token_ids_sha256": "7b65e676e5ab78d91af7cc5193f0e94f882ff05c2d8404d27acee96c145d90cc", + "code_tokens": 31, + "prompt_token_ids_sha256": "a1c36d905417d4776d2841dd20513511e19b2d226870869f415156fc4274667c", + "prompt_tokens": 224, + "task_id": 161, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "3124835db76bd7740908fb541d9fdbe2f6508112354ce3de32d4d15475af6de7", + "code_tokens": 34, + "prompt_token_ids_sha256": "16b57728c2fec8a0ec830129c4f2561c5eaa668ca1d77b5df9817c56db134991", + "prompt_tokens": 85, + "task_id": 162, + "total_tokens": 119 + }, + { + "code_token_ids_sha256": "2827091fe4281af4257f6f005c682e424937e0bad7ab417039aa510b1cdb3562", + "code_tokens": 38, + "prompt_token_ids_sha256": "d9fd25640d0e1c577e97e043f2f06bd1de1c06908c6661ccbd6608076408560e", + "prompt_tokens": 122, + "task_id": 163, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "0551a9ee47a087934926b90c67ecc5da61adecfdfc7cdca82189f456d93cd1c1", + "code_tokens": 93, + "prompt_token_ids_sha256": "0d95cbbcbd368aee1958a03c0e9aa8be159a6f040c88eecc93889175468704e1", + "prompt_tokens": 80, + "task_id": 164, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "4e613d573efe4dd3b2067ef7ba4da3b38cb9b8afdc3248e53ca4b0b29391cead", + "code_tokens": 70, + "prompt_token_ids_sha256": "5d9bf90c4f0b571844b77905850a35ad967f44905df5c987b68d7a2eb301545a", + "prompt_tokens": 91, + "task_id": 165, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "4272afe984164f919ce49f43f429e81b14ee48de8fbe978f6163a875261bcb91", + "code_tokens": 67, + "prompt_token_ids_sha256": "8849c0a85df756844d09323db8fdc60e246db719534357b8cdac26365ff999fb", + "prompt_tokens": 108, + "task_id": 166, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "85ea611135278d13118510c1cf313da6445dc8a5b38067e0091900035a9b6fe1", + "code_tokens": 66, + "prompt_token_ids_sha256": "928b71e5f5b590f5c885ad47112df5529634161f57daf398436bb4f461e51d89", + "prompt_tokens": 88, + "task_id": 167, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "0bf09a28758f3b3f7588f778f864ff265f1af16fc345ec99d154966a6136f585", + "code_tokens": 34, + "prompt_token_ids_sha256": "a01f02e7b5c05d0290cc55067c96e2c12b94795d2e61b038bcab2343b24208f5", + "prompt_tokens": 101, + "task_id": 168, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "df2d40e8f7715bc1cf0789e9f679a2cf785dff867d3ed3fbb21f4372ed6329c5", + "code_tokens": 64, + "prompt_token_ids_sha256": "2bf0ee661ccdb5d66cc20a89abf784823938070c144a4c72f7ce60234b605bca", + "prompt_tokens": 74, + "task_id": 169, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "7d69269381fe4f6824d1ebee9776b7d9d80f9ea5a0edf47a4ba58229af014cf8", + "code_tokens": 56, + "prompt_token_ids_sha256": "f7076f337c4691b85ba0d5ccb591fae7cc7392a717a72a7236c17d88fc2b10d7", + "prompt_tokens": 168, + "task_id": 170, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "73f55c33613a8a3d22f686272239a28b232d5f8735e278febd6df99162461981", + "code_tokens": 21, + "prompt_token_ids_sha256": "caf28bc058105e85f000dc2e31bbbc4f9f94d104e04b15de748ea7d177db8a24", + "prompt_tokens": 73, + "task_id": 171, + "total_tokens": 94 + }, + { + "code_token_ids_sha256": "85bf02c7e7aa3707b61999f1e8614db2e114211940282fed3aac1ccc403720ac", + "code_tokens": 60, + "prompt_token_ids_sha256": "ff7c369139f6405164386d8e2e73fda41879467391139991aa718aedaae99a3c", + "prompt_tokens": 125, + "task_id": 172, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "7e47ac58e15d67f48299fe39ce222db542baf856bba9f681516893762764d549", + "code_tokens": 29, + "prompt_token_ids_sha256": "22e999d411ebf2f19af4a88ef7d26888d1fd50722dfcc59829d922c7e98c6af1", + "prompt_tokens": 132, + "task_id": 173, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "1768391d2807cc702db439d00c532ebaede6d9c9e1ec7a83b11b78ebb9360f23", + "code_tokens": 34, + "prompt_token_ids_sha256": "8cb028ca4686b5fc37475841c42ef74c2e3e5da51666811ea2abdfa006ed7533", + "prompt_tokens": 232, + "task_id": 174, + "total_tokens": 266 + }, + { + "code_token_ids_sha256": "7f6e29484730f27049fd3cfd2b1ec5d46dafadf49b442a1bf30da8121ea0be45", + "code_tokens": 88, + "prompt_token_ids_sha256": "7e9a5c014fd78c4a1ed5540b2f45381d60c13a3bb096ec1c05508140d3a44c25", + "prompt_tokens": 75, + "task_id": 175, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "50873befcdf0de466218c1eec911012f8d68596b30806959e92cf699765d52df", + "code_tokens": 17, + "prompt_token_ids_sha256": "6476fc372890f2f90c93627b2bdb3a98d8758e64b47fbe8381aabc4ac8f5cbed", + "prompt_tokens": 83, + "task_id": 176, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "c28a73f62c4623227184363272dad3836e36f22c90f786cded9894cbb2c43918", + "code_tokens": 35, + "prompt_token_ids_sha256": "dc54c20c626d0213c025df696ecf99c190aef0b8e5d082d051e413eeeaad5129", + "prompt_tokens": 87, + "task_id": 177, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "7a6ac6c910fd91ed658a40b0d0e2f88d3f4d2953c02bac4cb4ef0f7740afe25f", + "code_tokens": 48, + "prompt_token_ids_sha256": "fe59b7588a3c4e4a70b2ddc76778b24ae7ff8849aa4d679a111b65fd5273f4f5", + "prompt_tokens": 85, + "task_id": 178, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "090e997d41882dc3f0b2038b28e6c8d8a9a741c72d9c3cf8e757b4c718d2c7c8", + "code_tokens": 130, + "prompt_token_ids_sha256": "f52906afbdcce4ba75b01631cbcbb14b00dff5be4348d6befef63c509670774b", + "prompt_tokens": 80, + "task_id": 179, + "total_tokens": 210 + }, + { + "code_token_ids_sha256": "0a058af7fdde81295fe95505497e4a38e5ea57e1213326d2ae905d1959574821", + "code_tokens": 65, + "prompt_token_ids_sha256": "67daa8192d32751e6b4f25bbb449175207289f97aab6ef07e5649b804550f8b1", + "prompt_tokens": 162, + "task_id": 180, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "5208c3e810cc7749d5f3fde59a6201f00d34c5582dd2aa6076fc2601b729e4f3", + "code_tokens": 145, + "prompt_token_ids_sha256": "678207325db6664c299354b69b846989498a0360b04ad07e8a2a906dbf69ddfd", + "prompt_tokens": 114, + "task_id": 181, + "total_tokens": 259 + }, + { + "code_token_ids_sha256": "06bff9841ccf3b25fb98673c7c463c9c2a3ec6846313a7c282712ee7a227eab3", + "code_tokens": 80, + "prompt_token_ids_sha256": "c57c1a76612ddd83673f2608fe8888fb965bce261d20d71019e0a31888815500", + "prompt_tokens": 199, + "task_id": 182, + "total_tokens": 279 + }, + { + "code_token_ids_sha256": "a763bd0cc44e5d0c45350cceb01e816020c8a0f6fce2a9b6d4faef68e0232787", + "code_tokens": 70, + "prompt_token_ids_sha256": "2729fd6304e4fbce6c880536aeb13cd5b928bf8935ddb074e4f7831e841e6bc7", + "prompt_tokens": 134, + "task_id": 183, + "total_tokens": 204 + }, + { + "code_token_ids_sha256": "5faac4bcf032efc76d6737bb0af0d7572a045ead033e45c2fba0fc83696c8746", + "code_tokens": 25, + "prompt_token_ids_sha256": "15a335d8d84d797e5679b5034777a58ef034d895d41493485f796611d344ace0", + "prompt_tokens": 109, + "task_id": 184, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "d14e4db73704bd9a7c36255132d7af191e96f58c4522f83928d9bf245a73c319", + "code_tokens": 50, + "prompt_token_ids_sha256": "25263fd4d6b54c025a503f6a6a0e6038d9f55511c2425e22f90ee71032b3ef4f", + "prompt_tokens": 122, + "task_id": 185, + "total_tokens": 172 + }, + { + "code_token_ids_sha256": "c66c82c44f89b9eed4e16607051cbf859c65280e9cc902c8e7bf49830f98ea56", + "code_tokens": 48, + "prompt_token_ids_sha256": "6efdb3ae209ee8322c25430ab2f18bed11d64ae575728945497b8aa39f54cd91", + "prompt_tokens": 111, + "task_id": 186, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "ae341e0408c6fd63275a4d81d320bca3bba2b92caf0149da510ea8c693d2ab5f", + "code_tokens": 104, + "prompt_token_ids_sha256": "fb1dfb9b3ee30d5e8b81ad1fb09199a919c11793a38e75fa06667a56477e630c", + "prompt_tokens": 121, + "task_id": 187, + "total_tokens": 225 + }, + { + "code_token_ids_sha256": "d139db20f2cfb66831d25173ad82898b84bc546f661ff849066d774b22cdcbd2", + "code_tokens": 68, + "prompt_token_ids_sha256": "d839a7fee352db62223ae664372142609319762829eee2ea72698cf9b4827f26", + "prompt_tokens": 80, + "task_id": 188, + "total_tokens": 148 + }, + { + "code_token_ids_sha256": "a5e4823dc708d80e6c310565a812d8af835269eaa92220eb286f667ad6322ad1", + "code_tokens": 145, + "prompt_token_ids_sha256": "4d87cb11a678f889c1766dcc1492eb6a7e9e74814e07bd2d3e11ee2e089052f0", + "prompt_tokens": 109, + "task_id": 189, + "total_tokens": 254 + }, + { + "code_token_ids_sha256": "5d6aa72826025fa3b03ed5fb9dca68d9cbf36cd784e753fbaa58e150d16745b1", + "code_tokens": 41, + "prompt_token_ids_sha256": "9f98dc121e182da76da9abfa4f087c1e10189e406b50f35b8b916469cccde28a", + "prompt_tokens": 105, + "task_id": 190, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "60d6e8b118a73813a82e328770401fdfc3d8c54de0cbf544289865c7c65bfefc", + "code_tokens": 53, + "prompt_token_ids_sha256": "b59a12ce39649bf7b7e727d20914c95e0f0cc46392e92daeeaec1bf558adcca5", + "prompt_tokens": 71, + "task_id": 191, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "ed97dcd06f5692c66e65a9eaa1dc6609db8589a3bb4bd9a3e903e6c4d628026a", + "code_tokens": 58, + "prompt_token_ids_sha256": "5ce7c5026ff360de3446160e5d6e5114fd1a7f828c83d89acb16fab0b70fb040", + "prompt_tokens": 74, + "task_id": 192, + "total_tokens": 132 + }, + { + "code_token_ids_sha256": "b0d5c23ea5188663e3ce069d380a3ff7370562999b6a49c2b4bc68481d53d10a", + "code_tokens": 24, + "prompt_token_ids_sha256": "13a2bede5c228292c472dee015a863e023e3afe4e24dc952acded14541580956", + "prompt_tokens": 191, + "task_id": 193, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "9a9b4ba27395b31f7f121f958a004e16c3fe02857372c6796fbe2a9dfd5c7433", + "code_tokens": 89, + "prompt_token_ids_sha256": "3c4d527dcf5228766e1a01af841a25057d709fe185fdd0a55e3e99e66d9cc408", + "prompt_tokens": 84, + "task_id": 194, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "b8269674bcf1f7c7c5c7c1449ea1fbc997b100f338734fefd665b4016fc86fe0", + "code_tokens": 101, + "prompt_token_ids_sha256": "c7c64aec93785460801b7e4c75ff155c50b74004fa9566ec9ccb95d35adbe545", + "prompt_tokens": 114, + "task_id": 195, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "07610d933ee70dae6bbf3c30154d4fd7e987ea88593696836a205759e7e98bd7", + "code_tokens": 33, + "prompt_token_ids_sha256": "90fd977da03056837c645c68ef7f2d89acf8b4c6a058e1a5f027f71ba7a4af30", + "prompt_tokens": 232, + "task_id": 196, + "total_tokens": 265 + }, + { + "code_token_ids_sha256": "a35ef97ab6eacd1bd835f2bfd423febbb7a9f78f1a5cf0a49234b9e151aac646", + "code_tokens": 50, + "prompt_token_ids_sha256": "ec124878a477c9ff740dc702354fa802305205576d0a7b7b5e94444cd1a502e3", + "prompt_tokens": 234, + "task_id": 197, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "0d567a2e28d792776fbd484a0ecb3a83b51fd2e10eb476523e7146a10b9aae96", + "code_tokens": 58, + "prompt_token_ids_sha256": "1bb457ccd1edd997976dec26b4cf4326d63604fb2b8c6bcc83a68ea458deed22", + "prompt_tokens": 122, + "task_id": 198, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "afa5d4703c4163f3ef1fc78769320bc8fee51e3f62f22f7f72177f23a62658ee", + "code_tokens": 62, + "prompt_token_ids_sha256": "c9d979e6bc3c062fd44b9ebf8aa0c805f4853ede33ce5167a53bbb690fd63db2", + "prompt_tokens": 92, + "task_id": 199, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "f2c3817b940baeb158cb474b534ee4743a1d0dc55aa9dc36cf52ba5ceb99eade", + "code_tokens": 42, + "prompt_token_ids_sha256": "828b3078cf641ebec87f86d0d196467e90ca49c2ef73c612d0c06d730a4206e8", + "prompt_tokens": 164, + "task_id": 200, + "total_tokens": 206 + }, + { + "code_token_ids_sha256": "cbf6c3afb4270ad87cf4239319d648167be413e25e69cbdca943d47d196029ff", + "code_tokens": 15, + "prompt_token_ids_sha256": "166f51a6361aacf262c919b17aa855bcaba1600be2f7800c65a0f36798ea05c4", + "prompt_tokens": 84, + "task_id": 201, + "total_tokens": 99 + }, + { + "code_token_ids_sha256": "986fa8cdc9d20da4c5cd62aa1a6ae3040012059e11d5277509b5af120020b774", + "code_tokens": 57, + "prompt_token_ids_sha256": "3a723c5fc4c6b48ee08f5a1195883e36ba06499bd382097ac5d672ec30689d2f", + "prompt_tokens": 69, + "task_id": 202, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "a113f943d3232fa433abf1fabf98be2d5a2f4c10486eaeda5b8414ff4d4eb2d5", + "code_tokens": 58, + "prompt_token_ids_sha256": "c8ca01ae05cca614e76aeb2c07f6cb574e5cd05b5d0767a828b6c7c09bd8ce22", + "prompt_tokens": 83, + "task_id": 203, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "cac2cfbfa6ebdca0db76a33007956f9b5f063d928cd4ed3317d4d3a23b6a231b", + "code_tokens": 45, + "prompt_token_ids_sha256": "b007b77e856e8ec5a5dc817b6e6280d4c9865bb2a2957fff05aa6388b5b9c3c8", + "prompt_tokens": 81, + "task_id": 204, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "cc665b15da9247e02d581103c60cce6bf1a0d29f2313ad531cad01744953354e", + "code_tokens": 32, + "prompt_token_ids_sha256": "e0b2d98e36365d9e77988f1aa14ba0d0319e0490372b2c226efec8b8890f2dac", + "prompt_tokens": 178, + "task_id": 205, + "total_tokens": 210 + }, + { + "code_token_ids_sha256": "09f19e1011b3ba62968a600964e1ec39a478b6fa36b2822791ce18e23d6e46ac", + "code_tokens": 39, + "prompt_token_ids_sha256": "1a8a0b74b650280ae43ec6d1d6cd5f43162215adb528aaa3caa0a25b9229b36d", + "prompt_tokens": 180, + "task_id": 206, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "bc6c3c51d82fc42a20e78c3fe1cd20e42e13905ff0bc1292e944667347244ef7", + "code_tokens": 136, + "prompt_token_ids_sha256": "2a40fa8bca66ae8fe488fda69e028cb3e7666c503c726a8266bb7a2774cff4e4", + "prompt_tokens": 102, + "task_id": 207, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "5401e9f353039fec9dd145b1d35afb150f4a0ad999549940222bd5f2119cfe98", + "code_tokens": 50, + "prompt_token_ids_sha256": "f18b57064600a5c68db29a2e18f7a7903256b62d1618f66696a029ab9e34aeb0", + "prompt_tokens": 86, + "task_id": 208, + "total_tokens": 136 + }, + { + "code_token_ids_sha256": "f58e9f2920ecd6fb3bf990b6cce6bd973174df3639dcd3a1e13dac8c5b7abec3", + "code_tokens": 34, + "prompt_token_ids_sha256": "89f590174b5a08b3bd04c2c612ea3da5746f635341c30b1125649f6c5dbf7f3b", + "prompt_tokens": 239, + "task_id": 209, + "total_tokens": 273 + }, + { + "code_token_ids_sha256": "97e4bbe3e436885e42cea753db4ad3f757b124cd7a2790d61005dbb1938f102c", + "code_tokens": 44, + "prompt_token_ids_sha256": "3b7461f32e0894b1fc598e3c65f5051bf455cfad1ab33fdc0cecd8fb2fb36a71", + "prompt_tokens": 113, + "task_id": 210, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "1db96b10534bb35bbca5c04eccc6feaaba0e34895fb8a6b7a081d299efa05635", + "code_tokens": 36, + "prompt_token_ids_sha256": "c15f789b4af27b11c1765fb70fecba72d879d7d2cda5e5488ce0b5006a41ae90", + "prompt_tokens": 72, + "task_id": 211, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "85a7f5857728b8a9c8af70514bf95752bc7beac5db9bbcdd5a259e3012fae730", + "code_tokens": 46, + "prompt_token_ids_sha256": "cae640bd293c193a0fd9869e1e49672c86b2a241cc1ed72adc4203e28f12edab", + "prompt_tokens": 87, + "task_id": 212, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "b1e83acf8b4cb425d70474c10abd7abc7c692ddd2a75c5c91e1d9e13ccc952a4", + "code_tokens": 48, + "prompt_token_ids_sha256": "a154fdbf54e4c26d9f0d8dafdd0ac4be1b327c8ffef4219b0c5c55f35c8df6a6", + "prompt_tokens": 192, + "task_id": 213, + "total_tokens": 240 + }, + { + "code_token_ids_sha256": "d0ba927eaf1737b41df05fc52b9b3a4ef44a74cec5e25e0729a4ffb45d01ecf7", + "code_tokens": 25, + "prompt_token_ids_sha256": "9e147c3bd288a523679c1d7e19f2828747cfa1573b6c67dbdd4e685df0f758a7", + "prompt_tokens": 114, + "task_id": 214, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "73e98d63c1c318dd63070fdbd353dc67def23b822fa11a4c5a09246966d71b47", + "code_tokens": 69, + "prompt_token_ids_sha256": "1d242f9989d3bacca8f2c7e506ab17e66045c0157061ef8e203762125b9ddb0b", + "prompt_tokens": 206, + "task_id": 215, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "b0ea084ce7bc7a71e7562cc8818a1ef31df55c4c85d5ff4c1b45e879b8f11413", + "code_tokens": 60, + "prompt_token_ids_sha256": "7f7bcf3ad6d41ecf4c626a880e7dff039afd9f42875d1a179fd559ac9ac833ee", + "prompt_tokens": 225, + "task_id": 216, + "total_tokens": 285 + }, + { + "code_token_ids_sha256": "de38e6e3a1764ad0177678550f1fd618b97616ac9c946db2dd80e6ce0e443f8e", + "code_tokens": 49, + "prompt_token_ids_sha256": "0c4dbb8b9d5dac3999183466b790f91740f4a0d24b354a0d6504fe3a74a12cc0", + "prompt_tokens": 80, + "task_id": 217, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "380ac12b1136a14b989f221ef4b04b7e189ec9416fb38ffbcd9be5e231a699ec", + "code_tokens": 44, + "prompt_token_ids_sha256": "1755292276bfe69746eac43c0e45c9086ad2df8b1e4e661b31d1245444898ff8", + "prompt_tokens": 85, + "task_id": 218, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "b717d54e1e569b50b32a9c61973d1d0adaf3444fd4507fb1b82db46568edd048", + "code_tokens": 81, + "prompt_token_ids_sha256": "a7414041095409c8e76e81c314fd86ddb19c4d47a94e756cbe74f4779ffe95d7", + "prompt_tokens": 180, + "task_id": 219, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "e1cec3b3ab423a8236cf9ae3f84a8e8a99b28e46c0b2c161059cc27a0d7b1ca1", + "code_tokens": 26, + "prompt_token_ids_sha256": "32383c5724e2d52280d67a1cbe310ebfab8a1e48661219e710e63b760f1849a3", + "prompt_tokens": 117, + "task_id": 220, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "d6ef8f4bc995bef359a1f6e722cb2a4e8f3974ae43c2b1ab2caa4f6aaf4fe25f", + "code_tokens": 32, + "prompt_token_ids_sha256": "97b5bb2b4625a0f77115248f16d9fc73abcdd88524ba92f9f8909c4f0600aeb6", + "prompt_tokens": 106, + "task_id": 221, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "475b43c3a4ceae032db29ae908dfc68ab08592448f6551138dc6f4a15bcdd573", + "code_tokens": 48, + "prompt_token_ids_sha256": "3237cf65c2cef968d5496bb3bc23252c780a7346b63803bed67fcd08d0e62708", + "prompt_tokens": 107, + "task_id": 222, + "total_tokens": 155 + }, + { + "code_token_ids_sha256": "3f24a5c0ef8ff166e016efda816d6bad63db4aced7957556dfc99eae29936cb3", + "code_tokens": 180, + "prompt_token_ids_sha256": "c675676eac006f650130a88fb8f2ad8c1ff536907c760823c093da6c43476e64", + "prompt_tokens": 140, + "task_id": 223, + "total_tokens": 320 + }, + { + "code_token_ids_sha256": "6e0cac86970b2cfe3f2fd3a2b0503b300b800c7b0803f20d8a2c631bab70aac7", + "code_tokens": 38, + "prompt_token_ids_sha256": "2936d60cb56c915430d4fa8e8728399005717f74cefb368817b81fce053f6471", + "prompt_tokens": 75, + "task_id": 224, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "34866b482398acc658f1baabddb841aa878563ea84a1c346c9bcabb653d7c11e", + "code_tokens": 90, + "prompt_token_ids_sha256": "f43bfc2879a8c32552b645dfffdfb06039bc65526f531d77def66eab53bc3392", + "prompt_tokens": 104, + "task_id": 225, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "abef974f6c39a8bd010f6d91d51cd54681b8f24ad10b71e6b7bad0d2b47defd1", + "code_tokens": 44, + "prompt_token_ids_sha256": "80a8c2f0dc4c11bccc662c0acd8279eb726049115abfe656bff402853f9a3f99", + "prompt_tokens": 80, + "task_id": 226, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "a783cac4b4229e50d41aa6c2736ad86dc6dc7265ab2360acd670210710bc53ee", + "code_tokens": 60, + "prompt_token_ids_sha256": "9d910fec15aa7d8bbd5e2e999bc5d02739ed0566f3752fb1edf12febe966271c", + "prompt_tokens": 86, + "task_id": 227, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "7e9387500fcc0957326cb4b4b1fb7ae5e2ba69ededfe2e4d689c33ffd4d97297", + "code_tokens": 69, + "prompt_token_ids_sha256": "db9a2071137b0f53d5c3b034ca1b081ea6d01ad43d92cc64d4ff698f6eea7793", + "prompt_tokens": 113, + "task_id": 228, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "47d4610c97a7b06c80b016c00ef196a037f8e834cecb2cb1cdf36b43b2363889", + "code_tokens": 71, + "prompt_token_ids_sha256": "96cff410463e11af70d48e4e1afbcbe76259e3cd4f2e0a443802b9b430a6124a", + "prompt_tokens": 230, + "task_id": 229, + "total_tokens": 301 + }, + { + "code_token_ids_sha256": "56b0b38e212ea05906be72fc78321530aaa48bf22be2218793f2eec1cdb1c514", + "code_tokens": 22, + "prompt_token_ids_sha256": "08dfeeb31c07a37b0a9d4e2f135cae30268fe6737ecefcb45e5dc8ec1cc0ca77", + "prompt_tokens": 87, + "task_id": 230, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "daebb9823bf70d74f6d681811d409bc5b7303194eaea763ec6f34db093026429", + "code_tokens": 215, + "prompt_token_ids_sha256": "7de30941067ec90af881026ebc10070653f2ff08c1a5ceb45eca13b4ada9f3fe", + "prompt_tokens": 126, + "task_id": 231, + "total_tokens": 341 + }, + { + "code_token_ids_sha256": "23cab822a328cb65b24da1fab6182bf4c848627cde6286b08b663fdbd5009b02", + "code_tokens": 26, + "prompt_token_ids_sha256": "e97752b024aa100e74d2d8c7c82ca70b20c70c1a6df1d2db6c73d392be54afe7", + "prompt_tokens": 231, + "task_id": 232, + "total_tokens": 257 + }, + { + "code_token_ids_sha256": "e2699c013f0b2c0bd2551a58d20fbbac0755b99dda5f2f35fbcfe1c2269ce88a", + "code_tokens": 33, + "prompt_token_ids_sha256": "9b0907adbfcbee6031d3e78fac17a06b6bdf25e8aee730028be0aff318aef97d", + "prompt_tokens": 134, + "task_id": 233, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "234ce61593f272c0ad5b711f5b306fa39ef751806a8ec3ac329b250aeaae9798", + "code_tokens": 18, + "prompt_token_ids_sha256": "d36eeec6ca1b857694096c551e87e09ad8cc4a27feaf1ccdbb9846e09c7c6c84", + "prompt_tokens": 64, + "task_id": 234, + "total_tokens": 82 + }, + { + "code_token_ids_sha256": "b239ca6968d8fd4f359e614a9b75b8862359576af2134d513977b84a00762b5d", + "code_tokens": 71, + "prompt_token_ids_sha256": "a97a589800360c271204da8bec930d10e65242c6e89ea8a3e949ac80a8c406d9", + "prompt_tokens": 82, + "task_id": 235, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "919ddf651a96238dec7924969f9d25d1ca34f678d0b4a7b17dfc643a9f51b612", + "code_tokens": 107, + "prompt_token_ids_sha256": "704c4a6bfecf81453265645c5a616eaf2355712476c514b5fcb526844000dc85", + "prompt_tokens": 92, + "task_id": 236, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "70bf0fc5a8c4f0d80de09a2b20b3b08c1d09ed2cd0703833d694b8b2b70dcd01", + "code_tokens": 40, + "prompt_token_ids_sha256": "c241057a9df2d59a5f69779f90bd1fee25c2441e343d9fcc54af34ca366d36a8", + "prompt_tokens": 264, + "task_id": 237, + "total_tokens": 304 + }, + { + "code_token_ids_sha256": "e0492e08873486c15f4e3066b114632317cc53c5b468ef2f2ab3c9a4a226aff0", + "code_tokens": 32, + "prompt_token_ids_sha256": "ff41fa7faa9f882631ae534c66c9bcf56ccfae1478dcb5b760b44dae839baa93", + "prompt_tokens": 82, + "task_id": 238, + "total_tokens": 114 + }, + { + "code_token_ids_sha256": "c6b3742bd8af5c6c71c801afaca3db9f92499e2ef09a9511b1b75ad08beb30de", + "code_tokens": 130, + "prompt_token_ids_sha256": "fcfc99f4ac0d6157901c3774fe7a325bd13fac5d727993ef907529a13bce4d1e", + "prompt_tokens": 116, + "task_id": 239, + "total_tokens": 246 + }, + { + "code_token_ids_sha256": "7af0a4ba16719b2c864581c31847c9604d24544087a50d7e1e7b0c2e22fe2828", + "code_tokens": 27, + "prompt_token_ids_sha256": "b19ca4896dd3a514c7f727ff412231acb29b95eb166de3d46a0f3da78a834c95", + "prompt_tokens": 163, + "task_id": 240, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "fc047b378c71f5c99427c52283791d0cb32ec001b132a9d4007994138b0a7be5", + "code_tokens": 42, + "prompt_token_ids_sha256": "76db572975115261a3a9f2f06c39f6afb71874632768cd83c27c9f3ab9bb8362", + "prompt_tokens": 284, + "task_id": 241, + "total_tokens": 326 + }, + { + "code_token_ids_sha256": "b00483be45fda7685c229083002af43a510fda97db098481e26a1d05f03c102a", + "code_tokens": 30, + "prompt_token_ids_sha256": "1d570c8d5e1f38ca2a29cf35d7e89bed6137ec83758e3f799765570f48f026a9", + "prompt_tokens": 66, + "task_id": 242, + "total_tokens": 96 + }, + { + "code_token_ids_sha256": "6609e9e0397f45000d3ddd967f5e3d9649f6f39c32e61a67649c0147a965cb60", + "code_tokens": 58, + "prompt_token_ids_sha256": "d43164ecaeb94871845f204bfcf960f73c8d78f55258c7058e011d58fcea907c", + "prompt_tokens": 198, + "task_id": 243, + "total_tokens": 256 + }, + { + "code_token_ids_sha256": "0f94ade8be0cbf360e4ef6a0134348c684efb856929e9c0be8908d4d6e3a63de", + "code_tokens": 34, + "prompt_token_ids_sha256": "470663206e8838228cdbcda6e1c7d2c6f24fcb6c351d8df4f6bfca568a81cc7f", + "prompt_tokens": 84, + "task_id": 244, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "e7f52dac737ecc657e2b3412fdfbc25cf85cac0a1e8b261f60a0d0e69d1324db", + "code_tokens": 208, + "prompt_token_ids_sha256": "defc82e72463ea2128edb4c26f8140eca796bb92e31e63237943d57f72cbe5ad", + "prompt_tokens": 170, + "task_id": 245, + "total_tokens": 378 + }, + { + "code_token_ids_sha256": "f92e626be53b19e58897729f0d5f6391ed5d7ed8764e922744c50d804fbe6ae6", + "code_tokens": 83, + "prompt_token_ids_sha256": "d2d77241adab8f2fe90e29472cb0bbad0370ef5c3a751facfe6279df200270e7", + "prompt_tokens": 114, + "task_id": 246, + "total_tokens": 197 + }, + { + "code_token_ids_sha256": "b2b98146b214dbf56e5dbbd0d0b0b69445068eeedb7ce636406cad869ca63494", + "code_tokens": 170, + "prompt_token_ids_sha256": "67cd6617d7aabdfeb1cd90c808346b314a49ad844a3065ae38b3812b38e1aa12", + "prompt_tokens": 89, + "task_id": 247, + "total_tokens": 259 + }, + { + "code_token_ids_sha256": "b0efb73be82ea96bc295fb098efaaa64f4604104d2eb2fa6466049b697d25f1b", + "code_tokens": 40, + "prompt_token_ids_sha256": "d82e13b99a99dbaeff6fc4f3671df3649a2c5e27f71cac10d710ea23f955852d", + "prompt_tokens": 119, + "task_id": 248, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "6f3d8c15aba88e12b5a695298a46adb02cfe5fa52a38744c086d94533732cb7a", + "code_tokens": 31, + "prompt_token_ids_sha256": "cb019137b599c0764c85ac4a45db9b81f0bfc053d490d32759ed3b396995eece", + "prompt_tokens": 186, + "task_id": 249, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "4a19fef1a12b3295a7eaa4b140c5297601d771b52002326e995d312ec4194ef2", + "code_tokens": 42, + "prompt_token_ids_sha256": "b364a3b0893356eb88ae8c53d6172b531654db8b9544797d0eeebc288f457fe0", + "prompt_tokens": 192, + "task_id": 250, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "41294bfe28efa11c4d3f4c32a6d0c0ea9a535e863a24248649134f8d9e548866", + "code_tokens": 27, + "prompt_token_ids_sha256": "bf6a8f33258bb94b7d1bba11f4c5d402084436a0ada5f3ea3eb283e2f86c834e", + "prompt_tokens": 127, + "task_id": 251, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "a1b0ef320f8c2c9e345fc6653ea52923d116ac1fffcb6b9d8e9273e056319599", + "code_tokens": 25, + "prompt_token_ids_sha256": "c894d9fa0c4c7d5ed922c52cb280125a3fb6791316b47371d947a9c5d178f2c4", + "prompt_tokens": 87, + "task_id": 252, + "total_tokens": 112 + }, + { + "code_token_ids_sha256": "e03acec21469f3c26439a2f82a1790468c9847336c554eddc04dea71d92e28ce", + "code_tokens": 40, + "prompt_token_ids_sha256": "7ccb43e8bacc6cd4c3a7e290e4685f873511b1cb03ad7013244db39404d93dec", + "prompt_tokens": 90, + "task_id": 253, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "5d54cb7aabfff45142e1c29acade9d71645f668cac51c2bcacf9da45a861abe7", + "code_tokens": 24, + "prompt_token_ids_sha256": "c17505b4a47c275546d0b1b18f4dd67b332c13b2bfdcd1cf412e7e75531ce426", + "prompt_tokens": 91, + "task_id": 254, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "e21afff0456159d4262195bece5c9e4af1858a95e25a48d9cbd8578fd7732480", + "code_tokens": 28, + "prompt_token_ids_sha256": "f8110b71f6d5c29a0f443e12c2fa34a7fcc64e61db9d85282279eb69f5299eb0", + "prompt_tokens": 230, + "task_id": 255, + "total_tokens": 258 + }, + { + "code_token_ids_sha256": "f973d8a03763420c5c08b409394ac80a1af0318b6b86f76d52e45ce5c5b6d9d6", + "code_tokens": 71, + "prompt_token_ids_sha256": "f31cde895412eae78889b8044eaaba99e3b0e8daefdc98f6845e82a5042d2756", + "prompt_tokens": 85, + "task_id": 256, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "2725f10ead9d860ca79dc40983c106aa18fed0e7b27919fbbe35acea7e3bca2c", + "code_tokens": 24, + "prompt_token_ids_sha256": "07e69d33d9aec68aa92a26ea0b962087382af56ebb79993ebab794c80288e778", + "prompt_tokens": 92, + "task_id": 257, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "3effc8deb7a150990a7b662cc91b00d53d60e2edf5c725ed1f1f245dbf0ade6e", + "code_tokens": 34, + "prompt_token_ids_sha256": "070b9f90e81c7309639f4ed8511197af41f20a2b3038b0b48718f5a109dc9993", + "prompt_tokens": 117, + "task_id": 258, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "da71ba9c5a6967c53868f4c9d6ef04b75b0a616fb5f2be5de3b75a8e49c50965", + "code_tokens": 64, + "prompt_token_ids_sha256": "50850da5980ea377bbb9c2c21a52941f8bc5eb01a7fa0c40c722fd2ae8d622f1", + "prompt_tokens": 281, + "task_id": 259, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "96630a5b55281558319792591d52b56d94541991787db888df2c994e46952239", + "code_tokens": 45, + "prompt_token_ids_sha256": "0b43ed39bd0a3c3b5d851459c9ad6190b04c50e727d7ad846e553f8fd99675bd", + "prompt_tokens": 79, + "task_id": 260, + "total_tokens": 124 + }, + { + "code_token_ids_sha256": "d40216bd02e20f150a228e735808481c5a17b5b3d731c0aa0553ac5426ef6bef", + "code_tokens": 48, + "prompt_token_ids_sha256": "1616d93398e765b85316ef9472886a1bac55a28379134a4f1855467f0350e7c4", + "prompt_tokens": 166, + "task_id": 261, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "2f7d8943c08ced9925e6b2b1c8d1223c8f0a6be0e8f075d61f4219bfed7dc3cd", + "code_tokens": 21, + "prompt_token_ids_sha256": "bdbdb49bbd46bce804ca213c65e444eeb1b28e44be7d5a8894af93f8a2d77c0f", + "prompt_tokens": 172, + "task_id": 262, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "84bd93ada6cb58b27eef46d3091526f68b3a53ba43f937558e873f02510bda72", + "code_tokens": 24, + "prompt_token_ids_sha256": "47a204156ce7270042607d0dade003b3b663b13f44c975cf5f98746d8007f2e7", + "prompt_tokens": 231, + "task_id": 263, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "d954c386cf791272f5bbf03524ca839e2c1698f357a25009fb404894b82f5e6c", + "code_tokens": 63, + "prompt_token_ids_sha256": "8e349bd15a44a1fbf6290180021c785c108b13c677b7b7b02626956f2a6a70ba", + "prompt_tokens": 71, + "task_id": 264, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "9cc89430cd08d19568cb051134953d518177f7247cdfd411abf70a58c6782728", + "code_tokens": 22, + "prompt_token_ids_sha256": "71abf479c06e19f0ed55e611f6fd7d9bc98acc48c3219c498366479911a1b6fb", + "prompt_tokens": 245, + "task_id": 265, + "total_tokens": 267 + }, + { + "code_token_ids_sha256": "b1816656c377bc4dd231013507a739fb29d562e8d63e389754e5e455bf17dfc0", + "code_tokens": 25, + "prompt_token_ids_sha256": "5097c110bed3b41d206bff1d4c4494e7e9dfc7adff0d00f369ea42f32cf6de6d", + "prompt_tokens": 76, + "task_id": 266, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "38953e7e0535306f280b002dcf250cb4ebf49634a3fce514623740236e207a3f", + "code_tokens": 21, + "prompt_token_ids_sha256": "318c479d5082d705c1563a01084f9f6cad1c56bf2b11db659378c7b0d4d87eeb", + "prompt_tokens": 79, + "task_id": 267, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "9eabc3b55391abdfaae730c6c17e4302cfdef3f8bfa8ef4127408842ad727043", + "code_tokens": 24, + "prompt_token_ids_sha256": "1bfe78bcd9bfaac89dfcfcec8c7fb5fc87cda1974a249b8b32b451f53183e6b6", + "prompt_tokens": 75, + "task_id": 268, + "total_tokens": 99 + }, + { + "code_token_ids_sha256": "fe95755c6cf002bb60952eba720d0d3fdb70c62efaa8fd4da0f8b89f423eeadb", + "code_tokens": 15, + "prompt_token_ids_sha256": "3c9254f418fc2802aa83a5f44e98d2d0742732ab6396b32ed0946bba62af1074", + "prompt_tokens": 65, + "task_id": 269, + "total_tokens": 80 + }, + { + "code_token_ids_sha256": "91e51e3c14ae4e993ae9e80c9212a7ccb2c3d2497011a1bb2056a92bcf5614d1", + "code_tokens": 59, + "prompt_token_ids_sha256": "0b679a3e505920dd147cc97430389671330273d140185e29d11c3f240fe44a36", + "prompt_tokens": 150, + "task_id": 270, + "total_tokens": 209 + }, + { + "code_token_ids_sha256": "b09036b5ab2fc96b5034376f248313484c321b48a06c4efa7d24537aeecb6a94", + "code_tokens": 54, + "prompt_token_ids_sha256": "4329b4a7e61193f2d9c342decf6ace1f15099fa0002be93c6a91c35536669740", + "prompt_tokens": 90, + "task_id": 271, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "2aed1363928fcfcca52a42ff95d22452b4fe0d6c9d2bdc7d3712cdf17d534dc4", + "code_tokens": 28, + "prompt_token_ids_sha256": "513675d6ccea89515e3dc8d8d092abbe7600eb19baf544adf500fbe824b3ddd2", + "prompt_tokens": 196, + "task_id": 272, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "6caf76ae44bca4b49763423e9e780dc5b565a65d8a3918b74b066e4c52b0f7a8", + "code_tokens": 46, + "prompt_token_ids_sha256": "b518c9b93bf1b2036412fb49d02855558f4a4718ae6290689b63686fc21edc82", + "prompt_tokens": 156, + "task_id": 273, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "47235192c831a9f8a91bb3b7e7f8cc352d64e378047c0b9784e67b7d8855f7bf", + "code_tokens": 27, + "prompt_token_ids_sha256": "5d4a69e25d8afab56faa8d83fa3eff421369920e5c4992d420c134051759757a", + "prompt_tokens": 86, + "task_id": 274, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "e8fccc08833967537afd1cec5ef1fdd45433e4af21caf9b3b2712b1c4870c03a", + "code_tokens": 97, + "prompt_token_ids_sha256": "2a80fa2b7ef1b12b2d80fda36bdcf63e67f5371e9a07a07c7eafed9c175e2019", + "prompt_tokens": 98, + "task_id": 275, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "ad29d0034bf8bfec19f00f9679116b46a62c96ebe0db659d82a032c4ecbc96b3", + "code_tokens": 24, + "prompt_token_ids_sha256": "89b9a6078ddbe833058b6f08f4323d02f510df634ecee0df416664c6258b2c1f", + "prompt_tokens": 123, + "task_id": 276, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "c514e8c1e1e022fc9864b19490e1deeeedaccd4cc78c84c402fcf36666a43baa", + "code_tokens": 30, + "prompt_token_ids_sha256": "5bc3c4037eb48633f8d08d540e26f510979355c1604e1e6e210e78bf10fcb68d", + "prompt_tokens": 245, + "task_id": 277, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "7b3e2c319f7328bd79fd7ed78e92e2d63cc39b34cd278147b1a678956be958ca", + "code_tokens": 38, + "prompt_token_ids_sha256": "cba934f062512e8c749bc155ab7b91700379f8ee1a091b23f47d27c4b6315c2b", + "prompt_tokens": 129, + "task_id": 278, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "8f390c0ad7e639b9dc62e44e40611ed07b27b81770702abd702fc23647049531", + "code_tokens": 21, + "prompt_token_ids_sha256": "20a77f262b7bab1e4342522f7581d7182a7f67937c2a5be81fc6f775cac7188b", + "prompt_tokens": 79, + "task_id": 279, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "f1b252af9269d57e57d1f72914184c91e7a8e8475c5f5c7723d545b1b9cb0c60", + "code_tokens": 65, + "prompt_token_ids_sha256": "b9654b8a48ad76824aeff8aaa31b8a84e1d792c108b524fa37e6c54bf0f9aa51", + "prompt_tokens": 178, + "task_id": 280, + "total_tokens": 243 + }, + { + "code_token_ids_sha256": "b6ff50c8503f5e49e0ab0d9c4febe7129689b9ee6387db743e3da2fa9e9d8e3d", + "code_tokens": 27, + "prompt_token_ids_sha256": "8468d1a6d0a79462dc2c37a002b240166b6d66dfe729b7aba14a56eea4a95248", + "prompt_tokens": 89, + "task_id": 281, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "588cb17cedf2755433004dbca17599d039d6e7f58c2173e038af268f78509683", + "code_tokens": 35, + "prompt_token_ids_sha256": "8abb5e76d813fed7e8c4394b380be2dfedf8bbfee8b7ee603baa892445ee6086", + "prompt_tokens": 110, + "task_id": 282, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "cfdf96907a4091d1f4a915809bf2882b632d3a72ff172e1a4f5da54294a11344", + "code_tokens": 75, + "prompt_token_ids_sha256": "f14a398c952be6b17a78984e101c9317fb0bfb02666c0ab5d121783ee0241591", + "prompt_tokens": 81, + "task_id": 283, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "2926f1c15fa71693cbbf7bd38bd4ba2e83eece458686470dba233aeebeaf57d7", + "code_tokens": 25, + "prompt_token_ids_sha256": "5c508c6de618883a83acdaf9a22435153676b0c310a4b322873f543e2f1febad", + "prompt_tokens": 100, + "task_id": 284, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "746a2dbfe3efa38185119e338352fe1faa9ed16003bfdbf3be91b2327486e6b5", + "code_tokens": 52, + "prompt_token_ids_sha256": "9ec060ed2c1fc11e6747c012bdf26386f4acc81f51248c895d8487ea33860d8a", + "prompt_tokens": 90, + "task_id": 285, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "a1f66ab85fb840ca70ddc44516828eddedf6a33115f10603f7372539607e73ee", + "code_tokens": 110, + "prompt_token_ids_sha256": "4d47e477fa56fbd6b6bfe2f971e3774dee99eafc3406b7dfda1189a2e8a617a7", + "prompt_tokens": 140, + "task_id": 286, + "total_tokens": 250 + }, + { + "code_token_ids_sha256": "06f0c99cd7dd703f7010d077408d81d7cd057a1ed6864108c96a9d3ebb7c94ef", + "code_tokens": 25, + "prompt_token_ids_sha256": "427903e46acca3e807d897f55d4ce4c3fa579b68b5c0ac23a11688ce05635d79", + "prompt_tokens": 80, + "task_id": 287, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "97abc37effd3462f77fc7a58397566b409cd4925e01b7cdc5a79e5ee812ddcb4", + "code_tokens": 56, + "prompt_token_ids_sha256": "d187d21ec8b25cc0aace1b38540057dec0f47cff29c6da713f62b8bd173c46e4", + "prompt_tokens": 127, + "task_id": 288, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "f94c23be8c3554d475069ab8f9be2b1e005f01d687a48c704ac62433c15b277c", + "code_tokens": 108, + "prompt_token_ids_sha256": "89af3ddff31e2d47969db57f24414a07deb950246557ce9d3126c919564b217d", + "prompt_tokens": 78, + "task_id": 289, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "b79cfeb9c49a151ee97aa2149703fed78cc0575e032bc1eca0b7140fe2d3d1fc", + "code_tokens": 47, + "prompt_token_ids_sha256": "9fd0997cef74ff3268e06292dcce6eacf7e99d4db0140df4bbc96b8fcc017329", + "prompt_tokens": 168, + "task_id": 290, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "6853b99b4dd6e9724213cbba9355d8ff619faa10880e2cb161ec1fdf09d8e51a", + "code_tokens": 102, + "prompt_token_ids_sha256": "c3cd9e976d4b1561bf833582c4ffd48296cabd0cf4dd237a4786ffa3db36befa", + "prompt_tokens": 114, + "task_id": 291, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "c080f5c882ac0dafd33b5e616fdbd906e79616be24efee9e8aae4a8cddf72624", + "code_tokens": 18, + "prompt_token_ids_sha256": "e2338e5cf8db36b32cd15cab268093b33b71027d9beb5ffe2eba3849ad53a999", + "prompt_tokens": 72, + "task_id": 292, + "total_tokens": 90 + }, + { + "code_token_ids_sha256": "b5b5be6ddea216f279c083ef842f9440e69f360cd9efbcba6e31222db4dd96f0", + "code_tokens": 28, + "prompt_token_ids_sha256": "86c27a9382b680f4b8e447ee4adebced0c10b2dabf1df63885165a04bea20202", + "prompt_tokens": 109, + "task_id": 293, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "fad0b2751211997e72184c62c1c822c7143c8953a8f2e062a0e49de446296a4c", + "code_tokens": 30, + "prompt_token_ids_sha256": "ecc4f8bf5bdd1ab4662692637fbe432b57d975506f07ebf217e07ca0dc0a3698", + "prompt_tokens": 114, + "task_id": 294, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "65eca9fb1b20e18d85062d37e1b35054673ffd0d7a7d42770d23b26aeae1f053", + "code_tokens": 48, + "prompt_token_ids_sha256": "64f48236577683da6309cc93f026f0d2ee92abf852b99bbebb22550611d8e848", + "prompt_tokens": 67, + "task_id": 295, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "8978a881872d24acb6fdd186430b3e206ca05b309e34753b4f7c7be3f85e2ac9", + "code_tokens": 59, + "prompt_token_ids_sha256": "ac1e4c70e4bc67a75b798550b41dcad0fe11a6990bd99aa6f039d06d61640d00", + "prompt_tokens": 101, + "task_id": 296, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "1026b4fb24043cc45c030ce6538d7390b65b820328dd62f00b35d4fb55b5a357", + "code_tokens": 102, + "prompt_token_ids_sha256": "77ac12db03bb4820e60544a2185c7fc8f8a861beffea0048a389d5836a679e5a", + "prompt_tokens": 310, + "task_id": 297, + "total_tokens": 412 + }, + { + "code_token_ids_sha256": "dfde2bbc6dd0e3d54048dac2a3eb6d63fb52d169f49fa6486d84513a85b80db6", + "code_tokens": 36, + "prompt_token_ids_sha256": "7319d50eee6865906e83e7c34c6c5e923d0502f4e9b8ffe86b2d636e8e480450", + "prompt_tokens": 264, + "task_id": 298, + "total_tokens": 300 + }, + { + "code_token_ids_sha256": "4a0f8b076df3656c1094d5a860bf5429371ebc1135dee12b5dd65b53acab4999", + "code_tokens": 51, + "prompt_token_ids_sha256": "3d28a716588cd5dd7eb0a23dbac2656439c96765030bbd49f6343e28ff082667", + "prompt_tokens": 210, + "task_id": 299, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "42187e9ba89f02ea4102218c0a1129f9fd982205b64e168fe97003d0c0a4673e", + "code_tokens": 62, + "prompt_token_ids_sha256": "f1ce69ac3e1182db673580a0437de2cccf6c0676769c704d8a84553fd9d18980", + "prompt_tokens": 98, + "task_id": 300, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "a9eb78bfcb93a2b81acec90b022773f83751e87af9d406f3614ab9f8f6867eb9", + "code_tokens": 39, + "prompt_token_ids_sha256": "e79860f59c0fc229bde5cd6d735d78e721b47f66178b7fe48d3dc6775eb79203", + "prompt_tokens": 103, + "task_id": 301, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "dcc15915869e48632b410e3481efe8eea98b769e99c0ba5c2af9c8e2161079e2", + "code_tokens": 76, + "prompt_token_ids_sha256": "7a2b4031414e743d46b883ed9c6014f1b0cc8674ba7e7e6f2e04eea51ba465f1", + "prompt_tokens": 83, + "task_id": 302, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "e27d043a13c36f65456c24b44d1575430be23646560d7c4a46614ca26e4ba030", + "code_tokens": 58, + "prompt_token_ids_sha256": "64cfdcc7cebf04420fed8451d23f94b3205ce5ecaccc072fb706869e791d18bb", + "prompt_tokens": 87, + "task_id": 303, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "eb20e601e18e47ea2f6e8345e527ca1834f294869abc8631865c10b3c2c96f3e", + "code_tokens": 93, + "prompt_token_ids_sha256": "ad69659912f31954e3140491b812b9be230e0ae612de417bcd666907334f535a", + "prompt_tokens": 135, + "task_id": 304, + "total_tokens": 228 + }, + { + "code_token_ids_sha256": "7f67bed1e023a23a0c496eddadf28d15a227103f438380d758737961428c2622", + "code_tokens": 45, + "prompt_token_ids_sha256": "81a0671a96af76f9ed286a90babe4067a4149f59905d2ca9c9e7a4b8521a7783", + "prompt_tokens": 109, + "task_id": 305, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "d6c68aa53bb8fd603fa3634acbc0058e53d0c116f2aa3848599b4f4bf0705a4f", + "code_tokens": 209, + "prompt_token_ids_sha256": "b3fbffd479e6f05feea4f344c65a51d1bd5fb2a6444c74d188e1dfd44609daab", + "prompt_tokens": 203, + "task_id": 306, + "total_tokens": 412 + }, + { + "code_token_ids_sha256": "5d3884471cdf672b49a9277eb767b583613ac938bc551800c62269219ba5d366", + "code_tokens": 46, + "prompt_token_ids_sha256": "e66e42622484ddae81143d1297770a69d50bc2d5a551a1c788778b9918474622", + "prompt_tokens": 146, + "task_id": 307, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "84d93d4b2d9c6400a32663c64ba64287f20ec777b9e2d6dc3fb866247c1ffbe6", + "code_tokens": 39, + "prompt_token_ids_sha256": "aa363b33e6ca8361ae3755fdbb469f829e78da6b25ae2b1064590e5cf23e90dd", + "prompt_tokens": 216, + "task_id": 308, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "aa5e67d8b6b32bb0cbf1c14c616b9ea524f190ab68618c46c64d0bef86e183f6", + "code_tokens": 25, + "prompt_token_ids_sha256": "687b6a2c12d4d4f45c839724d83b52e4b23a4e8b0f7ce062ca625063f55e0af1", + "prompt_tokens": 73, + "task_id": 309, + "total_tokens": 98 + }, + { + "code_token_ids_sha256": "25b30fbfead29dc756a712d9f2dcd0e4f5c2d1b5dcc32fbca57220cf9aa11a02", + "code_tokens": 28, + "prompt_token_ids_sha256": "0e420dbe94e95d4120b464cd452aae16a656331e769e5fdb289365665f305d9a", + "prompt_tokens": 124, + "task_id": 310, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "dfa4466dc980a8b41cdd9e05aca523499cff6fd38091bb1f6ca57cd40a15926e", + "code_tokens": 84, + "prompt_token_ids_sha256": "2d2f26e88e5b0550b5616de2a138d2bea32941081e19d7225a98baadb102e526", + "prompt_tokens": 83, + "task_id": 311, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "36754a97c72657d362c7da8971dfcdc9be0506253722aca0e632b06d9797a4ec", + "code_tokens": 34, + "prompt_token_ids_sha256": "a17b77da8d63b5e50d9837f5d1e332dd75447c7f021988d4f5788ffb3f6ceafe", + "prompt_tokens": 125, + "task_id": 312, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "666ee679d8acb5c4e083b3f1ac820f038dda37c3607609e6202cc97fe5874d57", + "code_tokens": 28, + "prompt_token_ids_sha256": "4c0ead4c7f8d8002e968db01da6a5d045eb72181091cc87a99156608b58a35f8", + "prompt_tokens": 89, + "task_id": 313, + "total_tokens": 117 + }, + { + "code_token_ids_sha256": "02c7888073976cd4491aa604a5407b13259e392e9681ca20bbdbf7df79c12878", + "code_tokens": 92, + "prompt_token_ids_sha256": "e8bdde83fbd0c6a9904c10947615efb962adcf512005271a060503ac759127be", + "prompt_tokens": 191, + "task_id": 314, + "total_tokens": 283 + }, + { + "code_token_ids_sha256": "83745feb4abbaac4f8b590bf009e44e0edc0f271206c567a174f1509f47b045a", + "code_tokens": 185, + "prompt_token_ids_sha256": "a89533c3f924be90d2d70ab9dea5de9a0264c2ed27cfc0f4462ec66dc67f127e", + "prompt_tokens": 88, + "task_id": 315, + "total_tokens": 273 + }, + { + "code_token_ids_sha256": "fc5b7a84f664f65a3b3fd433c56585ca32e29fe3a9479b640268ac9a70e593e8", + "code_tokens": 106, + "prompt_token_ids_sha256": "c7cfe87e58c3c90dda74d473355149fe8560f20c5de9ad90dadb40df69d27d52", + "prompt_tokens": 172, + "task_id": 316, + "total_tokens": 278 + }, + { + "code_token_ids_sha256": "fef63ed01fb8b56d2513f4fa4cbf7599ceaaaada850412ee9a9ffdf613c551db", + "code_tokens": 64, + "prompt_token_ids_sha256": "af97d343f8cf28d5c3b4c0a8f3156549408c958db01053de97489dea4e3c8bc4", + "prompt_tokens": 157, + "task_id": 317, + "total_tokens": 221 + }, + { + "code_token_ids_sha256": "2ad6d2b631d0acd52eb70ab455aee976c677f8fe5b3472c30ca1e0125f7f4561", + "code_tokens": 74, + "prompt_token_ids_sha256": "240974cb8469a3c11dcd617e1336a7ca9837ad1b69e608387c4e6bdaa07dc9a2", + "prompt_tokens": 76, + "task_id": 318, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "439efb016a802bab932ba127d1b9898cb25447273aacf9d7854d4a4a88fcc1c7", + "code_tokens": 27, + "prompt_token_ids_sha256": "796c6a121489ad5dd03b6eb067e9b7306af65b56fb5f7a8c6a667786a64435d2", + "prompt_tokens": 93, + "task_id": 319, + "total_tokens": 120 + }, + { + "code_token_ids_sha256": "542cfca1f68ce5c50b476187734ebb079b738bd17e25fcd67e6d867483ce4ac1", + "code_tokens": 72, + "prompt_token_ids_sha256": "9da7b83626d3bd03d1bdee055fc9bd85dc9ca10161aa73eaaac09ed2ac925c1d", + "prompt_tokens": 92, + "task_id": 320, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "684298cf3b7f594f38aa104bb86af3f81c44a66a507ad3252b96c732b64d7a1a", + "code_tokens": 62, + "prompt_token_ids_sha256": "307828c8068fe7a163939bae5d68a7e78bb2214bb10cb6ed82d1d14774a1c8bd", + "prompt_tokens": 133, + "task_id": 321, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "adec23ca1ff43fbf9d538cbec0ac6269ca09df88b9ca4371ea91ce08d0633f8a", + "code_tokens": 42, + "prompt_token_ids_sha256": "cf3a242e67aa9da2b470094c5f07d48193a7557ff4345df8ad1a0a4efe8b4e6f", + "prompt_tokens": 159, + "task_id": 322, + "total_tokens": 201 + }, + { + "code_token_ids_sha256": "95def8f8f6465df6d016cf09313c75900ba1a4f9b961c91f1e8c37212fc22405", + "code_tokens": 238, + "prompt_token_ids_sha256": "aba65add39e289dca1702fc09538b901b369690d642dda5f71f277f77567e1fe", + "prompt_tokens": 218, + "task_id": 323, + "total_tokens": 456 + }, + { + "code_token_ids_sha256": "e5eaf8ebe19523eb9c832ce9de098a6c41bfcc3b97e2073bd1be41e8ddbe3f15", + "code_tokens": 68, + "prompt_token_ids_sha256": "13f2f78aa587c939079908afe723ba0e4f6349cb63d46ef2e766b062a1c21a52", + "prompt_tokens": 135, + "task_id": 324, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "237c4e3d2c492303a223b365ad9984c8de705c68b07627bfd627540e8fc22c72", + "code_tokens": 83, + "prompt_token_ids_sha256": "b240fe55c97be864428bb39b276e534433c935efc8433b2ff5117c20f4ed53e3", + "prompt_tokens": 82, + "task_id": 325, + "total_tokens": 165 + }, + { + "code_token_ids_sha256": "1b3b66b3526a6ed9a5fdf1bd928f6fcab8122a463bd5d8842c4a8efede1afb2e", + "code_tokens": 64, + "prompt_token_ids_sha256": "9837018d7884b3624653c4be99fbfdaf42aa7f59db76be8c7fbef0a3a97ed71d", + "prompt_tokens": 136, + "task_id": 326, + "total_tokens": 200 + }, + { + "code_token_ids_sha256": "a6ee182fb5aec5b345a0f84203aa19242379da06374ffb380207846114cfd59e", + "code_tokens": 37, + "prompt_token_ids_sha256": "de96a97e59d9e3dc514281adf58e3e9896027a359121670cdb7596696c408760", + "prompt_tokens": 92, + "task_id": 327, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "d36287e90864aede6387e932fb00af6f38669203a7f6e52b7c2a81e0683c62a0", + "code_tokens": 27, + "prompt_token_ids_sha256": "0d4b41844c9b984e75895dc82cc92efc95fa9aa0362cd695705ef3a4233c831c", + "prompt_tokens": 248, + "task_id": 328, + "total_tokens": 275 + }, + { + "code_token_ids_sha256": "4ab5054fda608eb376d8819c9964efef36226f0899819098cbd9387ed23442db", + "code_tokens": 40, + "prompt_token_ids_sha256": "8227558c488e7aed1a7519dba055feaa226d07a59aac318c6f1ade0215229a6a", + "prompt_tokens": 90, + "task_id": 329, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "c41d02d348a7ad940e0ad03d79f15addff318c700acc0e2d7379de1c2377d239", + "code_tokens": 28, + "prompt_token_ids_sha256": "12005e71469be646cbe6dcfe348b673610912d18a81d274b2ee26c52e21edc3c", + "prompt_tokens": 126, + "task_id": 330, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "a4624bb218646fcfd3c303c260e1a719df503c265e6be59ea78b84ffa8d4c3e6", + "code_tokens": 59, + "prompt_token_ids_sha256": "34145f19a93eae3dc43521238cf0a6fcaf43cb62122c88c049e57f142b41bb51", + "prompt_tokens": 72, + "task_id": 331, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "ae0be1b3d5e9f49bc63c8c861d567ac4bdfe59d887ce253f82311cb5c71ed7d0", + "code_tokens": 57, + "prompt_token_ids_sha256": "d126aa076e0d9c4bfac8bfc79cce798361a907002995e5927f8334e1dbcda186", + "prompt_tokens": 167, + "task_id": 332, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "a9d36fb726b68cf942fd27a69edcd102c1df262876fc5522b64a6a6abee39d1e", + "code_tokens": 25, + "prompt_token_ids_sha256": "28738dac11e6712ee33f8d0f696e442d0da2c7f796606bfee4f57f5e841e8d10", + "prompt_tokens": 243, + "task_id": 333, + "total_tokens": 268 + }, + { + "code_token_ids_sha256": "1ec8296c47c8db2cfd1e43ad66ac5a05f70342fb26dd5921d31bf0ae4a2cdf1c", + "code_tokens": 48, + "prompt_token_ids_sha256": "8ef4b0816e51b242cc4bc18256b1f4a2712ef1c9299e5154cdf5b38928f2b108", + "prompt_tokens": 87, + "task_id": 334, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "a6fcf8367b3af5435373f2a8436dee690fd02d2ce5510769ddd01b5a04de953d", + "code_tokens": 35, + "prompt_token_ids_sha256": "e07d4705244bf0f22eb1fc7f52b714968e36de3e135e11e443e9c1c12175e787", + "prompt_tokens": 76, + "task_id": 335, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "3923556bc032c62f4aad3fff59d5c97c4b76de8c64f8f3d43c6dc64844255bce", + "code_tokens": 30, + "prompt_token_ids_sha256": "f5a0968516deb7420174666ede3d9de7656d3db61348dae38ab850ad9cbf636f", + "prompt_tokens": 71, + "task_id": 336, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "4db6017fb9028f1f2cb6580ecd9f34451e8ba63ca28e3ff59aec1b65b1547b72", + "code_tokens": 49, + "prompt_token_ids_sha256": "cb294e234e1cad4fff82c901e5cb73668045dccbf6485c9b05fbcfaebf6ffd8d", + "prompt_tokens": 90, + "task_id": 337, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "195731a0e50925a1a36e4a9b76797be22c7c65295b4cf49846cf817b239648ed", + "code_tokens": 98, + "prompt_token_ids_sha256": "62c0f739f67dfea77ae8673fda4493a25765d0944c8e369b338d0ca89a36cdd2", + "prompt_tokens": 96, + "task_id": 338, + "total_tokens": 194 + }, + { + "code_token_ids_sha256": "b99aa0daef6b2ba1da1a8d674baaf3326de8a2455d4c5b1bd732b375b5f79db3", + "code_tokens": 25, + "prompt_token_ids_sha256": "ea04fd88fce45e58b962a96abaabc74a679750e2c7e3fd63c2ad4ce5f292d333", + "prompt_tokens": 84, + "task_id": 339, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "fe51167b0ef159f2a786a606adabdd71931108b57c62ba177c89f49e413b91c4", + "code_tokens": 27, + "prompt_token_ids_sha256": "865df8c803ea8290d1178eb87dfe013283689d68f0bddb900ae48cfb3b36730b", + "prompt_tokens": 123, + "task_id": 340, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "b2e3951d0aab5cff48bc57a76e5a4a89bfe2d01435cdcc912fb01363b03d8a71", + "code_tokens": 20, + "prompt_token_ids_sha256": "ae2055c2aa4931b7f3461617699a7857c25cf751aa3988de37b00e50618e2eeb", + "prompt_tokens": 166, + "task_id": 341, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "772292d3062160610af811760cf6d077cf6adc0bbce644f454fd8ea9294cd423", + "code_tokens": 253, + "prompt_token_ids_sha256": "3f51bdcc447f45f9992ed54063f7d16571eca840e0f9e0e02b0cc3fc2096dd17", + "prompt_tokens": 229, + "task_id": 342, + "total_tokens": 482 + }, + { + "code_token_ids_sha256": "d403a7fce254bc4bf218da9a7dd6ab7de4ee664fd0717e51c9345a405b11d895", + "code_tokens": 55, + "prompt_token_ids_sha256": "59770ab0702278ee31c5404c9e827613c7917989dd9fcbbf255a8478f71fd08f", + "prompt_tokens": 83, + "task_id": 343, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "5e22cd3bf953dac4281c49d46b144de384dd88f3ff252160d72dbaa77fb56212", + "code_tokens": 31, + "prompt_token_ids_sha256": "6c7e985112b7077508c41e7d7990c6fe9b7aba12e427524f60b2b51e3d47d625", + "prompt_tokens": 91, + "task_id": 344, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "85458fabfa2d02df2b17e2d8f77c76fe25526c53405c4bd264da299911115bd3", + "code_tokens": 35, + "prompt_token_ids_sha256": "a17a839ef439ff4d6032c1a52811e3e378f68a224d76abc4eb26304f4048828c", + "prompt_tokens": 202, + "task_id": 345, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "8027fea79b9c972a4c64c4200de07e72ad24201ba579893b20a6d196c3b0fe54", + "code_tokens": 61, + "prompt_token_ids_sha256": "e5f48f605cde6d1973e16ab0510b7ce618a17f3da3ebc34e43db243d5b5507a7", + "prompt_tokens": 78, + "task_id": 346, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "4887ebb7c60d260cbb7169dc6623273eaa26675f15a1f0b97447c79263dcb8a5", + "code_tokens": 55, + "prompt_token_ids_sha256": "af79f03ea1b0517a37c0ed24f0e2c268eae1d459a3799aa0116da32d48ef7c1f", + "prompt_tokens": 80, + "task_id": 347, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "eaa4d51ac2b55e2fc4f725c701ad72ca98be7b0965b40ed8dec975251c21f856", + "code_tokens": 108, + "prompt_token_ids_sha256": "079c7fc451d5a33262031d5406184c50ecda6e589df58e913880e479a628867d", + "prompt_tokens": 82, + "task_id": 348, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "8a67942a8e59c2013873c38d12f0ce7490ac3809879c4da6179a57b862137e33", + "code_tokens": 58, + "prompt_token_ids_sha256": "80476620d9e6967c7bf4c32d054e3d2b7a9844309d7e6fca43e716725fe9263f", + "prompt_tokens": 86, + "task_id": 349, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "bcbf036c2860a234687649593b58524908e26787d0b54a1235b9aa7a53af8755", + "code_tokens": 91, + "prompt_token_ids_sha256": "2d6b9a4064109b1d82ee1f975abaada11fa260295f1c5350de6f51b77e8eacef", + "prompt_tokens": 78, + "task_id": 350, + "total_tokens": 169 + }, + { + "code_token_ids_sha256": "37b76b73d4f8cf80f82b1ab401ef3331cda9c0a96d9fd4127cc5256158317d53", + "code_tokens": 105, + "prompt_token_ids_sha256": "2fd11edb9c88ddbbda344e067f8d3a5ce0c4f1eb2fab06f4505924c755e8ceb0", + "prompt_tokens": 122, + "task_id": 351, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "409eb7b208a25c1c956db1560d293308fcdd0f7f3bac5ee1f58dda50b2496f16", + "code_tokens": 49, + "prompt_token_ids_sha256": "1beeda2eda56ca2b0f11c90b3f3521bdfd1af81896ea7c322c555ddffb6a2d58", + "prompt_tokens": 74, + "task_id": 352, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "849ec81b32b900723b7d22938d62553d8ac843192ef81095142128275b7c7ccf", + "code_tokens": 27, + "prompt_token_ids_sha256": "63f726f1aa84d708fba7d4f593d77319d6ccbc60a3597c0769cbd1db7c9a5dc5", + "prompt_tokens": 215, + "task_id": 353, + "total_tokens": 242 + }, + { + "code_token_ids_sha256": "b39b4a088bec15454ddcde6a881c098445c1c600882883d5ed22afa0c30e6800", + "code_tokens": 25, + "prompt_token_ids_sha256": "b049919a7467ea0f9a0796293f96f2251d443b09fdc515d2b68b7620a118dba5", + "prompt_tokens": 80, + "task_id": 354, + "total_tokens": 105 + }, + { + "code_token_ids_sha256": "d12ddda1c75aa6530c093585eaba56437093955ddf54e0b72999d52e4ffa9b07", + "code_tokens": 98, + "prompt_token_ids_sha256": "90c1994a8fc58e591520f7e6462964fd4da554921ecffa151d9a822e8b254b7e", + "prompt_tokens": 76, + "task_id": 355, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "4192f3aa0c0f653e4b6fa7e8dff04d4dfb404f4d1f5eab62d1d5953bf5fc8436", + "code_tokens": 23, + "prompt_token_ids_sha256": "1f3293f45e76caff204c93c27fea127b70f0c403b3fc13f3ce59d17eea916a49", + "prompt_tokens": 80, + "task_id": 356, + "total_tokens": 103 + }, + { + "code_token_ids_sha256": "e342a039a59ae2a284ffec73aec713be55ec70c34c48482f31dc96dee96b0656", + "code_tokens": 31, + "prompt_token_ids_sha256": "48381f7680b85f05fe1b227042859abed4c36a54ca39d540baf42b7da50ab466", + "prompt_tokens": 159, + "task_id": 357, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "5f6a2becf03b29ee57a198a96695999222a125bd585670597d1fb0a1c8583100", + "code_tokens": 36, + "prompt_token_ids_sha256": "b52cb7b071fc2a3a10fb765bd98b5d0fc8b767516217e5b8cb7085e38c9ea538", + "prompt_tokens": 118, + "task_id": 358, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "7e1d068c421b0938ff74edc530fe03ae74a5726c51a559d562d6380487fa48e0", + "code_tokens": 38, + "prompt_token_ids_sha256": "e19f4f725e6d555f05f3baceff228b52102d2deb16ef88da734f8bb7b73e8f21", + "prompt_tokens": 95, + "task_id": 359, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "b590b4657d8e19f45898d0df03446f7a535458c4c3f7bc8bced6f99a63e1238e", + "code_tokens": 25, + "prompt_token_ids_sha256": "c939a734a51c162844442ecc0aa66f2d2bf9970a3522f8a3a741ac3d3c1dfcf3", + "prompt_tokens": 76, + "task_id": 360, + "total_tokens": 101 + }, + { + "code_token_ids_sha256": "c0f711fc0300901f3905b670a0926a951fb556f8aeb3d1164f8f3aa07cabb44f", + "code_tokens": 26, + "prompt_token_ids_sha256": "99cb9e7018eb72f99a83830ca8ccf076c051407e7ba5d117fef800253b0b4d2b", + "prompt_tokens": 151, + "task_id": 361, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "d60a68d3e668c5415d196f68641731edb152e635946797c7c44d6e4e4d3e8a78", + "code_tokens": 62, + "prompt_token_ids_sha256": "be51826c30c10296f46c23a70dffa137ec467bce1453f78450e9526bddd63cd0", + "prompt_tokens": 123, + "task_id": 362, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "b4f8a0abf190c1a2f8d35cd17bb49f1f870c6f444836e41cd20219920992e890", + "code_tokens": 36, + "prompt_token_ids_sha256": "cf90d2c76d0595dc3512f208d193803c6f301b975bd8827a9770bd2573e41088", + "prompt_tokens": 258, + "task_id": 363, + "total_tokens": 294 + }, + { + "code_token_ids_sha256": "58453ae606bc9ce5c337c0f07e782e2673bfc65e700e6e5819863be933684dca", + "code_tokens": 119, + "prompt_token_ids_sha256": "b7e18e4afe94e2e4863b7edda20a8864b9591dd66e85c28c633987e180891bef", + "prompt_tokens": 112, + "task_id": 364, + "total_tokens": 231 + }, + { + "code_token_ids_sha256": "67cb1fd5920f188c5fd01c06d589a49f68320fce89292112ab5f4261e90ed4e4", + "code_tokens": 37, + "prompt_token_ids_sha256": "4fd6ac3c056477c6ebe1bee821cefc104fd533fc9dc95c45a089c64dc7d09195", + "prompt_tokens": 91, + "task_id": 365, + "total_tokens": 128 + }, + { + "code_token_ids_sha256": "d70289a1a5ae1baf8c20695645d756c4e868adbf5bf518e2135b2bb9c79ae1c5", + "code_tokens": 28, + "prompt_token_ids_sha256": "d9b3ab6c8a69f328e3294941481ef3ba4756113d84aa5f17d98be7c376918673", + "prompt_tokens": 103, + "task_id": 366, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "6e1aa8245e6b8e95e80b3ac9a18aa6594bafc4c40ac35774be512f67341d7fab", + "code_tokens": 143, + "prompt_token_ids_sha256": "ebb2431803c47b0cffe98853bc4dd1f8c312ecb7d0f71261f2249c00e23978c2", + "prompt_tokens": 72, + "task_id": 367, + "total_tokens": 215 + }, + { + "code_token_ids_sha256": "d8d158424b151053bade117e973184b45387c6570ce2af072154b841d06812f1", + "code_tokens": 29, + "prompt_token_ids_sha256": "0e2b9053ce043909a02fbe28402c8c00e48feab4a61bb5df7ceb14359307a707", + "prompt_tokens": 151, + "task_id": 368, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "86183483115ad988cdf65549bd8728b25b143dd9723a8a4dcbfa66a4356db4d9", + "code_tokens": 28, + "prompt_token_ids_sha256": "3b87672bf846b184319a4a1a28ae46c10a83bf406a23f3c6364493fd3094105b", + "prompt_tokens": 97, + "task_id": 369, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "b3aff4b5af85e39dabcd3935e56590e32081c5209de5fb521c8fc7702f7ba37d", + "code_tokens": 30, + "prompt_token_ids_sha256": "f51d8a9ef6418e4f4880405555b1b9ab07339c9ad40d5637959b239406f70117", + "prompt_tokens": 214, + "task_id": 370, + "total_tokens": 244 + }, + { + "code_token_ids_sha256": "e8a9b44ce3169a31cee0b4b08fd9bd0dc8ed44b82ff1c9ffc4296b7dbcbcd158", + "code_tokens": 85, + "prompt_token_ids_sha256": "69ace21e983ccf1a0e111de8b9bcea8e236bc3dc3710a8d7e125aeffea1a464e", + "prompt_tokens": 149, + "task_id": 371, + "total_tokens": 234 + }, + { + "code_token_ids_sha256": "f0f8aa3a89f09fde6c7185ff59ad9d14ddf49a085a005ce33d4291b1a57b5eb0", + "code_tokens": 44, + "prompt_token_ids_sha256": "9a51429a91d3ffe7eb130d07416b7f1e22cc54c51ce0ce02d05fe9a3178697a8", + "prompt_tokens": 263, + "task_id": 372, + "total_tokens": 307 + }, + { + "code_token_ids_sha256": "d8ba53b35d25e3abf10d9452572786da58fa84a8bab2ea045745b58132a51274", + "code_tokens": 19, + "prompt_token_ids_sha256": "fe5559f7baa780dc21a2e19aa414d92698620a9d0a8d693bc660d57706308f5d", + "prompt_tokens": 88, + "task_id": 373, + "total_tokens": 107 + }, + { + "code_token_ids_sha256": "a3c99473b9f60d43d446a4febf4eb4d82b4bbdabf3a1defa7f95ee10993e5848", + "code_tokens": 113, + "prompt_token_ids_sha256": "97399775583cefbc3b171a45e0b4d44f3d9e041938b41076e1b39900a3c445cc", + "prompt_tokens": 184, + "task_id": 374, + "total_tokens": 297 + }, + { + "code_token_ids_sha256": "c5979aa0e544c11587849e6860a5071220cd317ee68eadad182ad7d11062a203", + "code_tokens": 40, + "prompt_token_ids_sha256": "2311cd9efff67690345257d1ac0f357e38358b1329cf31979732569431e78a65", + "prompt_tokens": 90, + "task_id": 375, + "total_tokens": 130 + }, + { + "code_token_ids_sha256": "6c171ab7064edc0e495c79632fa20b37246a95fa28939d2c7c5ee0162943749b", + "code_tokens": 51, + "prompt_token_ids_sha256": "25a75f74cd5b1f58c73116cdcfbadb32d111b76ad34c41c7874c2257b9c599cd", + "prompt_tokens": 264, + "task_id": 376, + "total_tokens": 315 + }, + { + "code_token_ids_sha256": "0c8bb9a549ab959e5b223367a766849f8cec584189d97e3de8a2d71f8eb9b458", + "code_tokens": 55, + "prompt_token_ids_sha256": "f0cb1985683eaef935e8df72e13177ac42dc81b9306a3b93008d3a5ab0980f5f", + "prompt_tokens": 83, + "task_id": 377, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "1546f3c22ff924af68166aa14ea6c0e120c5ba6b998c90ecb2d965660c3e4ab3", + "code_tokens": 27, + "prompt_token_ids_sha256": "3d9d7abbb5cc7bc0b9fa824ef1017f79328fedf442a2fa45cdc6586efa6266a4", + "prompt_tokens": 111, + "task_id": 378, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "6a449490bed762cbf3e575678b7f132526921932c3835c191992552a2290d567", + "code_tokens": 32, + "prompt_token_ids_sha256": "a67ad7fa988f34eb1ad5d84ca74c89caf8471d4dbf7b86f30488e85f25870117", + "prompt_tokens": 93, + "task_id": 379, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "ee33965f22260b2d78b0b53aa3b2e86af08436af821fab4168d322ddc8671669", + "code_tokens": 64, + "prompt_token_ids_sha256": "ad125dbaf236a6f5f4504883b91febb934b75bba086a3f1e9229f60ee899af98", + "prompt_tokens": 773, + "task_id": 380, + "total_tokens": 837 + }, + { + "code_token_ids_sha256": "a997c10a20feb163bc8417fd5047cd2b9123e2de16fbc92dbb368c970e15e742", + "code_tokens": 35, + "prompt_token_ids_sha256": "79e69aaf19f270e2cda24d9ec88f62cac70468283e533706a8c4452e325c6f00", + "prompt_tokens": 392, + "task_id": 381, + "total_tokens": 427 + }, + { + "code_token_ids_sha256": "91636985bd68c0f7ff3fcae63344db5df6a0c4a8218ecfb02365c570258107f5", + "code_tokens": 152, + "prompt_token_ids_sha256": "29414f1d81ebce83e477b2f6d826d6734f46d5e15538d89b6a76a9b5f6071c94", + "prompt_tokens": 134, + "task_id": 382, + "total_tokens": 286 + }, + { + "code_token_ids_sha256": "2721bee9bec79f6ce5df6a0c28a15959cb6023f97037ab4e99acf58151ec62ba", + "code_tokens": 77, + "prompt_token_ids_sha256": "3419608e93fa37260e63461b4de5f69d01ca9ec2b0f9bd78eff0628b21bb9912", + "prompt_tokens": 81, + "task_id": 383, + "total_tokens": 158 + }, + { + "code_token_ids_sha256": "13a3efacb82cd889da45b7478eae5f2ca4d174feb32ba12dbbbf40fa13db6453", + "code_tokens": 77, + "prompt_token_ids_sha256": "636f473deb0d61a7ea6e702cb2cf1ca7a36a7481b512bf6e73998e1f5a6dddb8", + "prompt_tokens": 120, + "task_id": 384, + "total_tokens": 197 + }, + { + "code_token_ids_sha256": "d357f87b48fd3e4a9bd796135bbca7bf794beed3cb97ffd1ac05a4da2a9649ed", + "code_tokens": 71, + "prompt_token_ids_sha256": "a21a1ff6c6bc2c5b1a6053750d02510dcc33eee85460830613816a316e853a70", + "prompt_tokens": 79, + "task_id": 385, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "a277cbacbd56588ff0a772a5d61247049d06c03e2314763ddaba31e927147c3e", + "code_tokens": 112, + "prompt_token_ids_sha256": "24421ccff6c27a3db93b968d11d199ac3efdcb1f06a7dd312782a6085643d51d", + "prompt_tokens": 79, + "task_id": 386, + "total_tokens": 191 + }, + { + "code_token_ids_sha256": "f20c1e8b64f4a96033c322562c89c98a04e62ee78bb26ff83cccd59010ae21e1", + "code_tokens": 111, + "prompt_token_ids_sha256": "a5d33b4b9aa47ffe16b205501824b41cdc34e74fd77c7e7ae1fab5eea3657a97", + "prompt_tokens": 81, + "task_id": 387, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "96f1a64a367da5d6d51d3b4c7ed18b3e759ba31fcf1f2aba9c5c5ab802e7cddf", + "code_tokens": 62, + "prompt_token_ids_sha256": "2b9a667eebccabfb88964f1dbcbd76e438d4380afa0b48254f6eca5ff276fb98", + "prompt_tokens": 94, + "task_id": 388, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "823cf284762f5d8b9a1e0976ded228cfe3b1eafe1a61981b11bbe7f4e6d93e9e", + "code_tokens": 52, + "prompt_token_ids_sha256": "7821d7626442ca7f51556deed0172ffa3479d8c659ec47a3fb245e7664de9a63", + "prompt_tokens": 73, + "task_id": 389, + "total_tokens": 125 + }, + { + "code_token_ids_sha256": "5995708c32f5f2bda176b47f64964db2e528e3317678b31f66c7d5811884ca80", + "code_tokens": 24, + "prompt_token_ids_sha256": "63452ed2d85f3ce950d4dd1d578f1a9595c4cceda7269bdb927c8f25460a33ec", + "prompt_tokens": 149, + "task_id": 390, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "5137448c57393dac981dccc8482c0dcd97475447febe63efc964e0cdcea681e5", + "code_tokens": 48, + "prompt_token_ids_sha256": "83226adfb67c6af10f9bfcf212d2549146bbf243660fc5b46e80aa4db071de15", + "prompt_tokens": 327, + "task_id": 391, + "total_tokens": 375 + }, + { + "code_token_ids_sha256": "00c6aa7c052f039b3d792aa3d67bbd54157bf58a3056dfd6bc535fda7a28a411", + "code_tokens": 98, + "prompt_token_ids_sha256": "609f3591653f389b0e4a3159bf33026be69a8118ca922a309ce85ea73d23763e", + "prompt_tokens": 110, + "task_id": 392, + "total_tokens": 208 + }, + { + "code_token_ids_sha256": "3a086203b5f49dde2e307a0850668251b632f5ee43ba3ca43cfd089309b2c214", + "code_tokens": 48, + "prompt_token_ids_sha256": "fee9d597cbb952369acf4e3ff703c3b4059f8cb40adbdacb2a9effcdd2524f00", + "prompt_tokens": 183, + "task_id": 393, + "total_tokens": 231 + }, + { + "code_token_ids_sha256": "ae5341be933240227103fccf61541b810f70494a9b72e7e2853fe1d8e26676da", + "code_tokens": 56, + "prompt_token_ids_sha256": "cbc06f073b4acd29088affc0ea445d1d692a87e16079c5d0ee9f6aa28dbf1d21", + "prompt_tokens": 105, + "task_id": 394, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "820bebaaf445f7fc51ea9c48160c0ae8122d1b85fa71a53b0e6e0fef7496827f", + "code_tokens": 88, + "prompt_token_ids_sha256": "a214eb582b6f2b3f02bedd6067719bda6dc0cebb3bc0c640105e92d678576493", + "prompt_tokens": 85, + "task_id": 395, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "dcd9b9d6a4cf24751a06d5c43190643ee4164ea66ad3648005a2198ffbd2192e", + "code_tokens": 50, + "prompt_token_ids_sha256": "b4baf34f404c7f98a67c84f0f92fb609d9c29807070c28e8833b6b4e4cf5eac8", + "prompt_tokens": 81, + "task_id": 396, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "38e92ee145d22efbe257e76681d376b619075b507566676215b9d443e48ae292", + "code_tokens": 85, + "prompt_token_ids_sha256": "23e5b7f2b1aa86fdbb8b18a57217f5a9beea2e5a94feaf93e64cd7897e72cf7d", + "prompt_tokens": 92, + "task_id": 397, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "a3ec172a9ef7af4465d2e8c942525dc0fa7b7c2e62ede8dfc63134f023e39881", + "code_tokens": 27, + "prompt_token_ids_sha256": "8d907a47a974f5038e17b82e6b85f3db8349877824415921b177ad43b48bb1d1", + "prompt_tokens": 106, + "task_id": 398, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "a463443fcf09dc2143a8162fd74437a5c92143918d1e3496add86386350642b2", + "code_tokens": 48, + "prompt_token_ids_sha256": "d030d5a8e225c7fff03dc744c28e2aa6ed93cf518e852f3724b7c5ac5338bc9b", + "prompt_tokens": 176, + "task_id": 399, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "635b31da3e01768501a431afc001f54bdd4488868846f981e10b839adc66d720", + "code_tokens": 29, + "prompt_token_ids_sha256": "08f44c2770d7f54758b56b1eab97e114ed62638057a2b77e4fb641860b323029", + "prompt_tokens": 141, + "task_id": 400, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "3f6c6775c481d8ceb03c00fa1317c8d8b020ef6708650b6e3300f3cdadc3ae21", + "code_tokens": 63, + "prompt_token_ids_sha256": "893bfcba55be3b84eade07e0ad6ac451b85bdad49dca5f4c332fea373f78f9fb", + "prompt_tokens": 301, + "task_id": 401, + "total_tokens": 364 + }, + { + "code_token_ids_sha256": "0c92d11ef9139385899aec5ade13ea97574e3fe239fb4db3372f4697e7258fb9", + "code_tokens": 89, + "prompt_token_ids_sha256": "bc83780af6a581cec8b1124dceb306ca7ffd2bfa3d0fdd0cbf9edb2f8dc2e4cd", + "prompt_tokens": 89, + "task_id": 402, + "total_tokens": 178 + }, + { + "code_token_ids_sha256": "0e0aa0b1f63d65caedd6c5e45e1025d488f8968d7b7264ab57a29e046b1567dc", + "code_tokens": 133, + "prompt_token_ids_sha256": "789ca9a1d8d3ade903c927a2bcc5c8a8aadedadf85e33087c21bb62dc2e0df5c", + "prompt_tokens": 84, + "task_id": 403, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "72951670868af7649ae3f2a74bd6bf5c594da848c7956478efdd5f41ded22b5a", + "code_tokens": 25, + "prompt_token_ids_sha256": "fc731cc60cfd558e3308578e15c928671266927085d57759e57b5c5a4385442f", + "prompt_tokens": 71, + "task_id": 404, + "total_tokens": 96 + }, + { + "code_token_ids_sha256": "05e15a4af4e552a1cc8507eec62c9143b727ab3c53242f4f4a1c45dc80d8a028", + "code_tokens": 33, + "prompt_token_ids_sha256": "5f1d56b9d7f447fd6742bb25755a943436e8a101358e9f57eafd28b4a3890ac5", + "prompt_tokens": 157, + "task_id": 405, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "ecf25ed16a67c4005a6f2014d3eb6d8f5f0dafda6e64ace740932aff4e781f58", + "code_tokens": 93, + "prompt_token_ids_sha256": "56b38d99040b8a0f30157cda0d354fb083e48dfc3225d9368c04a2f6561c87c5", + "prompt_tokens": 83, + "task_id": 406, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "85bc3e34a51b0aa26c0fc5e0577f21da5472bf3f7ab3f023020fefad5d3996bb", + "code_tokens": 101, + "prompt_token_ids_sha256": "f75cfc8c46f45026f5ca6dee05ef113f71c45bb7f841cbd84a1177fd8ac1171b", + "prompt_tokens": 82, + "task_id": 407, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "9af9ab89cdab6fa859c6d6deb39febdce6aa843334df3d56e9f92e66e3365798", + "code_tokens": 152, + "prompt_token_ids_sha256": "fc780bd6d7eaa5f0ab628af70c710d9a0f5b30feadd23f6edadcd506aa4e16d9", + "prompt_tokens": 175, + "task_id": 408, + "total_tokens": 327 + }, + { + "code_token_ids_sha256": "27223c423c82104b640a3f08713271d463e1d8e2826e0bf9d641b500d510fad7", + "code_tokens": 33, + "prompt_token_ids_sha256": "f5272d1ec45c21307eb4d608f68fc65495d6c4544db0a4ae88e5bbaf35e1098d", + "prompt_tokens": 141, + "task_id": 409, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "37cac7ee1fd6a681350286e2950045a7ec1ca99ec97f2254bb3bece496f4b27f", + "code_tokens": 29, + "prompt_token_ids_sha256": "a465352d496ae0904c46a37e71f7c9bec6f28ece2c9d6d01a5ce75f2b8f5486b", + "prompt_tokens": 114, + "task_id": 410, + "total_tokens": 143 + }, + { + "code_token_ids_sha256": "80a58b664a6737b7817f9743c415bd91bb8a62e8df036b2f672dfd977acec891", + "code_tokens": 27, + "prompt_token_ids_sha256": "455c91598f7d878c2b5291f659161a23cca2d86340998cf8f5fc4648f158d8ff", + "prompt_tokens": 89, + "task_id": 411, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "0eb4c63fbb3fd81390ae58369522b14d83b2202acb527e6219fd6ddc9fbcc0f0", + "code_tokens": 33, + "prompt_token_ids_sha256": "4e6efa1802fc6ab70d8a1f383317f80cccf5a0c516a07ff7fa7580152d2fcdbc", + "prompt_tokens": 94, + "task_id": 412, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "c3148bf9916eb8149b150d55413031a4685a8c637a8635e9b188e25c9de2fedd", + "code_tokens": 27, + "prompt_token_ids_sha256": "e1171b65f3bb9b15d7ba7665a8118ec1e07e92b80e9126ece5e716483b26c674", + "prompt_tokens": 276, + "task_id": 413, + "total_tokens": 303 + }, + { + "code_token_ids_sha256": "103aec98276f6c978ed29d7ac5a7dc1f733cecd11ca0a03658ede4bb1cbdd1fa", + "code_tokens": 84, + "prompt_token_ids_sha256": "e116ab609c5ddfbe8383e34983b06601394ee35c1fb0498656f8eecf8c80c057", + "prompt_tokens": 102, + "task_id": 414, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "c090474d9d006654617b82a4f3f2d0c409af47cce8c9996dbe06a7f54eac7891", + "code_tokens": 103, + "prompt_token_ids_sha256": "95a5857065e5c72114488d0a80e6867485cc081e3600ef4a5eb7d9ed67d81167", + "prompt_tokens": 113, + "task_id": 415, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "c8a5531a6cda286e76eebf4cee3cb9104a5802f6a53d5037bf85927b137a8544", + "code_tokens": 88, + "prompt_token_ids_sha256": "16c19665451af699add770a22010b2259e7f179f504df6fc37a1e891550257e5", + "prompt_tokens": 91, + "task_id": 416, + "total_tokens": 179 + }, + { + "code_token_ids_sha256": "0f7946f1f38a764c289e93792a71b7b70358a420c051a8c83cf23160546105fc", + "code_tokens": 62, + "prompt_token_ids_sha256": "dee40171cf9139bd1710229202ae8e0e6d9cb1a812778f83fa67582f6a19d092", + "prompt_tokens": 157, + "task_id": 417, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "0c1ba8822c69c64a4e08ca0cf8ff94c782e459381487468147ac0665b041445c", + "code_tokens": 24, + "prompt_token_ids_sha256": "0ae465189c114c74caf81117ae489c217340154704faeffeea9895f29d0fc6be", + "prompt_tokens": 123, + "task_id": 418, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "416e3921b01eba108651c264684419d4ee2e7fea140f7783497f26ebac861d08", + "code_tokens": 34, + "prompt_token_ids_sha256": "b57956297e903971c3efc944e7ac1b3c0eda9a09eca658c16c0f01263680b0b9", + "prompt_tokens": 165, + "task_id": 419, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "ed563720c8e7e36ba0f0b8c6f44571ef7b54bfc8c7bd855f6274d1a6f246ed50", + "code_tokens": 43, + "prompt_token_ids_sha256": "d06729ed3777e8cc8854102b75d22322d0067d741d582eb38de837011f169d9d", + "prompt_tokens": 80, + "task_id": 420, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "55577b06e8abe081cdb62e3465f0033df6b85181af64a73c8388477762e64ff3", + "code_tokens": 54, + "prompt_token_ids_sha256": "7c95a031313bcce14b2d5ac3c00c39b06b3e6ab457a44122230b59f238ba7012", + "prompt_tokens": 123, + "task_id": 421, + "total_tokens": 177 + }, + { + "code_token_ids_sha256": "f881546e4f9b82b445897f95fc6e368850dc78ade8ba249919b0563cb3b5e934", + "code_tokens": 50, + "prompt_token_ids_sha256": "6a7b892a2ee90cf8db3e76fee22f6b7255962bed50ea6f9f7b43388e5a968bcf", + "prompt_tokens": 87, + "task_id": 422, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "cc47ced335e30e39614015b42bde0138ad0567944b602c91bdd4bdaefc5fc819", + "code_tokens": 235, + "prompt_token_ids_sha256": "46462c425fc3fc0a3aa41c799209b15a953060dabaf55d652ba8650dea81f9e2", + "prompt_tokens": 135, + "task_id": 423, + "total_tokens": 370 + }, + { + "code_token_ids_sha256": "a5757499fc458ebdda35dc3a8178a161da3064fd99046ef7184344df7df021f1", + "code_tokens": 33, + "prompt_token_ids_sha256": "349fa06dd4da7471429c894160eeb63bfbe459fceb43e09160208fb3bda7ee4d", + "prompt_tokens": 122, + "task_id": 424, + "total_tokens": 155 + }, + { + "code_token_ids_sha256": "6fbadc54f386994f6f6bcb8c03eca1475102eb54bb2c6aa7d4851340df12dfc9", + "code_tokens": 45, + "prompt_token_ids_sha256": "6d4e5a36fca0233a5bf86f7b7cb365768a217bb896ae3617fbed4e699274c57d", + "prompt_tokens": 161, + "task_id": 425, + "total_tokens": 206 + }, + { + "code_token_ids_sha256": "762cc4ad066786ee50182f00db9718b023bc27f8340d326cc067deb452704faf", + "code_tokens": 28, + "prompt_token_ids_sha256": "53595738e4caa23fc00a9569e246549e8aa81cd1766491a309722cf8de68d0ca", + "prompt_tokens": 147, + "task_id": 426, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "2734a53246d3612708451eea0a031cd7d0d58a4f3aff48debb95d79e36de44ad", + "code_tokens": 49, + "prompt_token_ids_sha256": "174d245920464c5f9a88078932de3f46fbf6efa2fbf731e43f458fae47016a38", + "prompt_tokens": 137, + "task_id": 427, + "total_tokens": 186 + }, + { + "code_token_ids_sha256": "823cf9efc60c2504e0cf59f91a641f0369451e244bbe9a99cc4ac9ff6c6e24f9", + "code_tokens": 107, + "prompt_token_ids_sha256": "7900258b1a84b3bf0352e4dfea4279a688d642efe542901bb6391d90679ef825", + "prompt_tokens": 238, + "task_id": 428, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "872449a2dddd16b72f8fe54e471f2fcf093ebba60f2bcbc366186d16d4b8bec5", + "code_tokens": 48, + "prompt_token_ids_sha256": "3b3aeff1aa8fe22143fee0c988193a998385dda70d9f0ac1749b9fac789c4201", + "prompt_tokens": 172, + "task_id": 429, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "2374f55f2eb70a148b00bfc01d0812897d6ade093bfc4e588ab1e9a880fe01ce", + "code_tokens": 40, + "prompt_token_ids_sha256": "84bc19434955ffa4b052e8fc3d3853fdd940a2c9e36d15944cb20168c30bf4da", + "prompt_tokens": 91, + "task_id": 430, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "2d376f8643273c69a0a18dfa7c2fba4ceaac947436aa6db6a8dccea35abfb42a", + "code_tokens": 46, + "prompt_token_ids_sha256": "97379b69a13f7cee3cc09aae32f0362488e73c6f02905da70c9c54d1e929e21b", + "prompt_tokens": 117, + "task_id": 431, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "cbb91b3f27690b9f25fed6abfff80de8b90b6df81bad299845d10a560292f97e", + "code_tokens": 31, + "prompt_token_ids_sha256": "b0990fa3dee7262c2f6652f98676bfcffb9befd57c605e1b723f3519f2cf7003", + "prompt_tokens": 95, + "task_id": 432, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "42cb4bbaefa5777bcf36279cb38c9084b83119ca54757dc61074e817388ab345", + "code_tokens": 58, + "prompt_token_ids_sha256": "5630d7a71a1b2cf37832356fbce22ba73a221c1cf6c0fc07e744553d27570cd1", + "prompt_tokens": 159, + "task_id": 433, + "total_tokens": 217 + }, + { + "code_token_ids_sha256": "203c06f15dec2a207f1d4490d8863473b6a32fb785ffed8e1fb779009cc0ca51", + "code_tokens": 49, + "prompt_token_ids_sha256": "dd64caf7867c97ab6618234ab47be74de336ff7aae7f0cc25dd772b543fb6f73", + "prompt_tokens": 86, + "task_id": 434, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "73086fe83e31e9e9be41c31fcae3dcf4d9886bbaac5b054a991eff49f9ac80da", + "code_tokens": 18, + "prompt_token_ids_sha256": "604487311020ee2c6ebcd5fbcc4d454fdf17e3bb6f51000285e92aa799a714c8", + "prompt_tokens": 77, + "task_id": 435, + "total_tokens": 95 + }, + { + "code_token_ids_sha256": "992678a188681c0ad9ab8003f6c2a9a19a24ccc59d0a618697c5e57f15f28539", + "code_tokens": 28, + "prompt_token_ids_sha256": "35976196b41caa49eb3c1fe6521edfde2abe52a9c30ebdf85c20680cc275e75a", + "prompt_tokens": 95, + "task_id": 436, + "total_tokens": 123 + }, + { + "code_token_ids_sha256": "8a77573db293c0aad800c2558c798ca7befa168b39ba57890f1a5d0218344d70", + "code_tokens": 57, + "prompt_token_ids_sha256": "77fc6b28aebef71eaadb14c44f158dea12d03218c4b02d499f34a3ebd01fc5c0", + "prompt_tokens": 70, + "task_id": 437, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "2c59ea890e8aa414ccce863e7db3686c8c8fb343e3e5b1e8f1f5588ed69a5a42", + "code_tokens": 88, + "prompt_token_ids_sha256": "529fa7b7047729a6b4764494666af1c9d5690e0a44469efca9d3ef2ebbdded4e", + "prompt_tokens": 174, + "task_id": 438, + "total_tokens": 262 + }, + { + "code_token_ids_sha256": "c9f9fb41598c3ec7d6c7243369c85fb4f1f20411689dd83f1faa5a6feb830efc", + "code_tokens": 23, + "prompt_token_ids_sha256": "f11cde80a6d617aa4e4e5f6e8bfaaac83078ab078ee8075ed2a1566db498e62f", + "prompt_tokens": 115, + "task_id": 439, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "475d9244d478d6237f4c50c99c82d1aa174dd1a7722dcdeee60f20ee76ce76ba", + "code_tokens": 40, + "prompt_token_ids_sha256": "0386bb74c084796f760c80ae97e4aef8cc75c5a6eec6edfd99e29bd385678698", + "prompt_tokens": 121, + "task_id": 440, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "6c230a53f8428e29fa7a30eeaaf13b4db4dfa9542084230724bb284f46fe6e7c", + "code_tokens": 20, + "prompt_token_ids_sha256": "4bf8dce0b63b2ee0b897e2f2d0ed99c3749f28901430fb36da10374be2272469", + "prompt_tokens": 71, + "task_id": 441, + "total_tokens": 91 + }, + { + "code_token_ids_sha256": "0163f0925202b022d1ccedef4e2ad86400018880f999b03ca69f9b50c6f07f99", + "code_tokens": 63, + "prompt_token_ids_sha256": "b9635048aefd24aa680c37540e7bedd8b93ebaa0198735ed5820b2a19438a583", + "prompt_tokens": 175, + "task_id": 442, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "f4e79808580ac06e875cc7fce515cc676df790d0bec666fac4477d6d062dbeb8", + "code_tokens": 39, + "prompt_token_ids_sha256": "bd98aa978a2cae16d50459ef7c9f4d473ef469ac51613323ed6e7ff8b90ef119", + "prompt_tokens": 95, + "task_id": 443, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "8b1a557de1267d8bc76f94ed68d2d2e3bdeeacc31df451c037f2f48217077fa4", + "code_tokens": 50, + "prompt_token_ids_sha256": "25633bffefede34fa34c4a08c21dea8a3265a2d7c715bffbb02c0f586eb4e9a4", + "prompt_tokens": 313, + "task_id": 444, + "total_tokens": 363 + }, + { + "code_token_ids_sha256": "788797b88a26e6f6b554cf7954bcafbb5744f6e776b3950414c7d3a92a8af897", + "code_tokens": 63, + "prompt_token_ids_sha256": "637458d930e1dd6b32ee7492cc0d4cf6d3c908ce478b127d8b61004566d5015c", + "prompt_tokens": 306, + "task_id": 445, + "total_tokens": 369 + }, + { + "code_token_ids_sha256": "0957a08c823bad2b6e811a7a9eb67fbe51c4e9b826e7dac39828afeefbc90b89", + "code_tokens": 46, + "prompt_token_ids_sha256": "0aaa62d8023f33a89ec73023cd6ce8b035444dcbd1ae51f9edad7014b0128457", + "prompt_tokens": 147, + "task_id": 446, + "total_tokens": 193 + }, + { + "code_token_ids_sha256": "857e14e30669cbb5b6f43075c51fdea3884fdb15f329ca52306612558e461643", + "code_tokens": 25, + "prompt_token_ids_sha256": "fd18aaa72f1042a17f7c89ca4da38a2210c3db2d6a519175e4589317f471d472", + "prompt_tokens": 178, + "task_id": 447, + "total_tokens": 203 + }, + { + "code_token_ids_sha256": "1ff6fb4edb0097a02e9783deb02bfbef21abb4ab2977ac6e5d6aa7851d647062", + "code_tokens": 111, + "prompt_token_ids_sha256": "e53a7b44807a1933ae23881384ed18992f3ddc40c87369d811fe047f8cf6facd", + "prompt_tokens": 74, + "task_id": 448, + "total_tokens": 185 + }, + { + "code_token_ids_sha256": "89be437c743d039505a98ee91fb20263da5721338887dea61afdf18cb97e6bfe", + "code_tokens": 71, + "prompt_token_ids_sha256": "4503b1f6f2b6b10aaadf3c8d34d645e3b9cbd64d31b030972266c42b3686e344", + "prompt_tokens": 112, + "task_id": 449, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "414fa5518358eaa1791aae45d1a2fd000d31d4b78b6edfca6bb1fbe0d84ccd04", + "code_tokens": 28, + "prompt_token_ids_sha256": "4e4f96ebb780ea4b4fa6c88daed1d4e10ae72248fd516ce84995e408b4884845", + "prompt_tokens": 125, + "task_id": 450, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "6e6d233b1f10c4b46e88fafad4ce5874246ab25f58fb60780fbb504f8c3f9c08", + "code_tokens": 25, + "prompt_token_ids_sha256": "71ae000b6c83590c335c1975e71759d65dd054f209ef0e598b8de4df165fe978", + "prompt_tokens": 92, + "task_id": 451, + "total_tokens": 117 + }, + { + "code_token_ids_sha256": "34bf668847489a9d1069389f36fa47eb8b0ec2b7f4ac0089a05664e31eae56de", + "code_tokens": 40, + "prompt_token_ids_sha256": "883b5dc650318f91b94a26af76d82519873b0393da18042ba7ab559805b7a14e", + "prompt_tokens": 94, + "task_id": 452, + "total_tokens": 134 + }, + { + "code_token_ids_sha256": "d8c9646e51abc74deaac841b8d38b050cecaa19375af7636e36007e9231767a4", + "code_tokens": 177, + "prompt_token_ids_sha256": "e8e15f3b452d0d1613ed04b9a9c6ab024eedabc7a9aadc760f758830e2e38704", + "prompt_tokens": 78, + "task_id": 453, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "95d843b6be0b136cf9f7acdd434c4e1c85173f81005011b7ca98b0f829f2fee4", + "code_tokens": 51, + "prompt_token_ids_sha256": "25efb54301bb5216fccb11c79fab3e2ce86c8e7bafec2e6ea815f0d135d3d15c", + "prompt_tokens": 87, + "task_id": 454, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "47cebe9b5b9156daa250e0a62f2460e7a23a8db4508ce6fd31fbff5a350e95ef", + "code_tokens": 69, + "prompt_token_ids_sha256": "4c646f17ea739fe1e63b62114e9bb7012f87235825e5b80e13c9cba2977d08cf", + "prompt_tokens": 77, + "task_id": 455, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "92289c7b0cd3f62545c3b7f22ee6616a6a091dfa11811dc1a6db66a8e58aaf92", + "code_tokens": 26, + "prompt_token_ids_sha256": "9724b389e3d1d2d634d3e0d6cc8a51da7ae6bd68bdea108d0f25719b1fc37b43", + "prompt_tokens": 140, + "task_id": 456, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "edab4f21625e8e1d5f509083bd9cdb02dd53e4cab419ab1b285fb561c9ce3031", + "code_tokens": 24, + "prompt_token_ids_sha256": "46564286d737c960d48362b61c7f29dbfc27a67d81ea176ab38d33621ca8c32d", + "prompt_tokens": 111, + "task_id": 457, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "824975efc00afe503739a15468bce0baae64302a9781e033c304ff5ae9feb99a", + "code_tokens": 15, + "prompt_token_ids_sha256": "09aa8f2c1f932b831e1bfa1c8b89b882cab53300e940c8d889353d5706bb3687", + "prompt_tokens": 73, + "task_id": 458, + "total_tokens": 88 + }, + { + "code_token_ids_sha256": "73fd5fcf063f2bba2e83ffd1e378b53595949578dd3145be5b2901b53cd0ca66", + "code_tokens": 43, + "prompt_token_ids_sha256": "e19e81dc399814df8c1ba8e712a346d191f80f96cdcf6ec0c370f20d2d79f281", + "prompt_tokens": 129, + "task_id": 459, + "total_tokens": 172 + }, + { + "code_token_ids_sha256": "38fa159347900dedc1599c25acf807e48f74165b377709d3b50f9cbb8781d0ec", + "code_tokens": 18, + "prompt_token_ids_sha256": "c958ca500b117601d3ad6ce2c03aa257a056229aad1ab2a2f707b198b724a4e5", + "prompt_tokens": 120, + "task_id": 460, + "total_tokens": 138 + }, + { + "code_token_ids_sha256": "b72e9bfc3a743f38515d2b7298346d87c9635caccd734d6c20c0eb25c48fed53", + "code_tokens": 49, + "prompt_token_ids_sha256": "803ae460ed2a08cc0ef97b62e7e812dca864495782d539f9480b93dec5cacd14", + "prompt_tokens": 73, + "task_id": 461, + "total_tokens": 122 + }, + { + "code_token_ids_sha256": "7be49b4cefe908a8520da5b464a8bea62498e7ae5dbadc0262c679fe61d8ad31", + "code_tokens": 59, + "prompt_token_ids_sha256": "4faa9ab17b29609052377a74bae62d607d9234acd9c0148de02b0551ba8c3d21", + "prompt_tokens": 873, + "task_id": 462, + "total_tokens": 932 + }, + { + "code_token_ids_sha256": "799876ae1c56cf410b868e35accf00202233c51b14fabbf853a421727b0d5115", + "code_tokens": 221, + "prompt_token_ids_sha256": "d29d77055068c1db171cc086e8821cbad5a7bdc9c100b6a0a4bbb51d03698484", + "prompt_tokens": 126, + "task_id": 463, + "total_tokens": 347 + }, + { + "code_token_ids_sha256": "8d2276f4ac0a54f9422d1ce67552b63cb612459adf9549a4e60e2745ba279202", + "code_tokens": 25, + "prompt_token_ids_sha256": "1ffe4c05437be989b0d0568cae7027aa37acb450204176bd542f45bfa7a335ca", + "prompt_tokens": 174, + "task_id": 464, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "4584abb6e69723060b114fda9d0b2197352b5257196fcd00c4095f9ca830b5f7", + "code_tokens": 36, + "prompt_token_ids_sha256": "6220a0d8257135b27191d36796a2d52d4908759575030c2e8d2b2da488c6523f", + "prompt_tokens": 140, + "task_id": 465, + "total_tokens": 176 + }, + { + "code_token_ids_sha256": "990cebba5654db5efdc3d6cbf2ab1f99f3f345a6d520d75e0def916e67a809ef", + "code_tokens": 158, + "prompt_token_ids_sha256": "8a752480a526df351c61f7a73a61c5796b27c6c4ea6543d81bdcfa20984a44f9", + "prompt_tokens": 125, + "task_id": 466, + "total_tokens": 283 + }, + { + "code_token_ids_sha256": "a16b8839c438a368ffeb9444e4357e41cbe70e978d0bd33532080bf575c8c95d", + "code_tokens": 92, + "prompt_token_ids_sha256": "800b42f5b9770f9fab69ff9bdc55d938c71539830113d30102b7a86a3ee692ad", + "prompt_tokens": 79, + "task_id": 467, + "total_tokens": 171 + }, + { + "code_token_ids_sha256": "adfd4dcf6190ed87d08d6f2523f49bb530ae6194b0d47bea8cade40a9baed761", + "code_tokens": 102, + "prompt_token_ids_sha256": "1c9c8b3b494e38033a7c59950df3721490dfacb66921cb39e679e7779377717f", + "prompt_tokens": 167, + "task_id": 468, + "total_tokens": 269 + }, + { + "code_token_ids_sha256": "21d7c5cc64c3aaf577d4fcc5c90c4ff25c1b2f5cc8b098d8689bd897e0cdc4b1", + "code_tokens": 171, + "prompt_token_ids_sha256": "29f8be02ca6f6e9d828c2f0a31851f23a169e0dc57ac11a5e9c6726cb2ffffa3", + "prompt_tokens": 134, + "task_id": 469, + "total_tokens": 305 + }, + { + "code_token_ids_sha256": "df52258f1eb4c2a24531d4908358282888360e708a7176f5496db30854057e40", + "code_tokens": 40, + "prompt_token_ids_sha256": "94bf6f3ac13cd6a4db30ff107bfe0769c6e25aba8339715062b574ad6617d047", + "prompt_tokens": 155, + "task_id": 470, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "4a6293101f7ffda2f1fc0c84585a18c347e35d3c0da93168adca4b657dd27773", + "code_tokens": 48, + "prompt_token_ids_sha256": "06db1e0c8c8003557a916b0c2936c754d15063ef718a77bd1d7ad8c4b1ac16f6", + "prompt_tokens": 115, + "task_id": 471, + "total_tokens": 163 + }, + { + "code_token_ids_sha256": "596b353305959444bd94475c0733ffd4c034599b23738073e66a8015c0c37dd8", + "code_tokens": 24, + "prompt_token_ids_sha256": "3bc93469404b2e69233b7d24d719f3fad8213503018ba8ee08947aef9b0597bd", + "prompt_tokens": 92, + "task_id": 472, + "total_tokens": 116 + }, + { + "code_token_ids_sha256": "51bd7c0375cf44db345777e2902d9292a1517fa24adcc0455147e356c654e65f", + "code_tokens": 48, + "prompt_token_ids_sha256": "a186c4d63207e1a827c173b8883d3dcb8b57658a555d706226207a86299910d6", + "prompt_tokens": 261, + "task_id": 473, + "total_tokens": 309 + }, + { + "code_token_ids_sha256": "1b9b4f55c9726a28215353bdd18277e740a988fecf66ccbd11b9b39185f69d85", + "code_tokens": 25, + "prompt_token_ids_sha256": "d769f5f91345494bdf0da25da73d62005c3a6b34a7199a593c31f296be29ca22", + "prompt_tokens": 81, + "task_id": 474, + "total_tokens": 106 + }, + { + "code_token_ids_sha256": "f69b9853ef0b4193c96571de882730ec5a0269623c113e7939c726537a25298a", + "code_tokens": 30, + "prompt_token_ids_sha256": "0f4a8990ad88e764f821040b7d8162c6999c0200abc2fbddb8b38cf350f3438b", + "prompt_tokens": 192, + "task_id": 475, + "total_tokens": 222 + }, + { + "code_token_ids_sha256": "c3e0f2bdfeabfd30cc0750011ead9573751c2f7465b1a79e763ba96b173d96c3", + "code_tokens": 19, + "prompt_token_ids_sha256": "e6228be40e60a0a5327ca9260bbabc3005e98542c2d33707914cc1a28fb136c3", + "prompt_tokens": 89, + "task_id": 476, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "70fe9cd235e043be67292905a0e830fa2be9eebf2eb2d526e8e49f673b28ca6c", + "code_tokens": 12, + "prompt_token_ids_sha256": "0052cd659431cf7bb94fbf81f8e7f606ca2589201c207a7d92243d95c99fde3e", + "prompt_tokens": 77, + "task_id": 477, + "total_tokens": 89 + }, + { + "code_token_ids_sha256": "4f274f8f398e92267a50ee9cece5b8edc0266a24c93b26590491961a68e67b12", + "code_tokens": 37, + "prompt_token_ids_sha256": "38f21a40da258e41a3f88ad325cd39f90abb9c3cfdf77359b10bf48d2d0ae907", + "prompt_tokens": 78, + "task_id": 478, + "total_tokens": 115 + }, + { + "code_token_ids_sha256": "7d1f8fb8a6b69d8487ad188517a2519dd18b54972bf7ae178b00122472b27fdc", + "code_tokens": 33, + "prompt_token_ids_sha256": "45fe823074f6ac983ffe28bf3435d0cd9ac1b8733c6598da43ddc5f47829d5de", + "prompt_tokens": 78, + "task_id": 479, + "total_tokens": 111 + }, + { + "code_token_ids_sha256": "64000266374211d8ea3f7ae5c714108713917f57e4786d67a54ff38e30ac8262", + "code_tokens": 94, + "prompt_token_ids_sha256": "2f6ff6b642a4ead3906882ef0d30ae6192a427e821d9d16d00cda635060715e5", + "prompt_tokens": 86, + "task_id": 480, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "fce84b3cb604d6f5c3ab35ac907eb18696be5179704b2d729c67b3fe234ce69a", + "code_tokens": 90, + "prompt_token_ids_sha256": "767ec1c109fd90852cc10ece0f23de3d3cb5536bba41c146fcb65901cab3fa68", + "prompt_tokens": 149, + "task_id": 481, + "total_tokens": 239 + }, + { + "code_token_ids_sha256": "7cab131088a1530bdf64ad42763b2edc941329874297ae93e91ff50de3749d66", + "code_tokens": 47, + "prompt_token_ids_sha256": "da3e0af41366aaa6d451475e0a1977366cbc2b12ef764da5a550b47512154653", + "prompt_tokens": 86, + "task_id": 482, + "total_tokens": 133 + }, + { + "code_token_ids_sha256": "c45579954174b51a53a047fd77285672bd3def31192bc602ba9a3b69929db01e", + "code_tokens": 61, + "prompt_token_ids_sha256": "863989ac3beed98b638186a735d29ed7a3fa6fb5b9ed8030cf91014f9e025689", + "prompt_tokens": 93, + "task_id": 483, + "total_tokens": 154 + }, + { + "code_token_ids_sha256": "1abab0e6128672771933873952d5c3f249af74348f3d615ec0f6a1d1a3d075c4", + "code_tokens": 39, + "prompt_token_ids_sha256": "2183358c70f6319ed8463ab717d8b542aac94e5a5dc9488b5da8a6de9e38ad33", + "prompt_tokens": 193, + "task_id": 484, + "total_tokens": 232 + }, + { + "code_token_ids_sha256": "bd04eb67f3a68081d023ed7ee5fcc5058171674765be71389853a9a8fce5ca40", + "code_tokens": 152, + "prompt_token_ids_sha256": "72a1181ffe60ceecc6dd38f4db031d6ec54ea3777c64ccd276780b85dd2cd3bc", + "prompt_tokens": 139, + "task_id": 485, + "total_tokens": 291 + }, + { + "code_token_ids_sha256": "55114f55cb7d2c6aabb527a32a63e6393757f9562b5334e5279bc59fbfcd7171", + "code_tokens": 100, + "prompt_token_ids_sha256": "f26210376f5c7fcf085c0afde5cac8eba2ef44b32a584014830787f0943d1bf9", + "prompt_tokens": 147, + "task_id": 486, + "total_tokens": 247 + }, + { + "code_token_ids_sha256": "82db5c50df7c9f327daf79557117aa4a22220115f28237543b33f156a1398a8b", + "code_tokens": 89, + "prompt_token_ids_sha256": "f59c61f303b8f48562dbeb75ec214a978110ce058001e5f843df1c3663b1ff48", + "prompt_tokens": 172, + "task_id": 487, + "total_tokens": 261 + }, + { + "code_token_ids_sha256": "1a5367ba7807b275e6543221bcbd574e6ab905dcfe91955755589d28fc591017", + "code_tokens": 40, + "prompt_token_ids_sha256": "14c48b11c25ffe32d2b9aef310e2eebf713324ce76055db26e3c862274680892", + "prompt_tokens": 119, + "task_id": 488, + "total_tokens": 159 + }, + { + "code_token_ids_sha256": "35d7facc91dbdf87feab39224f21ae2cc0d6ac86629582c897295e132da151bd", + "code_tokens": 76, + "prompt_token_ids_sha256": "0f20d0b8f2623f1f553a05b87954d1375ad2ee36a4d08a606e49c7c824067041", + "prompt_tokens": 103, + "task_id": 489, + "total_tokens": 179 + }, + { + "code_token_ids_sha256": "77f881c9580e5eb6cbc686a3eb57d3f4b665ebb893e7461ace6ffe17a9fdb603", + "code_tokens": 56, + "prompt_token_ids_sha256": "5feca945d0ab02f193d1bb2813c43b06bd9dce2720354940e1fea96d078d81b8", + "prompt_tokens": 220, + "task_id": 490, + "total_tokens": 276 + }, + { + "code_token_ids_sha256": "ab00cbf4671cf9568f64ea18a0fce2473cbf7a87412287f38b4d7d99dd9c2cd7", + "code_tokens": 34, + "prompt_token_ids_sha256": "a1074586b7d5e8ba54eb74979360ddc0985dedcc4690ea0e96aa05c66a852eac", + "prompt_tokens": 79, + "task_id": 491, + "total_tokens": 113 + }, + { + "code_token_ids_sha256": "dc7f9d03e15ff91b0afaffa2c18f82ef619790057cf796079e2a86eb2351d9dc", + "code_tokens": 96, + "prompt_token_ids_sha256": "b642a1f2501144f1dabbbb1cb9949b300278345dcc62521dece305bae7ea475a", + "prompt_tokens": 123, + "task_id": 492, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "23c40de2385c008e5be4b0194e11dce1dee0f80f3c6776ff59b284bf95cc1c9a", + "code_tokens": 451, + "prompt_token_ids_sha256": "c2cfb3be95303df90702ee8a9bdc4219c2fe02b87565f5fdfd878ac9ef0c02cc", + "prompt_tokens": 3722, + "task_id": 493, + "total_tokens": 4173 + }, + { + "code_token_ids_sha256": "93c994f70d3bc6a1f67d304d12b5dca6d3daaf6b3697bfee0f89076bed3c6993", + "code_tokens": 37, + "prompt_token_ids_sha256": "6d667f7560e92609220e768b7bb31e345b653941a9429f2063c06568befcce7d", + "prompt_tokens": 133, + "task_id": 494, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "78e752283c2744cbf23e66d626b36293032b6b3b8f566201d02fb6ec96914f22", + "code_tokens": 42, + "prompt_token_ids_sha256": "21ee124724ffa301449ae66148543b3fa5c34b21402ed3f321e0d11f560b3118", + "prompt_tokens": 132, + "task_id": 495, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "f44d9f573a5beac095b5e1aa840edb8539a0103b71082ded5dfae64ad02a64cc", + "code_tokens": 33, + "prompt_token_ids_sha256": "5f41fcc745a14d5a878ffed67d413f03e42540aa3b16fe0d94d23c598a2b4dfd", + "prompt_tokens": 220, + "task_id": 496, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "8a397b991fa55ec79eadfb6488a15ea897df0899ae7c8981c443ad06a57a1ee3", + "code_tokens": 43, + "prompt_token_ids_sha256": "2da55871caff716f28954caf30fa2db2ea6b2daea63d26837c43d7bdced9723f", + "prompt_tokens": 127, + "task_id": 497, + "total_tokens": 170 + }, + { + "code_token_ids_sha256": "6661f7ed19e406f2317fec4f44f0ada57ec1f0b10ea366d79b058d1f71267bbf", + "code_tokens": 73, + "prompt_token_ids_sha256": "359c1d8b8fd2d67651a7101634dd0185fc4b9b07f14ce424359f2ffa35ec1aa8", + "prompt_tokens": 74, + "task_id": 498, + "total_tokens": 147 + }, + { + "code_token_ids_sha256": "61b550bacbf4f5ade681ce6d3316f754b9b8632fad3ee11c8a0ae1c07709ddcb", + "code_tokens": 15, + "prompt_token_ids_sha256": "34a2ce2710676122000b47fab50cf955787eacb5ce87982294b291ee7d4cb080", + "prompt_tokens": 67, + "task_id": 499, + "total_tokens": 82 + }, + { + "code_token_ids_sha256": "7e42ecfac3cb92bcd549e06a3a6a118309e62316020f39621ad785ec9f656ab9", + "code_tokens": 34, + "prompt_token_ids_sha256": "18b4ce9c80f0e050e7718669fac870f89183cb4d1457262daab964172973a2f1", + "prompt_tokens": 133, + "task_id": 500, + "total_tokens": 167 + }, + { + "code_token_ids_sha256": "272ae1e07db15913e36686e828b6c6bee67b4ad5c15f959b237fe08281c6b697", + "code_tokens": 136, + "prompt_token_ids_sha256": "355babcc0017d88140442a53287e969a4fb18f8392a3ad62f668cd964a3a2fba", + "prompt_tokens": 83, + "task_id": 501, + "total_tokens": 219 + }, + { + "code_token_ids_sha256": "38dcc2c5f3608e347aae5b8c529611a497f5a4da064f97f2ed8aca95ed2f26b3", + "code_tokens": 17, + "prompt_token_ids_sha256": "f71db5cdd1104bed62b243ec11f2cdf048ef71d25650ba50d817e40a7a9242db", + "prompt_tokens": 72, + "task_id": 502, + "total_tokens": 89 + }, + { + "code_token_ids_sha256": "23d54ad1d9f9835151238f004ac6ec1d6a7a07eaf534236358da20919ae977b2", + "code_tokens": 33, + "prompt_token_ids_sha256": "1658555e651da81eb5e2a379d1eca2367bab07e7e57488ad54e826002f58949f", + "prompt_tokens": 204, + "task_id": 503, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "322ea773ac4d585f26c9018703a7fbe22c2e5fb20d6d70f0ee78242612d9a81d", + "code_tokens": 38, + "prompt_token_ids_sha256": "d979c6b02c5170f9edba9f3e40f138ce375ef5684bf6969a554863aa2210af60", + "prompt_tokens": 80, + "task_id": 504, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "612e4915029db18a774d073a4c8c06326fca881421ec262f904b724410817339", + "code_tokens": 61, + "prompt_token_ids_sha256": "8b2d23fe7b7e6ee8e2f05e2cbb590a48185455c802e87da445286d65ddc45359", + "prompt_tokens": 226, + "task_id": 505, + "total_tokens": 287 + }, + { + "code_token_ids_sha256": "97bc8713661d8215d82e9e1620c4b1f1a8255335b7c1df84a4a7c2bc1646c3cc", + "code_tokens": 143, + "prompt_token_ids_sha256": "84073a618bed6d26ec08d9b924a5bdc328364505d0b4361b079a4f5d7862e991", + "prompt_tokens": 90, + "task_id": 506, + "total_tokens": 233 + }, + { + "code_token_ids_sha256": "f0267cf2f4af677e78c2c9534a374e5600bf4530170a92850abd69ab1c2311ca", + "code_tokens": 41, + "prompt_token_ids_sha256": "c0a21f6cb63c71a868553c0f4f9efd0ce28994217d86dcdce03adf06ab7ae3d5", + "prompt_tokens": 158, + "task_id": 507, + "total_tokens": 199 + }, + { + "code_token_ids_sha256": "54a0439163c990f4b8bd409e0011c43266ce73efff02004e5819acffdbfea07c", + "code_tokens": 67, + "prompt_token_ids_sha256": "33d0a6c82b049e809e4a32abed723f7488a3dba8f09207305e4f7bcd0f2ab995", + "prompt_tokens": 122, + "task_id": 508, + "total_tokens": 189 + }, + { + "code_token_ids_sha256": "c919d532de1cc40a854433d29cd0a02a68f75fb068cc721aa8e8679abd46ae50", + "code_tokens": 76, + "prompt_token_ids_sha256": "55426ecf1f35826eb1932ae647a05e1df389b4feb38e2e2666c5815a816cf95b", + "prompt_tokens": 77, + "task_id": 509, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "55c06b75b4e56aa5bc66ce0005c5346ff8bb309046d7cedbcda2a87e4ffed01b", + "code_tokens": 137, + "prompt_token_ids_sha256": "095088515de6040e32a3d4dc9e9e68dcf3dedcbe44fa4571650d81ef1205eb34", + "prompt_tokens": 115, + "task_id": 510, + "total_tokens": 252 + } + ], + "token_manifest_sha256": "199a8836489af9bd0af3fec027e85d57df356bd9919492b24015de51d143f525", + "tokenizer_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + }, + "validity": { + "configured_gdn_layer_indices": [ + 0, + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 16, + 17, + 18, + 20, + 21, + 22 + ], + "packed_qdq_preflight": { + "absolute_tolerance": 1e-06, + "maximum_absolute_difference_by_candidate": { + "mse_selected_nearest": 0.0, + "random_l13_nearest": 0.0, + "random_l18_nearest": 0.0, + "random_l4_nearest": 0.0, + "read_risk_l0_nearest": 0.0, + "read_risk_l0_stochastic_seed_2339": 0.0, + "read_risk_l0_stochastic_seed_2340": 0.0, + "read_risk_l0_stochastic_seed_2341": 0.0, + "uniform_int4_nearest": 0.0, + "uniform_int8_nearest": 0.0 + }, + "passed": true, + "task_id": 945 + }, + "reference_recurrent_state_bytes": 18874368 + } + }, + "schema_version": 1 +} diff --git a/evidence/mbpp-v02-development-manifest.json b/evidence/mbpp-v02-development-manifest.json new file mode 100644 index 0000000..5a1462a --- /dev/null +++ b/evidence/mbpp-v02-development-manifest.json @@ -0,0 +1,1214 @@ +{ + "artifact_kind": "recurquant_mbpp_prepared_manifest", + "canonical_evidence_sha256": "2b13dfb1799472b2fd0006cb87cea60c67a7af37825a87f1686a6fe3d8e38d7c", + "created_at_utc": "2026-07-22T04:05:29.027563+00:00", + "evidence": { + "claim_scope": { + "confirmation_touched": false, + "outcomes_computed": false, + "phase": "development", + "protocol_eligible": true + }, + "environment": { + "command": [ + "python.exe", + "scripts\\evaluate_mbpp.py", + "--phase", + "development", + "--calibration-artifact", + "evidence\\mbpp-v02-calibration.json", + "--manifest-only", + "--local-files-only", + "--output", + "artifacts\\mbpp-v02-development-manifest.json" + ], + "datasets": "4.8.5", + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "tokenizer_class": "Qwen2Tokenizer", + "torch": "2.11.0+cu128", + "tracked_worktree_clean": true, + "transformers": "5.14.1" + }, + "schedule": { + "candidate_plan": [ + { + "default_bits": 4, + "name": "uniform_int4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + { + "default_bits": 8, + "name": "uniform_int8_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + { + "default_bits": 4, + "name": "read_risk_l0_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "mse_selected_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "random_l18_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 18 + }, + { + "default_bits": 4, + "name": "random_l4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 4 + }, + { + "default_bits": 4, + "name": "random_l13_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 13 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2339", + "rounding": "stochastic", + "seed": 2339, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2340", + "rounding": "stochastic", + "seed": 2340, + "upgrade_layer": 0 + }, + { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2341", + "rounding": "stochastic", + "seed": 2341, + "upgrade_layer": 0 + } + ], + "row_count": 90 + }, + "source": { + "calibration_evidence_sha256": "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee", + "dataset_manifest": { + "config": "full", + "dataset_id": "google-research-datasets/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "development", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "row_count": 90, + "rows": [ + { + "sha256": "16539782772e967a03051a2424573c762df4d1a0b59fbf720916caa9f988c859", + "task_id": 511 + }, + { + "sha256": "f76d352612a44fc427a30d6a58e5248a52d9e3be67e22e5879db0f565824d11f", + "task_id": 512 + }, + { + "sha256": "33557131013ba79f2690961b71d2144c4c40a64d1384aaeece38c4b4daef0f46", + "task_id": 513 + }, + { + "sha256": "aa78e8ee8b097226d95cb1e3725711aafc807b76ac501cd4741b983763af352a", + "task_id": 514 + }, + { + "sha256": "7b6f0b9f04efaa01338dd58b5e752158ed79dda7efddbbdd72346b72a9a61385", + "task_id": 515 + }, + { + "sha256": "dfd91fce57caf634b348738899534dcaa25f01f399c9eb3d226f8cd7abd7f349", + "task_id": 516 + }, + { + "sha256": "99174fe6e04ee92839869dc8c3e41e5d08b7777ba5659d674e36b30e8a336b40", + "task_id": 517 + }, + { + "sha256": "b9a1893c5fe1eefde29928bc5a4e92dae8de3a14f6db55cc9a970c063dbe69f4", + "task_id": 518 + }, + { + "sha256": "b2cbba2b099669b6d34afb7aa1faba00425831e57544d6f10832e0fd222c5e72", + "task_id": 519 + }, + { + "sha256": "3d333ac553e93e284761c6ccefa1c12a8dc79029a6164c885c153637a6cf6151", + "task_id": 520 + }, + { + "sha256": "09e4fa8324f54eb313947ffee0c3cc905fb95ea82d74d85eeb39e73555f48911", + "task_id": 521 + }, + { + "sha256": "048a2a45595f7ffb348c4751a64579a58d27aaeb13c6892318bbcf79d06bc5e5", + "task_id": 522 + }, + { + "sha256": "cc57ed07e290bcb08265c17225fd97f414f8b8cc7b71b377625c9de373f451a2", + "task_id": 523 + }, + { + "sha256": "7137b48b5888b18f28401d92529ec16111b5fd3193ffaca44b515a6063968405", + "task_id": 524 + }, + { + "sha256": "350a6b69a72702e6118599a3c89489a8992035e9f34c6a5ab2d7222f523ac76a", + "task_id": 525 + }, + { + "sha256": "462863306f0b5a3b547ffb08bdf59913700b3e591a7111f34b8c4155630c867c", + "task_id": 526 + }, + { + "sha256": "1c9ced29d24570a5cf27a6ae168799bf794e229f623306f885ad4f9f5e5febe2", + "task_id": 527 + }, + { + "sha256": "88660b5a56b37453ac32739a92cd8c7fa870906057b09d87e679f80925c02639", + "task_id": 528 + }, + { + "sha256": "f2c36ff713e36744ca27b046dda2eff681a29bf2b70c38d375344a3814c64e90", + "task_id": 529 + }, + { + "sha256": "fc4f6f81691062ff7a8a10627345b1a4189e4f069f1d15295bec02b1eba34d65", + "task_id": 530 + }, + { + "sha256": "63c5923940787de5251261d2ab48c51c43ed505b004c7abe67e2a304b80c6bb9", + "task_id": 531 + }, + { + "sha256": "5cffdbcc7332c69e5fadba155c1a56d20214275a2290b1faea43b9d7454ae0ac", + "task_id": 532 + }, + { + "sha256": "34507c24ec617e760d596c33536375821d165844c053d909aec94e69a5457c5e", + "task_id": 533 + }, + { + "sha256": "981115e5f0d1d2eabe511128f24790c317b8603d4703a9514eddbecfb852c4f0", + "task_id": 534 + }, + { + "sha256": "d44acaada238005e937cb67ffd3294028f3e9ab9225f36d395dea28b9ad1a8f2", + "task_id": 535 + }, + { + "sha256": "df8e21d38a6aa5fce8e18c704c3b9f2046ac39f18283d1c8b68ab67a37564240", + "task_id": 536 + }, + { + "sha256": "6fcfcf8527b287979934b381c55da12c4aadf3dfef6c50c97a0c3d702bf8564e", + "task_id": 537 + }, + { + "sha256": "5bba58d2fe0ae36a471ed13d20fe8f47dbc14673ec10bdb8edcf7398b3b3eae0", + "task_id": 538 + }, + { + "sha256": "2264ff589d94af8120dcd7409c7258d0fdd11f0dc51140949bfd3ec9d2fb6cf9", + "task_id": 539 + }, + { + "sha256": "50d5ad5b1b4b44d60a3e781ecfd4413823d35f32bd40dac603911273c195cca7", + "task_id": 540 + }, + { + "sha256": "d08d7abec63ce529ac868d965e93bd7569f36a4c306cc17cbc37396b29a3554f", + "task_id": 541 + }, + { + "sha256": "561a90eab2d613d8ae2a2ddca21b834e74e6222c9d2f558e1c0c73bb27169080", + "task_id": 542 + }, + { + "sha256": "9b8d91e99303fa2f55a734d90423b5f8efb03b9b0ebb301a69f37dfd8e18b159", + "task_id": 543 + }, + { + "sha256": "1356820047fc2bdfe32e7da566bb4a8d8dbed14f969f3f92ab9c336a40427f1c", + "task_id": 544 + }, + { + "sha256": "4e67548c2742da86a6330bb8d19d71d83f8968942f700369888d545d4cad7764", + "task_id": 545 + }, + { + "sha256": "9a73544e4d97993c354e5eac432a167ba0cef62fd6a278f7548cf87b9bddbc68", + "task_id": 546 + }, + { + "sha256": "57cbf36bde98ad9274dae823802dfe24b9691c166d64be25c3f816efd61d670a", + "task_id": 547 + }, + { + "sha256": "4b0aa73f6d6578d0eae9afd0c325ecd9330aa703e04fb86fa80bc2f8d7860365", + "task_id": 548 + }, + { + "sha256": "9b9468ec54691d614a877f4c12c76f485fa9a4b9f6e9da039ffbdd9a98a6fec5", + "task_id": 549 + }, + { + "sha256": "ccb08366a997c78e4ef6ad25895814c0710963d48954854ba892df32ac0ce178", + "task_id": 550 + }, + { + "sha256": "532dccbc3b4271ae51e5e9fb2d699111c42ccee8e92444e7146524c2e1f10df4", + "task_id": 551 + }, + { + "sha256": "72e288f580dd793c46766ca2e47956ed77cbcffa3b1b6fa04fe9d52b0456fce7", + "task_id": 552 + }, + { + "sha256": "1b908764f7f82f56e0fff3be241f18d9bde7dbfd40368cb99a42ca3edb587ea6", + "task_id": 553 + }, + { + "sha256": "f427548f544ac578a6a256cb50c50a4daee3a95cb6d9149895bad518d7dda229", + "task_id": 554 + }, + { + "sha256": "1451ea9e106beb9cb1279cefdbb1e8062d841f8b4c92b36d81d7e5a6f042659b", + "task_id": 555 + }, + { + "sha256": "34a10f500bcd085581e015c639910ed51630b50ea00760299fa0d6292d8d6819", + "task_id": 556 + }, + { + "sha256": "2c6163d2bb9ce3e8196681c440a9065ce290ab0e2a0f2763cf13d4c7a422a3b5", + "task_id": 557 + }, + { + "sha256": "efb5bff598fa1d04b6351fd14d94fbee3cff1c87f5ef46747408389a0350a396", + "task_id": 558 + }, + { + "sha256": "1eede60a0944c7972fc59bc112eaf31006052fb7cee778e73b64a35e34e8fd88", + "task_id": 559 + }, + { + "sha256": "55bdf6ded6325a008afed7382e77f41cbe1dd8401d7d04d2385fe41e621f6c09", + "task_id": 560 + }, + { + "sha256": "a2148f88d51a1d86d8e11ac3c2eff694c79c1b1dd5ba0edfaf16442f7c5f93e7", + "task_id": 561 + }, + { + "sha256": "666d4b0edc42c6167660973da91c6ac5e60121c4bc3bc9f15e13e7c65b1a208a", + "task_id": 562 + }, + { + "sha256": "682d4ba4668cffba0d4b249070175c6a8fa0e8a01df9466dc2b8378432dffc8b", + "task_id": 563 + }, + { + "sha256": "7787b4664632abd06d95b797075cab545108767ae1866a451a52895b291c4e21", + "task_id": 564 + }, + { + "sha256": "2b46b57870a6449c6618e2f71e5572b2c7efef268d4349cac7a9e41fe555b2a0", + "task_id": 565 + }, + { + "sha256": "9406fc598cede4b4a947644f7d348c168d5f8c108a1dc6564273f79973c94cad", + "task_id": 566 + }, + { + "sha256": "8c693a1b6d0296ffc204899a9d11c972986a1e3c2124110d18d9a87a83d8e725", + "task_id": 567 + }, + { + "sha256": "3e9462a2f5b09c148b1100396544b17c7ce10139b140be0c043aed4cc133cee2", + "task_id": 568 + }, + { + "sha256": "9f62b87560eb770a9f91a6945c8ecff1d7986355780f8e4cadb18b8dc50423f6", + "task_id": 569 + }, + { + "sha256": "8c2d543628a5813257413b916291a434a63b984ea8bed827340ca44a4dd17bb3", + "task_id": 570 + }, + { + "sha256": "61afb38ed4f433618e8800c09242f76faaab53a56a3ab09b690d313b7bab7755", + "task_id": 571 + }, + { + "sha256": "2c8a7379729fc7ec649d043decb0fbfe095623a81c1e863ed817b3d233b7b833", + "task_id": 572 + }, + { + "sha256": "0d282a2ed583a62b8a59069b53937214ba7d9afefd445308255ed340fd6421d1", + "task_id": 573 + }, + { + "sha256": "752df590804b76d952d8bc9878a9488b9e22813ba4c677eda742f63c7b8a0591", + "task_id": 574 + }, + { + "sha256": "734ae9ec818699b804770e128c8c26a97d6bdc595fdac4e9b96baec68342e3bd", + "task_id": 575 + }, + { + "sha256": "9d1d17fb32a8e4ddf053297f52eaf42d352a4f63c4965b69b4557f6556d3b708", + "task_id": 576 + }, + { + "sha256": "300e057c2818103dba1d7aa9ec1ac7375b8e6849545cd1cbe93d4cb6eb17e157", + "task_id": 577 + }, + { + "sha256": "30521482439370a44035109c6905e6e264509da5f50a25850e42f373cb616cad", + "task_id": 578 + }, + { + "sha256": "475595ff73dd1d2640b641f5457fd420ea1edfea26e5140eceab81550c3788e7", + "task_id": 579 + }, + { + "sha256": "a052fee8ed395eb8b7b33186776580eea3085af74ddda97c39993ced93b5c93c", + "task_id": 580 + }, + { + "sha256": "8c1674250fba855d56cadcb2fab7cf900b18a302bcab68fcdb01fa72b0c95ec0", + "task_id": 581 + }, + { + "sha256": "285dbf2c4747ad5230c093ca5013ebfbf7a4f003b95dd00f25deb4e0e3d46b89", + "task_id": 582 + }, + { + "sha256": "e01d6880195ba3de7dc9a2fca236b770a12a8b894b79b20c52c783053c04b74a", + "task_id": 583 + }, + { + "sha256": "cc2b5c6c9c995e0fc0620114e27e63c49bbb75aecc65095558b6f68639eb9793", + "task_id": 584 + }, + { + "sha256": "cf0921125e9bc2fd91ea0e210fb64214adb1a17e91bb7e065de3ff81c1f15fba", + "task_id": 585 + }, + { + "sha256": "cd84ae05c7934dfa84c07ce6db49b746451e500a5d5085ed370bbe9a8da52a34", + "task_id": 586 + }, + { + "sha256": "05e81cc4ddf7cc06f5331be8a8843cfff301058c5021493e3cda6754a1b5bbd7", + "task_id": 587 + }, + { + "sha256": "ff3d99e259cf33a48760347e7e72307ac9cdeddf41593e9106cb32da6403bcb5", + "task_id": 588 + }, + { + "sha256": "207117d0ce0432269271a0884d44d9386c843777421358da52fd9f8f6155b7fb", + "task_id": 589 + }, + { + "sha256": "858c5ff0068fb00087cea9a9517827e516ef3cb65245a2ba9f151c349aa960ce", + "task_id": 590 + }, + { + "sha256": "ca5e685cbc5a7c79af583dcf87b4b755248219a40cf06f9fa85ad2557fd19075", + "task_id": 591 + }, + { + "sha256": "41667aa322e7a77985945c02c82463e4dd0d8b404b9f3ebaea8bbf1adb971d00", + "task_id": 592 + }, + { + "sha256": "c8314d475632313295105669890a5afb95439738ac913806b440b8852ab4b71b", + "task_id": 593 + }, + { + "sha256": "15a67602e1d8469b29e2f8e6d91f362d75f52ab837f35cad404e4d0766fdcc91", + "task_id": 594 + }, + { + "sha256": "b1dd5af59b621b44a8c02d82957f9492a40010290f96d774c58ab6db33b39fe9", + "task_id": 595 + }, + { + "sha256": "0c7ad60813363a6d56a1477ce9f97fecfb10672c5c294bec621e03a86f602d63", + "task_id": 596 + }, + { + "sha256": "a7395a4e215b49910fdf20542f6f01bc57956d8c83c439c93ab7762f68cc649b", + "task_id": 597 + }, + { + "sha256": "2d65c727ca52be489f53491583c17306291d15997fde25bb061997a1bda4b745", + "task_id": 598 + }, + { + "sha256": "f1fce5b002964d65de0ff3ea547ff863953d76c66eccdb4cf9d351f262052c49", + "task_id": 599 + }, + { + "sha256": "5bc8c7767d4fde18f1b51f6fda718af7754e8102de557a96948d3846649e2952", + "task_id": 600 + } + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": null, + "source_split": "validation" + }, + "dataset_manifest_sha256": "8fed3da0aae864f4e30c70ad70b0269f759d3592dccb9ab87f24444fa24d65dc", + "model_id": "Qwen/Qwen3.5-0.8B-Base", + "model_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + "repository_commit": "3a3c4a2a11c0822f6c456a74327127f294ce67e1", + "token_manifest": [ + { + "code_token_ids_sha256": "51374b6a1aaaa4f94ea298d6c0ffd6cbff3feb07292b21ad8bb817749a0efb42", + "code_tokens": 63, + "prompt_token_ids_sha256": "75259b4b9ad144f8051ae373b42264664a1269f4a7b931145eb6fe023168cd77", + "prompt_tokens": 81, + "task_id": 511, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "77c6cbc907a8142e0967355cbe47e1383ff152405e182630e09dba83e5c9ddf9", + "code_tokens": 92, + "prompt_token_ids_sha256": "dd8d15e4f657d6628474dacc457182c8d1dd0c54ab2a11778c77a8e7b763bfbf", + "prompt_tokens": 231, + "task_id": 512, + "total_tokens": 323 + }, + { + "code_token_ids_sha256": "22100a0192c9be5cda5095a25f4f3bdfd5ec658d286b988cbdd4594f31b8e981", + "code_tokens": 36, + "prompt_token_ids_sha256": "069c694dc4cd41040430380369a4821bf38002a6cc85ed4aefff2810505aab53", + "prompt_tokens": 216, + "task_id": 513, + "total_tokens": 252 + }, + { + "code_token_ids_sha256": "0f48dd5315bc1cdb3f5535c4a8671bc275b8345f05d0b441899c7feadbe1ad1e", + "code_tokens": 24, + "prompt_token_ids_sha256": "d52b7be5af0294918bb9505625012c7fe12e114f908da8666f48204ab9d4fd6d", + "prompt_tokens": 122, + "task_id": 514, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "3159d14cd3c75e4d185b30935c27126c1af21ff91ee6bba3d7491da2fe940d0b", + "code_tokens": 153, + "prompt_token_ids_sha256": "cfcfc5b2d2cd0276903a66f889439012b326cce0cb7dc409595f98b53d301cfa", + "prompt_tokens": 102, + "task_id": 515, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "000c9196d18f4cdb0ffe066e528c9ffb20aba524f43c7defb02fb55236f98792", + "code_tokens": 133, + "prompt_token_ids_sha256": "d4b8a13b6e52b6e720ee8ca4d1e49f0ff192b2d6c5d3daab08878b7e42cb54e4", + "prompt_tokens": 172, + "task_id": 516, + "total_tokens": 305 + }, + { + "code_token_ids_sha256": "074f33d4d8b723867b2cecb32db5523ef4fab52ea7685b9a6ebb4c1bc5cb24c9", + "code_tokens": 39, + "prompt_token_ids_sha256": "00f9858ee09f99ee47de95b53b74e8c8289ae718e1b9fdbdaffd483d474779e5", + "prompt_tokens": 96, + "task_id": 517, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "769a9f5dec545efe90416aff1c3092d35e4ad33d7e0366ecde6d3fcf461b6c06", + "code_tokens": 26, + "prompt_token_ids_sha256": "3b5a4c340457d6d619ecf1b0873619439dea3ee4db377c6fd07c8b802aa7a83f", + "prompt_tokens": 67, + "task_id": 518, + "total_tokens": 93 + }, + { + "code_token_ids_sha256": "4601be81bcfb579e9985d389a9bfd95d01de6d7b34089f49f52d5b070fec303f", + "code_tokens": 38, + "prompt_token_ids_sha256": "10701accb1e298180004e4faca7dca1941a92f4cfd48c6a0c7c8baf45154b02f", + "prompt_tokens": 93, + "task_id": 519, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "32f7307f163387d6ac00fa8ed642db762ed28eabc2ad53b9259b55b5abcc7958", + "code_tokens": 173, + "prompt_token_ids_sha256": "9f4f84a3737a6307b999d18d3c33f4b0167e897ca21ec55319705298b8494912", + "prompt_tokens": 111, + "task_id": 520, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "82b30bc50679559260db86fbe7ec7c9c813ca57a40d8569aa18030ec4afc0ee9", + "code_tokens": 37, + "prompt_token_ids_sha256": "446652e33075f3163e9f51b858ee2f47f9ed74277b41361215777184214faac5", + "prompt_tokens": 90, + "task_id": 521, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "bdb5c51e5be3003a1977927d8cfc86c1a173411fad980ad763e5f102f41fb3fd", + "code_tokens": 205, + "prompt_token_ids_sha256": "254263abe8129d6ba10402421cfa441b86486a6a0adf383bdaef1bdda87050a2", + "prompt_tokens": 164, + "task_id": 522, + "total_tokens": 369 + }, + { + "code_token_ids_sha256": "92604d63a2c91ea949c6422282f570352135688738bd36a48a60afa9c8ab97eb", + "code_tokens": 162, + "prompt_token_ids_sha256": "2676c6487486db2d9d24f9402c2db16d2d0398b4edb9d610383b9e28cc17d0f2", + "prompt_tokens": 122, + "task_id": 523, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "18f82cf0a7adda89da894b33582306c3d65b03e82444a657f1f381fa666d1c3b", + "code_tokens": 132, + "prompt_token_ids_sha256": "2d7972ca89b84d7df49723b706a76057f6512d98f094afd3f14ea566a0ffbf38", + "prompt_tokens": 140, + "task_id": 524, + "total_tokens": 272 + }, + { + "code_token_ids_sha256": "adbd4743c7317532925a3538361651dfb7648a76d95f444557b8bf7e9d0cf2dd", + "code_tokens": 33, + "prompt_token_ids_sha256": "3b4c9309ec726ff6903516c27fd4ad78c81adc968d05be79b40d06dfe49bec2d", + "prompt_tokens": 96, + "task_id": 525, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "c933d3b7225620deb30b98c5dcdb75530fe71324523e45d42b41ee37fc981a8e", + "code_tokens": 60, + "prompt_token_ids_sha256": "27a68f8991973a0d22b0d3a2aac80206f9054298ee64dd5f0a7de6646c5c86e7", + "prompt_tokens": 92, + "task_id": 526, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "cc3c1f5fea2e1aff7b35f0d792546a3586ad8f6451f29ff934adac7fabb27cad", + "code_tokens": 63, + "prompt_token_ids_sha256": "b9ed5d0bb5681bcf79b906ca643db22f66e72f00e00601749d95cf1f728939da", + "prompt_tokens": 127, + "task_id": 527, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "9c751aa9dc10a946442d4588bf03dcf7c9e261c685441aea99d1ae3e719bd9b4", + "code_tokens": 48, + "prompt_token_ids_sha256": "e36501d3283d8b41af319cb3fa079080eab9b2f5e99c310b374d7845d4d84e63", + "prompt_tokens": 140, + "task_id": 528, + "total_tokens": 188 + }, + { + "code_token_ids_sha256": "143ec378ef4a3491fcefd1cd895cdd4481b0d3e84ca81a44c8e0f4b401f4a0a1", + "code_tokens": 76, + "prompt_token_ids_sha256": "4cb8a5b00a5a292cb2ed0c860ec6de91dceaaeb8d6927af1285ad2df6e67f205", + "prompt_tokens": 85, + "task_id": 529, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "8138852a6a1ab85b2ce53557ac38dd11fe3d67b0763225027c6108f42d72479d", + "code_tokens": 63, + "prompt_token_ids_sha256": "6d2b7047e1098ae60806bfa835123a5da10f882ed0b9ae8072ac3ec48e353b05", + "prompt_tokens": 175, + "task_id": 530, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "3d00f2afaae35918bc269ee7c39bcd0d546375083cdbb6260e3c5f3245292f9c", + "code_tokens": 107, + "prompt_token_ids_sha256": "eb0d6242db32c79d92aad0ae39255447d6fa5072a6673127d19dfda8b3838f2a", + "prompt_tokens": 107, + "task_id": 531, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "20aac8f787234bcaa336e6ae465d6a10a395e328e85700937f1f24edb10dc1de", + "code_tokens": 108, + "prompt_token_ids_sha256": "06294dbec467aa598b4371d9d9d813b7a437b967041d26094a7364a88245a57c", + "prompt_tokens": 84, + "task_id": 532, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "46d59a814add2a2778a5e40848bfef52ce2bcf53dc4b432443756e7048830764", + "code_tokens": 46, + "prompt_token_ids_sha256": "c09c801c7c6ac0b649151d1849e098ce881c25a6fcb9e12e60c70ab7c792f107", + "prompt_tokens": 136, + "task_id": 533, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "e35bd424ab07b25c36f31bd857e6b218edf65b42db4435e3714633d07e6cabd3", + "code_tokens": 37, + "prompt_token_ids_sha256": "575d23bccdbee14ecb2a336e4590c89ac9ecd69802a31a08db34866d12f06111", + "prompt_tokens": 102, + "task_id": 534, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "4f65a9707281c698d80cec5c5422f8b142d54115f6d9d948d759ed0de696a02a", + "code_tokens": 29, + "prompt_token_ids_sha256": "3ee1ca9226cb8adadc0932ea4785e1a9141ea7b119b3086940ee52df83fd1b67", + "prompt_tokens": 110, + "task_id": 535, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "e26dfa7ced89a887560aceff73afc50e9c2bd530b4237113873f3cbd6953b418", + "code_tokens": 13, + "prompt_token_ids_sha256": "3289151a3436f9f0d6b4e0e86e19e49706da28817f7c278fb8205a6a965f028c", + "prompt_tokens": 147, + "task_id": 536, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "d38931f56e911f2b37ca025f685c144e0b91198aba10e4934e306ec0c223c18a", + "code_tokens": 51, + "prompt_token_ids_sha256": "33c14d0229cade7ac382322706f276d02dfddef6588774dfa4c548900ee25611", + "prompt_tokens": 90, + "task_id": 537, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "4cf4521bbd35856831644bcb747c1925ffaddb8a00bec610a71a87db3f4972a0", + "code_tokens": 29, + "prompt_token_ids_sha256": "f7363fdc3b2b06194c20e182a14171b4118c78bc7d2fa5b08af71ee6d19f3aff", + "prompt_tokens": 144, + "task_id": 538, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "e6f786238b93dddcc4ce4fe1211e622c2f39c49bbbfca7cc06387cc357c1d75d", + "code_tokens": 30, + "prompt_token_ids_sha256": "7c64a69f1a3710020356623911ae68407e8c77f117ed60148666e91ad7be2b6c", + "prompt_tokens": 701, + "task_id": 539, + "total_tokens": 731 + }, + { + "code_token_ids_sha256": "ebfd1768c1c99bb8e2b5861818c73fdfe71ce2ef9725d65dc074e1ae056f194f", + "code_tokens": 103, + "prompt_token_ids_sha256": "e0c55cd39206f43105dbf5350e8d8ada362ddaa569801e04613be2e285a319cb", + "prompt_tokens": 124, + "task_id": 540, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "d9354f12861dec6dce841b82c6622d90d6b30271b8d09885282ee2852cc2f1a0", + "code_tokens": 119, + "prompt_token_ids_sha256": "05a59f82d30668cb33b13052dcfc64216005b820ec7331267951a00459b3d650", + "prompt_tokens": 76, + "task_id": 541, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "cea3a4282e98a85f19f1205dddd0e6dde09d24ca6e89d2694576f95d4b0ebffe", + "code_tokens": 22, + "prompt_token_ids_sha256": "1163a0581db48fcd93d6a868d446004a50afa275a664a4a42b92ac6299cb48b1", + "prompt_tokens": 120, + "task_id": 542, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "97c3db09b220d423604683a36b402917de5ef017595b5fe127b03033680326a8", + "code_tokens": 51, + "prompt_token_ids_sha256": "8a901d20e8392578080c95c58fdd664005a24577338ce80ade129c03ffc26b70", + "prompt_tokens": 102, + "task_id": 543, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "a4e481b7ad37f0fdc86a49666d9b76d0904317dda1e496bb9872122e7cdaf515", + "code_tokens": 32, + "prompt_token_ids_sha256": "0634b3db81a69635cbf3b87ee15272ff7e1f8636b3e435c27de19bcb37729164", + "prompt_tokens": 214, + "task_id": 544, + "total_tokens": 246 + }, + { + "code_token_ids_sha256": "024f5f88354da3fe4564aef78da670ce8fd0dfa2c4841da17e21ef48661f9af7", + "code_tokens": 116, + "prompt_token_ids_sha256": "9023a4481fc5950af9ebb1c1f997fc5c4c1e581960997207b7228eb740a1ebc1", + "prompt_tokens": 84, + "task_id": 545, + "total_tokens": 200 + }, + { + "code_token_ids_sha256": "b515f1f4c958b0461b960569600bca4a378fa19a9ee822f9030348ae49c68900", + "code_tokens": 56, + "prompt_token_ids_sha256": "a4a0328eb44f57e83b2e9333ebcba25eb7dd20b2a6ea3994fec0ff40738a7707", + "prompt_tokens": 79, + "task_id": 546, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "c4a23f4ee95a4913df228749ed6716734c5fa8514a81f882d4cdc8b50f1a642b", + "code_tokens": 53, + "prompt_token_ids_sha256": "65f76fdbff8c9bec92167d208b27f552857a87c5aa5074b1348de1f02adaac3f", + "prompt_tokens": 89, + "task_id": 547, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "0198e0a8df3b9ebe2cb0ecb13b3e3af2737b8a049c64ec617f58319d7e6a40d0", + "code_tokens": 127, + "prompt_token_ids_sha256": "acd3f8a15a3349d5bbb4940e11200a437fbac95f8936ce403c20cd5f17e7af67", + "prompt_tokens": 144, + "task_id": 548, + "total_tokens": 271 + }, + { + "code_token_ids_sha256": "c0464aa780660588b93032dd6e3f725c05bf5de9c6ee6dcce9d8e2ff52b63149", + "code_tokens": 61, + "prompt_token_ids_sha256": "21fff0ae75e9ab5439adb71bfe4c61c78b7e1126846aac5a15fe300dcf9a0183", + "prompt_tokens": 85, + "task_id": 549, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "0393dde6e334233444b072427256a83ce1733d121aed9aefa1012f60290f1fad", + "code_tokens": 147, + "prompt_token_ids_sha256": "c1c4083ea9b0310f65f2ffcc4f9f4670da5f8c0614679e349f487730ebd6d962", + "prompt_tokens": 104, + "task_id": 550, + "total_tokens": 251 + }, + { + "code_token_ids_sha256": "916b13753989ac021c075b35a952673224a9c6472d2169cf6e0ba31db04b945f", + "code_tokens": 28, + "prompt_token_ids_sha256": "12bbc144384b85e069d3902d8dc9a6a9d1c490c373edabd87faac4e38ba471e6", + "prompt_tokens": 192, + "task_id": 551, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "032bb173044d1b1baad318413f1e9a319de5fb55cc3970d095ba60ea9661bb01", + "code_tokens": 66, + "prompt_token_ids_sha256": "50529dbe74b719b38f1ed65f449a60c15cfd00fa0cb0a37e443c7bae3faddaad", + "prompt_tokens": 100, + "task_id": 552, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "7c356523732a7257d7795ace17f6949088120e447cabd6d8e79054e2bf2ba861", + "code_tokens": 33, + "prompt_token_ids_sha256": "a0885f78db1f8f9dd822fef4e9ba4563285905ec835f90d5774bfdf9a4fb6b83", + "prompt_tokens": 98, + "task_id": 553, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "a03f695dd30a01cbbaa31308ddfeb9d51d49684c571fc9bdce218bbb064b21ea", + "code_tokens": 41, + "prompt_token_ids_sha256": "67d69de764f87d5cf6afc07ca06ae5cd4214993c4a0a4b0dd872b60382017064", + "prompt_tokens": 107, + "task_id": 554, + "total_tokens": 148 + }, + { + "code_token_ids_sha256": "4c96f929f3c9cf8910e259e2b54308c7fa4858dc8ab59fac3202b6e08522484e", + "code_tokens": 36, + "prompt_token_ids_sha256": "ff9f070b889222df347d50b3873a368ff8d35eca5c3d5395a8c2bf7e50253418", + "prompt_tokens": 82, + "task_id": 555, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "7215dc188f10a72e1420e7ff3ab14f7260be052a9f4095d408d5e262ee94b666", + "code_tokens": 70, + "prompt_token_ids_sha256": "1ecf16e96af48a484540536f9b1bbc6d01cea072dca6a2574d0fb568fe4500bd", + "prompt_tokens": 112, + "task_id": 556, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "06df1dab9774538ddae69cd618fc09e7f63f2c97c5e4818770d3ed38ba908259", + "code_tokens": 17, + "prompt_token_ids_sha256": "07983a4a48a52077b6ff90eaaaf6920db154647cdfe9511a772d294583c163eb", + "prompt_tokens": 78, + "task_id": 557, + "total_tokens": 95 + }, + { + "code_token_ids_sha256": "0c461b835bbc821963a03738489506b9e9de4d394c39626b80d38c462ef96e77", + "code_tokens": 23, + "prompt_token_ids_sha256": "d253628f23c9e1bd5bef177cb0c90fb20a7ebae0d84b2b022e73d405030b03ea", + "prompt_tokens": 84, + "task_id": 558, + "total_tokens": 107 + }, + { + "code_token_ids_sha256": "5129d280fb187905d8f85b6f1d65906dcf3d30e1952b92cb316ff4f275ba964b", + "code_tokens": 101, + "prompt_token_ids_sha256": "87541f80a7e96d417c952a6d0b3221993a533cc896561e28926363b96da64445", + "prompt_tokens": 152, + "task_id": 559, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "c2c08c5013c0499acdab47c393de668803b0b9104eec67319389a37d4c0a5f25", + "code_tokens": 36, + "prompt_token_ids_sha256": "11258d79258bb892a11ffbef932656b8d6854baf7183e72728f9c25e7285fce8", + "prompt_tokens": 201, + "task_id": 560, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "5d95e6e698c4a0356b7e5cda82d46476cb44b66fe434bc4aa2edb704d0720bde", + "code_tokens": 46, + "prompt_token_ids_sha256": "5561ed193f416f7b3bc28893683b061159c9efc497a2274b45d64711b59ec0f6", + "prompt_tokens": 266, + "task_id": 561, + "total_tokens": 312 + }, + { + "code_token_ids_sha256": "e27e7bfdb239e4650f0614f134b84d1d988e9b4e48935e706ef19c00b7ed3a3e", + "code_tokens": 24, + "prompt_token_ids_sha256": "b2b117219ac089754a20f737af8241a878354c891f7223b88d891feef1bfcc85", + "prompt_tokens": 126, + "task_id": 562, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "cab54112e86c322a8e94c3f0333b5484b6c3937a8af2d6f9117f506b8cb2ad9c", + "code_tokens": 22, + "prompt_token_ids_sha256": "c8a705344cba3f1b53d50cf0d20ebc9abf1558025a8ed6390ec5c8e666dc9d7c", + "prompt_tokens": 104, + "task_id": 563, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "86de1f8f78bfb419746e498637d064512cb8bc9bbe389d5b9116fd548df14ee7", + "code_tokens": 58, + "prompt_token_ids_sha256": "523346c7e4a28f98c0a353c1afd625c7d4ac18c59facdcefb2e68f76e6391597", + "prompt_tokens": 98, + "task_id": 564, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "1225aeb027e811b628fdc7e265bac65a2cccee232f70f778ff683a47f305acaf", + "code_tokens": 15, + "prompt_token_ids_sha256": "af8a292c6485ff7a67aa7be7b61dd4f32a0adabb6cc637e841f315db89900c4a", + "prompt_tokens": 95, + "task_id": 565, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "c0cb7caed68465ebca3f4e0c860f4222e5d3911cdac8462ecfe52f3819dfebc9", + "code_tokens": 40, + "prompt_token_ids_sha256": "1eaf3e3744abb794f24c4a9c5868e0d414eb9ef8807d572384f1e8e095478711", + "prompt_tokens": 69, + "task_id": 566, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "3f6ef7be7f6afc6346e63d6dadc064c64b0d68a7c2247333c40941fc0eafabcf", + "code_tokens": 37, + "prompt_token_ids_sha256": "78a699beff16fe1ec862cd98e30dec9685bc79279179a4110eacc0390c04b627", + "prompt_tokens": 147, + "task_id": 567, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "c513feb62303b9292018bec3d45145c495d4288f5f9f70f58d15bafa0d82e977", + "code_tokens": 21, + "prompt_token_ids_sha256": "8982a4f08613e6078cb5a9bc7eea08f200d718ce53457c0f685d8dd201d0d338", + "prompt_tokens": 79, + "task_id": 568, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "0bfe140a51a356bc36803306c997a4deede74b240b28c3b2dcb31a9b809c8c1f", + "code_tokens": 21, + "prompt_token_ids_sha256": "6abbb94c85a6bf620395238e7e19749180c779e4293bd5f5fd88700e3f97015a", + "prompt_tokens": 184, + "task_id": 569, + "total_tokens": 205 + }, + { + "code_token_ids_sha256": "05850d31a888779f204340efca917fd9b19fddd87d630bd7cb442352bd7a3e66", + "code_tokens": 66, + "prompt_token_ids_sha256": "63f7b0721b16ddc2fd30e54479a7108b4a93763c21fbe12468f47855932638d7", + "prompt_tokens": 169, + "task_id": 570, + "total_tokens": 235 + }, + { + "code_token_ids_sha256": "04fd2b294cb34945613e5b9de88984a32d58796306b949f79414f253fdf65860", + "code_tokens": 138, + "prompt_token_ids_sha256": "73727137a63fae90dd5520430e344582c703eccd31d7ce84799137d07a28d649", + "prompt_tokens": 171, + "task_id": 571, + "total_tokens": 309 + }, + { + "code_token_ids_sha256": "f3b601ee8f39328aa6168c673aa7a25133e2146709681bc6bb696cf33dad5f54", + "code_tokens": 22, + "prompt_token_ids_sha256": "57d6b5a4961d3b8ab3e375fec11a10b37c759d067f83b0eebc7bad4c0294d79b", + "prompt_tokens": 135, + "task_id": 572, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "ab34c0081e5c216cb42afadfca0adf3c19b0dbe1e9a6ab497a786980efa46137", + "code_tokens": 37, + "prompt_token_ids_sha256": "beca07f10658235a9bc69d69405832cfbe1d6d84ae1d58afcf1678d68ac5d0f1", + "prompt_tokens": 136, + "task_id": 573, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "e4c376d85d55085bed7bde727b22416668f2567755d06476c0ef484a139cc6e8", + "code_tokens": 41, + "prompt_token_ids_sha256": "dc30e7d94350b7dc255ec828efbbeb0dee52859d5b98d6c74df26dc275ff1911", + "prompt_tokens": 104, + "task_id": 574, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "470667f4d5f96fdc5a083bdf428fc4c28bbac0db1192f14ac1a47db679617fff", + "code_tokens": 63, + "prompt_token_ids_sha256": "d023ce639ec2dcbc7e7a0ce0f2234119d1940d1a050b691ce8c583eecc2a6eb7", + "prompt_tokens": 101, + "task_id": 575, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "88d005f8c44e0a678858e25205e9eb911ca580807e4b4a9a0d2ea563a0d14903", + "code_tokens": 99, + "prompt_token_ids_sha256": "3a88fc72092f5076beec4b32b5a630f8df7355d9af2bcb895d42ef78daf74e7b", + "prompt_tokens": 117, + "task_id": 576, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "1681d2006b64d27c99a4add5ceaa2f0e11de29623677271ac6aac49b609b6ea2", + "code_tokens": 65, + "prompt_token_ids_sha256": "1267efcc04062cbda430893ade55eac4fca2ec7f21426024d180e6f770eae9e5", + "prompt_tokens": 86, + "task_id": 577, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "b2f7a02a32599611162313b8ec78e99927fa8de8033e5a50bbf731c96f29bfbb", + "code_tokens": 39, + "prompt_token_ids_sha256": "0ef0064fd0f4dbe294dd7db67c7a80d723171b72b954babe4fa5951ac544861f", + "prompt_tokens": 273, + "task_id": 578, + "total_tokens": 312 + }, + { + "code_token_ids_sha256": "36cfcd07cc222f406192992d0004a955ada75bd94bba1bd4127f4b2a996332af", + "code_tokens": 40, + "prompt_token_ids_sha256": "31b0c6f7aeab1caa69330d6ceff3781ba69ac2bb2f46a00c2ef2affb19117f14", + "prompt_tokens": 184, + "task_id": 579, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "7b4567d0f07f4d5852ce8684b988be0841a65366c2a9096063d04cc462239062", + "code_tokens": 97, + "prompt_token_ids_sha256": "edf415720df08853c0463118174d20e5daa5e58df4ac941d9d6522eabe18b524", + "prompt_tokens": 177, + "task_id": 580, + "total_tokens": 274 + }, + { + "code_token_ids_sha256": "2638278c24f225b62586e1dbd81c2a2747549f39bf002fab89eea18a49e42e05", + "code_tokens": 23, + "prompt_token_ids_sha256": "a679e29787a668b865f07fff79d29288f63ff4fbeb7c869b7ee49df872a1cb4e", + "prompt_tokens": 81, + "task_id": 581, + "total_tokens": 104 + }, + { + "code_token_ids_sha256": "60bdae86096f5b23913acf6eace3638935051c4ee6312860b4d18735ac68907d", + "code_tokens": 25, + "prompt_token_ids_sha256": "045d1683f12b963599f152102d9bb9d3e9b820a7d6c08b4879f4c40788c4cff1", + "prompt_tokens": 67, + "task_id": 582, + "total_tokens": 92 + }, + { + "code_token_ids_sha256": "d616ceeb5baf6144088303469aaad4f54625879238b4854a9aa38125be917cc4", + "code_tokens": 57, + "prompt_token_ids_sha256": "b64dcce5c6936277b647d6e13611de98831e6fbe61857987475ae71ec0be7779", + "prompt_tokens": 72, + "task_id": 583, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "6cca857ae596513638ba100c5de4ffdeb88df0acbb51642271e1261efc8a410f", + "code_tokens": 50, + "prompt_token_ids_sha256": "b784b1b932fe81aac8959dd6437b6bb02a1f02797087ab84a486598b54d15dcb", + "prompt_tokens": 114, + "task_id": 584, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "aba30fa670c5f07f2b26e0c5f6314c31268993bd210b6f32463eeffe179f02c3", + "code_tokens": 35, + "prompt_token_ids_sha256": "3f3d1b321ad2d9cedbc03840b0b8e53d1674f2df51496a9ff6821f73aa780d0b", + "prompt_tokens": 310, + "task_id": 585, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "96c31933f9fdca36e7cefc47888825398ca0a8450205d4bbf3e010b280906704", + "code_tokens": 29, + "prompt_token_ids_sha256": "5c17a36d98e4571cf2da7b0200b11d30a32ba5d52a7ba9a472912609214e64cf", + "prompt_tokens": 159, + "task_id": 586, + "total_tokens": 188 + }, + { + "code_token_ids_sha256": "71625271f8ed82e626b3c864229dad2a05008c15c8243ac229d352293e21f693", + "code_tokens": 20, + "prompt_token_ids_sha256": "c09ab27564d859ff4f20e7381932285ccbf9882f4d92caf229545a28d045d17b", + "prompt_tokens": 163, + "task_id": 587, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "7625cecdcfaf215b042d36e4bf4e036373f11e7bb00b404dd5047b7e52a92b2c", + "code_tokens": 19, + "prompt_token_ids_sha256": "3d1a5aece2df5d1a46135904ea9e2329bfa1cab7635348956e5b174cefad7410", + "prompt_tokens": 89, + "task_id": 588, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "8237eb0d6ac12638b2b7d912fbc2e7f463be40a997bbc34ba838eb58978f3a5e", + "code_tokens": 71, + "prompt_token_ids_sha256": "980ca6f6dc5995989f774a2c0ad0ab1868477a1be7fe355d7dc0a97d3bc19bb3", + "prompt_tokens": 131, + "task_id": 589, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "aa89cf8d51a5e5c6451b748749ffa09c015a08874708c9d5221d1490ab573e61", + "code_tokens": 47, + "prompt_token_ids_sha256": "11b67c36adc8358b170008df9bb3e82a9b709aab9cbcc908eb2084e3ec8bf04f", + "prompt_tokens": 249, + "task_id": 590, + "total_tokens": 296 + }, + { + "code_token_ids_sha256": "7b07b1e6c1f0050b648a8cb7e315156bcbdc3fc41f2072fafc66b3bee84d76cd", + "code_tokens": 50, + "prompt_token_ids_sha256": "9ca2ba10212bb44e5c594536e8c56d109e300d873ba4d8ce6c1bf8929dcf8934", + "prompt_tokens": 130, + "task_id": 591, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "527507dc33ed195f959ba5d9eb4a9319095b88ea156e4c90a1207ab13e59dfdd", + "code_tokens": 113, + "prompt_token_ids_sha256": "4781f3198d0e1616079cd96eb30cc938ea3c1c0ceb5741e8980c3f3ae51312b4", + "prompt_tokens": 77, + "task_id": 592, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "f0102775691666d079133bf1824e8a244c8625708e333807f283a722586f8135", + "code_tokens": 26, + "prompt_token_ids_sha256": "5c4de560c133f6d86b586eaf9091855a43c52d1647735dd0dfed53ec11a03ac3", + "prompt_tokens": 134, + "task_id": 593, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "cea1449991b5740774cb67f9b41b4d21138ce0d715912221375ebe8b6d391985", + "code_tokens": 62, + "prompt_token_ids_sha256": "c860f17a1e8b23e40417cbd3ea961144577844cd93c8daf78e3fc6b8d7e04eaa", + "prompt_tokens": 113, + "task_id": 594, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "a0bc157b791156284cb59efa7c8b5323e5ceff2cd0570c4c613b2a60b4c74dce", + "code_tokens": 79, + "prompt_token_ids_sha256": "68dc877d1339ccd102b221aac28e7289332c535debf453ece9b207008ddc469a", + "prompt_tokens": 105, + "task_id": 595, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "1ed12c70ac0e1f42b45254e355df9a2f2fb344bc83d23709ab50d39daf1e9a72", + "code_tokens": 20, + "prompt_token_ids_sha256": "50bc50fa175cd0ab564bac33ffd115aacf5e5b59b541ef0c70467fe3f3d13ddf", + "prompt_tokens": 202, + "task_id": 596, + "total_tokens": 222 + }, + { + "code_token_ids_sha256": "7e9b720e1a6de57fda397dba6a641c141745797fbbdf065c7d2682521e6a2710", + "code_tokens": 186, + "prompt_token_ids_sha256": "1f175f5a46d86e9fb077540932e5420027773ceb901c18665caf88ba2ed6126a", + "prompt_tokens": 209, + "task_id": 597, + "total_tokens": 395 + }, + { + "code_token_ids_sha256": "5c1585934b732425f8a4a794d598ee0d72bcb0c36e2a91efb1c5bf4e558a854a", + "code_tokens": 99, + "prompt_token_ids_sha256": "4580aa7ec549d56af454e8619ab5e37750b99259c83564f622cc2b3274cb6213", + "prompt_tokens": 75, + "task_id": 598, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "101147ad3965a807926a2f2dca4a0b64cb4170b35f412e3e7bb32030fa44587a", + "code_tokens": 43, + "prompt_token_ids_sha256": "b5ec314f6027e558c46e6bb121bbbbb03cfd3447dd6878de9c6f58e348f67720", + "prompt_tokens": 94, + "task_id": 599, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "ad71990faceebd8ab5e0df092325c417c0cec66bdf5dc1d8d9aacc06ba3d2ec5", + "code_tokens": 35, + "prompt_token_ids_sha256": "f5292225c83fe88c95f576f67804ffbef8674929550913a759f1adc1f1394977", + "prompt_tokens": 74, + "task_id": 600, + "total_tokens": 109 + } + ], + "token_manifest_sha256": "3c19f37f3c35cb22f17e66a5438bb9968b214e0aff64067930bc3a7124f63f5c", + "tokenizer_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + } + }, + "schema_version": 1 +} diff --git a/evidence/mbpp-v02-development.json b/evidence/mbpp-v02-development.json new file mode 100644 index 0000000..b0a173d --- /dev/null +++ b/evidence/mbpp-v02-development.json @@ -0,0 +1,12852 @@ +{ + "artifact_kind": "recurquant_mbpp_teacher_forced_evaluation", + "canonical_evidence_sha256": "301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488", + "created_at_utc": "2026-07-22T05:22:23.426874+00:00", + "evidence": { + "candidates": { + "mse_selected_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 0.5179035793180051, + "macro_mean_kl": 0.5404429111791693, + "macro_top1_agreement": 0.7802550585373588, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 0.6800926509110824, + "macro_mean_kl": 0.7177636649297632, + "macro_top1_agreement": 0.7299536233362944, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 0.8626517951488495, + "macro_mean_kl": 0.8604557893492959, + "macro_top1_agreement": 0.6792999749833887, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 1.020532862706618, + "macro_mean_kl": 1.0432947765697131, + "macro_top1_agreement": 0.6560033803636377, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 1.732251524925232, + "code_tokens": 63, + "cvar95_kl": 3.474870443344116, + "delta_nll": 1.141204833984375, + "max_kl": 3.741807460784912, + "mean_kl": 0.9814460277557373, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.5873016119003296 + }, + { + "candidate_nll": 2.248811721801758, + "code_tokens": 92, + "cvar95_kl": 4.760335922241211, + "delta_nll": 1.2430880069732666, + "max_kl": 5.1050190925598145, + "mean_kl": 1.1677846908569336, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.6630434989929199 + }, + { + "candidate_nll": 2.7197532653808594, + "code_tokens": 36, + "cvar95_kl": 4.589881420135498, + "delta_nll": 0.8062640428543091, + "max_kl": 5.348085880279541, + "mean_kl": 0.9984854459762573, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.6111111044883728 + }, + { + "candidate_nll": 2.2496488094329834, + "code_tokens": 24, + "cvar95_kl": 3.2688493728637695, + "delta_nll": 0.4322397708892822, + "max_kl": 4.011963367462158, + "mean_kl": 0.7629708647727966, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.8724886178970337, + "code_tokens": 153, + "cvar95_kl": 6.034920692443848, + "delta_nll": 0.9555368423461914, + "max_kl": 7.151743412017822, + "mean_kl": 1.1941282749176025, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.6405228972434998 + }, + { + "candidate_nll": 2.5365004539489746, + "code_tokens": 133, + "cvar95_kl": 8.025484085083008, + "delta_nll": 1.4699668884277344, + "max_kl": 11.771374702453613, + "mean_kl": 1.4831041097640991, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.5639097690582275 + }, + { + "candidate_nll": 1.7924917936325073, + "code_tokens": 39, + "cvar95_kl": 3.900834560394287, + "delta_nll": 0.924458920955658, + "max_kl": 4.592340469360352, + "mean_kl": 0.7618184089660645, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.7271370887756348, + "code_tokens": 26, + "cvar95_kl": 2.3618905544281006, + "delta_nll": 0.4661813974380493, + "max_kl": 2.5585296154022217, + "mean_kl": 0.4578700661659241, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.9001494646072388, + "code_tokens": 38, + "cvar95_kl": 2.387478828430176, + "delta_nll": 0.5844765901565552, + "max_kl": 2.5290989875793457, + "mean_kl": 0.5287707448005676, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.9828054904937744, + "code_tokens": 173, + "cvar95_kl": 5.090429782867432, + "delta_nll": 1.129013180732727, + "max_kl": 5.9992475509643555, + "mean_kl": 1.149426817893982, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.6358381509780884 + }, + { + "candidate_nll": 1.4193477630615234, + "code_tokens": 37, + "cvar95_kl": 2.053830623626709, + "delta_nll": 0.5025250911712646, + "max_kl": 2.103363037109375, + "mean_kl": 0.5058015584945679, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.7837837934494019 + }, + { + "candidate_nll": 2.055155038833618, + "code_tokens": 205, + "cvar95_kl": 5.599775791168213, + "delta_nll": 1.2429649829864502, + "max_kl": 6.802768230438232, + "mean_kl": 1.2813493013381958, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.6146341562271118 + }, + { + "candidate_nll": 1.7530744075775146, + "code_tokens": 162, + "cvar95_kl": 5.451320648193359, + "delta_nll": 0.854078471660614, + "max_kl": 6.91378927230835, + "mean_kl": 1.0569254159927368, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.6913580298423767 + }, + { + "candidate_nll": 1.5984687805175781, + "code_tokens": 132, + "cvar95_kl": 3.898916721343994, + "delta_nll": 0.9361112117767334, + "max_kl": 4.909263610839844, + "mean_kl": 0.8876777291297913, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.6439393758773804 + }, + { + "candidate_nll": 1.1327953338623047, + "code_tokens": 33, + "cvar95_kl": 2.5398707389831543, + "delta_nll": 0.4656016230583191, + "max_kl": 2.936553478240967, + "mean_kl": 0.44754263758659363, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 2.111821174621582, + "code_tokens": 60, + "cvar95_kl": 4.277328968048096, + "delta_nll": 0.7700865268707275, + "max_kl": 5.780797004699707, + "mean_kl": 0.7908816933631897, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.5509346723556519, + "code_tokens": 63, + "cvar95_kl": 4.594099998474121, + "delta_nll": 1.1441290378570557, + "max_kl": 5.859574317932129, + "mean_kl": 1.0865414142608643, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.9959763288497925, + "code_tokens": 48, + "cvar95_kl": 3.144836664199829, + "delta_nll": 0.3798454999923706, + "max_kl": 3.6520800590515137, + "mean_kl": 0.7425467371940613, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.6196094751358032, + "code_tokens": 76, + "cvar95_kl": 4.821486473083496, + "delta_nll": 0.8369253873825073, + "max_kl": 6.931477069854736, + "mean_kl": 0.8121975064277649, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.7236841917037964 + }, + { + "candidate_nll": 1.6314399242401123, + "code_tokens": 63, + "cvar95_kl": 3.1793127059936523, + "delta_nll": 0.6803451776504517, + "max_kl": 3.800250768661499, + "mean_kl": 0.729357123374939, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.7936508059501648 + }, + { + "candidate_nll": 1.940843939781189, + "code_tokens": 107, + "cvar95_kl": 5.487821102142334, + "delta_nll": 1.0761098861694336, + "max_kl": 9.730924606323242, + "mean_kl": 1.0419647693634033, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.6261682510375977 + }, + { + "candidate_nll": 1.4738730192184448, + "code_tokens": 108, + "cvar95_kl": 5.022552967071533, + "delta_nll": 0.8587003946304321, + "max_kl": 5.433715343475342, + "mean_kl": 0.9610470533370972, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.6851851940155029 + }, + { + "candidate_nll": 1.5193557739257812, + "code_tokens": 46, + "cvar95_kl": 3.1560914516448975, + "delta_nll": 0.5460243821144104, + "max_kl": 4.615480899810791, + "mean_kl": 0.6608195304870605, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.760869562625885 + }, + { + "candidate_nll": 1.7791805267333984, + "code_tokens": 37, + "cvar95_kl": 4.812079429626465, + "delta_nll": 0.8499225378036499, + "max_kl": 5.208279609680176, + "mean_kl": 0.8442744016647339, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 1.9646399021148682, + "code_tokens": 29, + "cvar95_kl": 3.024362564086914, + "delta_nll": 0.8124837875366211, + "max_kl": 3.2402076721191406, + "mean_kl": 0.7503215074539185, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.7586206793785095 + }, + { + "candidate_nll": 1.6979126930236816, + "code_tokens": 13, + "cvar95_kl": 0.7281028628349304, + "delta_nll": 0.08378326892852783, + "max_kl": 0.7281028628349304, + "mean_kl": 0.31850001215934753, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.9230769276618958 + }, + { + "candidate_nll": 1.7905513048171997, + "code_tokens": 51, + "cvar95_kl": 4.860567569732666, + "delta_nll": 1.16745924949646, + "max_kl": 6.935393333435059, + "mean_kl": 0.9186844229698181, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.686274528503418 + }, + { + "candidate_nll": 1.8261600732803345, + "code_tokens": 29, + "cvar95_kl": 1.67676842212677, + "delta_nll": 0.6957350969314575, + "max_kl": 2.1295695304870605, + "mean_kl": 0.41461825370788574, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.8620689511299133 + }, + { + "candidate_nll": 1.5178250074386597, + "code_tokens": 30, + "cvar95_kl": 3.6664505004882812, + "delta_nll": 0.4882298707962036, + "max_kl": 4.343649387359619, + "mean_kl": 0.6858417391777039, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.9049439430236816, + "code_tokens": 103, + "cvar95_kl": 4.801912784576416, + "delta_nll": 1.1142059564590454, + "max_kl": 5.96905517578125, + "mean_kl": 1.0409752130508423, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.5631067752838135 + }, + { + "candidate_nll": 1.7872135639190674, + "code_tokens": 119, + "cvar95_kl": 3.8967525959014893, + "delta_nll": 0.8866696357727051, + "max_kl": 4.89632511138916, + "mean_kl": 0.9269121289253235, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 1.8620305061340332, + "code_tokens": 22, + "cvar95_kl": 2.795938014984131, + "delta_nll": 0.565159797668457, + "max_kl": 2.9193925857543945, + "mean_kl": 0.7613252401351929, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 1.2055070400238037, + "code_tokens": 51, + "cvar95_kl": 2.6508543491363525, + "delta_nll": 0.5676289796829224, + "max_kl": 5.191623210906982, + "mean_kl": 0.5249396562576294, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 2.443270444869995, + "code_tokens": 32, + "cvar95_kl": 2.4545235633850098, + "delta_nll": 0.6837217807769775, + "max_kl": 2.8483693599700928, + "mean_kl": 0.5845952033996582, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.7185503244400024, + "code_tokens": 116, + "cvar95_kl": 4.748171329498291, + "delta_nll": 0.6552495956420898, + "max_kl": 5.980818271636963, + "mean_kl": 0.7927326560020447, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.7241379022598267 + }, + { + "candidate_nll": 1.7016364336013794, + "code_tokens": 56, + "cvar95_kl": 3.491488218307495, + "delta_nll": 0.9164514541625977, + "max_kl": 4.3265061378479, + "mean_kl": 0.6824652552604675, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.7321428656578064 + }, + { + "candidate_nll": 1.8299857378005981, + "code_tokens": 53, + "cvar95_kl": 3.5960323810577393, + "delta_nll": 0.8821116089820862, + "max_kl": 3.892441987991333, + "mean_kl": 0.9184355735778809, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.6037735939025879 + }, + { + "candidate_nll": 1.8149334192276, + "code_tokens": 127, + "cvar95_kl": 6.560326099395752, + "delta_nll": 1.2037125825881958, + "max_kl": 9.002546310424805, + "mean_kl": 1.1901766061782837, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.6299212574958801 + }, + { + "candidate_nll": 2.148423194885254, + "code_tokens": 61, + "cvar95_kl": 3.989513635635376, + "delta_nll": 0.9358888864517212, + "max_kl": 5.814340591430664, + "mean_kl": 1.063145399093628, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.5573770403862 + }, + { + "candidate_nll": 1.450118899345398, + "code_tokens": 147, + "cvar95_kl": 5.18900728225708, + "delta_nll": 0.9041231274604797, + "max_kl": 8.659774780273438, + "mean_kl": 0.9226810336112976, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.7482993006706238 + }, + { + "candidate_nll": 1.9640789031982422, + "code_tokens": 28, + "cvar95_kl": 2.654557704925537, + "delta_nll": 0.4860924482345581, + "max_kl": 3.840364694595337, + "mean_kl": 0.579189658164978, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.6785714030265808 + }, + { + "candidate_nll": 1.4519189596176147, + "code_tokens": 66, + "cvar95_kl": 4.233325481414795, + "delta_nll": 0.6155489683151245, + "max_kl": 8.123661041259766, + "mean_kl": 0.7767921090126038, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 2.3811237812042236, + "code_tokens": 33, + "cvar95_kl": 5.815465450286865, + "delta_nll": 0.5543190240859985, + "max_kl": 6.136956691741943, + "mean_kl": 0.7374077439308167, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 2.0118210315704346, + "code_tokens": 41, + "cvar95_kl": 3.478135347366333, + "delta_nll": 0.8741217851638794, + "max_kl": 3.7473278045654297, + "mean_kl": 0.7758055925369263, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.7317073345184326 + }, + { + "candidate_nll": 2.2031726837158203, + "code_tokens": 36, + "cvar95_kl": 2.3229804039001465, + "delta_nll": 0.5269949436187744, + "max_kl": 2.323989152908325, + "mean_kl": 0.6381937861442566, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.6388888955116272 + }, + { + "candidate_nll": 2.0672099590301514, + "code_tokens": 70, + "cvar95_kl": 3.522871971130371, + "delta_nll": 1.1500751972198486, + "max_kl": 4.446450710296631, + "mean_kl": 0.9686630368232727, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.8653390407562256, + "code_tokens": 17, + "cvar95_kl": 2.0200142860412598, + "delta_nll": 0.5658745765686035, + "max_kl": 2.0200142860412598, + "mean_kl": 0.4210987389087677, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.8823529481887817 + }, + { + "candidate_nll": 1.5949524641036987, + "code_tokens": 23, + "cvar95_kl": 1.5197210311889648, + "delta_nll": 0.4097452163696289, + "max_kl": 1.6185063123703003, + "mean_kl": 0.41086018085479736, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.602938175201416, + "code_tokens": 101, + "cvar95_kl": 5.3965067863464355, + "delta_nll": 1.2250646352767944, + "max_kl": 6.981556415557861, + "mean_kl": 1.259110689163208, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.5742574334144592 + }, + { + "candidate_nll": 1.9242193698883057, + "code_tokens": 36, + "cvar95_kl": 5.365108966827393, + "delta_nll": 0.5093426704406738, + "max_kl": 6.128144264221191, + "mean_kl": 0.7643920183181763, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.69818913936615, + "code_tokens": 46, + "cvar95_kl": 3.960514783859253, + "delta_nll": 0.4924354553222656, + "max_kl": 5.271826267242432, + "mean_kl": 0.8176872134208679, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.0165300369262695, + "code_tokens": 24, + "cvar95_kl": 2.4906840324401855, + "delta_nll": 0.4856834411621094, + "max_kl": 3.6217143535614014, + "mean_kl": 0.4304535388946533, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.875 + }, + { + "candidate_nll": 1.8642551898956299, + "code_tokens": 22, + "cvar95_kl": 2.5553269386291504, + "delta_nll": 0.4889259338378906, + "max_kl": 2.8784468173980713, + "mean_kl": 0.5888121128082275, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.5909090638160706 + }, + { + "candidate_nll": 1.5638474225997925, + "code_tokens": 58, + "cvar95_kl": 3.671574354171753, + "delta_nll": 0.8749749064445496, + "max_kl": 3.9545581340789795, + "mean_kl": 0.7755099534988403, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.6432263851165771, + "code_tokens": 15, + "cvar95_kl": 2.7998435497283936, + "delta_nll": 0.3303558826446533, + "max_kl": 2.7998435497283936, + "mean_kl": 0.4439100921154022, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 1.2727930545806885, + "code_tokens": 40, + "cvar95_kl": 5.503000259399414, + "delta_nll": 0.8332054615020752, + "max_kl": 5.872326374053955, + "mean_kl": 0.7590802907943726, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.7749999761581421 + }, + { + "candidate_nll": 1.475277304649353, + "code_tokens": 37, + "cvar95_kl": 5.659841060638428, + "delta_nll": 0.8707277178764343, + "max_kl": 7.1288909912109375, + "mean_kl": 0.892333447933197, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.7027027010917664 + }, + { + "candidate_nll": 1.535710334777832, + "code_tokens": 21, + "cvar95_kl": 3.0329627990722656, + "delta_nll": 0.5154644250869751, + "max_kl": 4.079394340515137, + "mean_kl": 0.4050963222980499, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.8222547769546509, + "code_tokens": 21, + "cvar95_kl": 1.8580646514892578, + "delta_nll": 0.5097389221191406, + "max_kl": 2.379983425140381, + "mean_kl": 0.37436068058013916, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 2.17185378074646, + "code_tokens": 66, + "cvar95_kl": 6.8722310066223145, + "delta_nll": 1.1634891033172607, + "max_kl": 11.786860466003418, + "mean_kl": 1.2447997331619263, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.6060606241226196 + }, + { + "candidate_nll": 1.6720259189605713, + "code_tokens": 138, + "cvar95_kl": 4.463130950927734, + "delta_nll": 1.028365135192871, + "max_kl": 5.386237621307373, + "mean_kl": 0.9903431534767151, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.4151910543441772, + "code_tokens": 22, + "cvar95_kl": 3.607597827911377, + "delta_nll": 0.6809484362602234, + "max_kl": 5.018783092498779, + "mean_kl": 0.7517454028129578, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.4702600240707397, + "code_tokens": 37, + "cvar95_kl": 4.183772563934326, + "delta_nll": 0.724675714969635, + "max_kl": 4.6542649269104, + "mean_kl": 0.6333199739456177, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.7027027010917664 + }, + { + "candidate_nll": 1.7952399253845215, + "code_tokens": 41, + "cvar95_kl": 3.1770858764648438, + "delta_nll": 0.8444822430610657, + "max_kl": 4.2851715087890625, + "mean_kl": 0.7267717719078064, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.707317054271698 + }, + { + "candidate_nll": 1.962184190750122, + "code_tokens": 63, + "cvar95_kl": 3.191837787628174, + "delta_nll": 0.803693413734436, + "max_kl": 4.139280319213867, + "mean_kl": 0.8838675618171692, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.5873016119003296 + }, + { + "candidate_nll": 1.5951545238494873, + "code_tokens": 99, + "cvar95_kl": 3.0328381061553955, + "delta_nll": 0.7484027147293091, + "max_kl": 3.474276304244995, + "mean_kl": 0.7106393575668335, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.3291159868240356, + "code_tokens": 65, + "cvar95_kl": 1.767317771911621, + "delta_nll": 0.39902228116989136, + "max_kl": 1.9927705526351929, + "mean_kl": 0.5239065289497375, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.7384615540504456 + }, + { + "candidate_nll": 1.5504562854766846, + "code_tokens": 39, + "cvar95_kl": 4.638823986053467, + "delta_nll": 0.6329271793365479, + "max_kl": 5.931685924530029, + "mean_kl": 0.6130590438842773, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.7179487347602844 + }, + { + "candidate_nll": 1.8587223291397095, + "code_tokens": 40, + "cvar95_kl": 5.913382530212402, + "delta_nll": 0.6999615430831909, + "max_kl": 8.238985061645508, + "mean_kl": 0.8137434720993042, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.4069576263427734, + "code_tokens": 97, + "cvar95_kl": 4.830368995666504, + "delta_nll": 1.1120054721832275, + "max_kl": 6.578426837921143, + "mean_kl": 1.095460057258606, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.6391752362251282 + }, + { + "candidate_nll": 2.215336799621582, + "code_tokens": 23, + "cvar95_kl": 1.4431116580963135, + "delta_nll": 0.3003079891204834, + "max_kl": 1.4984580278396606, + "mean_kl": 0.5395223498344421, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.5652173757553101 + }, + { + "candidate_nll": 1.5997490882873535, + "code_tokens": 25, + "cvar95_kl": 1.6611899137496948, + "delta_nll": 0.5658794641494751, + "max_kl": 1.7832651138305664, + "mean_kl": 0.4811854064464569, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.9063208103179932, + "code_tokens": 57, + "cvar95_kl": 4.0191121101379395, + "delta_nll": 1.0507922172546387, + "max_kl": 5.517393589019775, + "mean_kl": 0.8299940228462219, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.913771629333496, + "code_tokens": 50, + "cvar95_kl": 3.6857547760009766, + "delta_nll": 0.8714720010757446, + "max_kl": 3.9767940044403076, + "mean_kl": 1.0066502094268799, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.6200000047683716 + }, + { + "candidate_nll": 1.771997332572937, + "code_tokens": 35, + "cvar95_kl": 3.831244468688965, + "delta_nll": 0.9434136152267456, + "max_kl": 3.856947898864746, + "mean_kl": 0.9803882241249084, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 2.775933265686035, + "code_tokens": 29, + "cvar95_kl": 3.468865394592285, + "delta_nll": 0.4315204620361328, + "max_kl": 4.713822841644287, + "mean_kl": 0.6695300936698914, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.5862069129943848 + }, + { + "candidate_nll": 2.14585542678833, + "code_tokens": 20, + "cvar95_kl": 2.4081037044525146, + "delta_nll": 0.8124374151229858, + "max_kl": 2.4081037044525146, + "mean_kl": 0.5416251420974731, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.8500000238418579 + }, + { + "candidate_nll": 1.506408452987671, + "code_tokens": 19, + "cvar95_kl": 1.9445544481277466, + "delta_nll": 0.38976550102233887, + "max_kl": 1.9445544481277466, + "mean_kl": 0.3775123655796051, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.8947368264198303 + }, + { + "candidate_nll": 2.0468297004699707, + "code_tokens": 71, + "cvar95_kl": 7.276410102844238, + "delta_nll": 1.10054612159729, + "max_kl": 8.970968246459961, + "mean_kl": 1.1704801321029663, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.6338028311729431 + }, + { + "candidate_nll": 2.611591339111328, + "code_tokens": 47, + "cvar95_kl": 5.950962543487549, + "delta_nll": 0.8766984939575195, + "max_kl": 7.460782527923584, + "mean_kl": 1.1716316938400269, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.7021276354789734 + }, + { + "candidate_nll": 1.6417804956436157, + "code_tokens": 50, + "cvar95_kl": 4.791374683380127, + "delta_nll": 0.839758038520813, + "max_kl": 6.7698259353637695, + "mean_kl": 0.7675440907478333, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.321031093597412, + "code_tokens": 113, + "cvar95_kl": 4.14279317855835, + "delta_nll": 1.2690701484680176, + "max_kl": 4.709221363067627, + "mean_kl": 1.1301106214523315, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.6194690465927124 + }, + { + "candidate_nll": 2.601745128631592, + "code_tokens": 26, + "cvar95_kl": 5.0964765548706055, + "delta_nll": 1.1615873575210571, + "max_kl": 5.340967655181885, + "mean_kl": 0.8116962909698486, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.255149245262146, + "code_tokens": 62, + "cvar95_kl": 5.216206073760986, + "delta_nll": 0.6868906021118164, + "max_kl": 10.479104042053223, + "mean_kl": 0.7311791777610779, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.8387096524238586 + }, + { + "candidate_nll": 1.4854358434677124, + "code_tokens": 79, + "cvar95_kl": 4.315554141998291, + "delta_nll": 0.7327173352241516, + "max_kl": 6.2377777099609375, + "mean_kl": 0.6723738312721252, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.7848101258277893 + }, + { + "candidate_nll": 1.862290620803833, + "code_tokens": 20, + "cvar95_kl": 1.6864235401153564, + "delta_nll": 0.2336418628692627, + "max_kl": 1.6864235401153564, + "mean_kl": 0.4521408975124359, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.1973932981491089, + "code_tokens": 186, + "cvar95_kl": 4.050142765045166, + "delta_nll": 0.7942488789558411, + "max_kl": 6.547865390777588, + "mean_kl": 0.8819687366485596, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.7043010592460632 + }, + { + "candidate_nll": 1.8554657697677612, + "code_tokens": 99, + "cvar95_kl": 5.436880111694336, + "delta_nll": 1.0123178958892822, + "max_kl": 6.03115701675415, + "mean_kl": 1.1155928373336792, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 1.2948133945465088, + "code_tokens": 43, + "cvar95_kl": 2.5436809062957764, + "delta_nll": 0.483005166053772, + "max_kl": 4.766280651092529, + "mean_kl": 0.4108973741531372, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.8372092843055725 + }, + { + "candidate_nll": 1.886618733406067, + "code_tokens": 35, + "cvar95_kl": 2.3529393672943115, + "delta_nll": 0.41282498836517334, + "max_kl": 3.4648971557617188, + "mean_kl": 0.44174471497535706, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.8571428656578064 + } + ], + "policy": { + "default_bits": 4, + "name": "mse_selected_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 1.4699668884277344, + "median": 0.7989711463451385, + "minimum": 0.08378326892852783, + "q25": 0.5183470547199249, + "q75": 0.9360556304454803 + }, + "task_macro": { + "candidate_nll": 1.8232128222783406, + "delta_nll": 0.7664886196454366, + "mean_kl": 0.7869029300080406, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.7123497055636512 + }, + "token_weighted": { + "candidate_nll": 1.8130292892456055, + "cvar95_kl": 4.917716026306152, + "delta_nll": 0.885370671749115, + "max_kl": 11.786860466003418, + "mean_kl": 0.9094873070716858, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.6833816170692444 + } + }, + "random_l13_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.523981939191404, + "macro_mean_kl": 2.6475038113801377, + "macro_top1_agreement": 0.3778522008139154, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 2.768257649048515, + "macro_mean_kl": 2.8711210333782695, + "macro_top1_agreement": 0.3655728898618532, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 3.1821901581504126, + "macro_mean_kl": 3.207676063884388, + "macro_top1_agreement": 0.30907866765152325, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 2.9602893699299204, + "macro_mean_kl": 3.034127479249781, + "macro_top1_agreement": 0.334279174154455, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 4.0014328956604, + "code_tokens": 63, + "cvar95_kl": 8.251551628112793, + "delta_nll": 3.410386085510254, + "max_kl": 8.637548446655273, + "mean_kl": 3.362363815307617, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.01297664642334, + "code_tokens": 92, + "cvar95_kl": 8.085668563842773, + "delta_nll": 3.0072529315948486, + "max_kl": 9.237343788146973, + "mean_kl": 2.855117082595825, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.928340911865234, + "code_tokens": 36, + "cvar95_kl": 7.587061882019043, + "delta_nll": 3.0148515701293945, + "max_kl": 8.352484703063965, + "mean_kl": 3.347895622253418, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.8548424243927, + "code_tokens": 24, + "cvar95_kl": 6.612549781799316, + "delta_nll": 2.037433385848999, + "max_kl": 6.61313009262085, + "mean_kl": 2.4631330966949463, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 4.224169731140137, + "code_tokens": 153, + "cvar95_kl": 8.14366626739502, + "delta_nll": 3.307218074798584, + "max_kl": 9.357083320617676, + "mean_kl": 3.4281792640686035, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.2679738700389862 + }, + { + "candidate_nll": 6.040244102478027, + "code_tokens": 133, + "cvar95_kl": 12.619207382202148, + "delta_nll": 4.973710536956787, + "max_kl": 17.399648666381836, + "mean_kl": 5.069884300231934, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.19548872113227844 + }, + { + "candidate_nll": 3.1856818199157715, + "code_tokens": 39, + "cvar95_kl": 7.1118879318237305, + "delta_nll": 2.3176488876342773, + "max_kl": 7.372398376464844, + "mean_kl": 2.394238233566284, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 3.589808225631714, + "code_tokens": 26, + "cvar95_kl": 7.229292869567871, + "delta_nll": 2.328852653503418, + "max_kl": 7.5444841384887695, + "mean_kl": 2.6271631717681885, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 4.230515956878662, + "code_tokens": 38, + "cvar95_kl": 10.027053833007812, + "delta_nll": 2.9148430824279785, + "max_kl": 10.549477577209473, + "mean_kl": 3.0166499614715576, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 4.628305912017822, + "code_tokens": 173, + "cvar95_kl": 10.785141944885254, + "delta_nll": 3.7745137214660645, + "max_kl": 14.533775329589844, + "mean_kl": 3.866896152496338, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.31213873624801636 + }, + { + "candidate_nll": 2.881598711013794, + "code_tokens": 37, + "cvar95_kl": 6.952797889709473, + "delta_nll": 1.9647760391235352, + "max_kl": 8.12847900390625, + "mean_kl": 2.109632968902588, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.4864864945411682 + }, + { + "candidate_nll": 3.2102785110473633, + "code_tokens": 205, + "cvar95_kl": 8.728716850280762, + "delta_nll": 2.3980884552001953, + "max_kl": 11.233299255371094, + "mean_kl": 2.5570380687713623, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.43414634466171265 + }, + { + "candidate_nll": 4.223400592803955, + "code_tokens": 162, + "cvar95_kl": 9.532209396362305, + "delta_nll": 3.324404716491699, + "max_kl": 11.8277006149292, + "mean_kl": 3.6478796005249023, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.32098764181137085 + }, + { + "candidate_nll": 3.074800968170166, + "code_tokens": 132, + "cvar95_kl": 6.618199348449707, + "delta_nll": 2.4124433994293213, + "max_kl": 7.699738025665283, + "mean_kl": 2.3867990970611572, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.46212121844291687 + }, + { + "candidate_nll": 2.1715736389160156, + "code_tokens": 33, + "cvar95_kl": 4.952328681945801, + "delta_nll": 1.5043799877166748, + "max_kl": 5.140102386474609, + "mean_kl": 1.6382328271865845, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.5151515007019043 + }, + { + "candidate_nll": 4.714090824127197, + "code_tokens": 60, + "cvar95_kl": 10.737584114074707, + "delta_nll": 3.3723561763763428, + "max_kl": 12.91954231262207, + "mean_kl": 3.4287638664245605, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.849351406097412, + "code_tokens": 63, + "cvar95_kl": 6.737327575683594, + "delta_nll": 3.4425458908081055, + "max_kl": 7.282605171203613, + "mean_kl": 3.4847254753112793, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.2063492089509964 + }, + { + "candidate_nll": 3.4458065032958984, + "code_tokens": 48, + "cvar95_kl": 5.3551788330078125, + "delta_nll": 1.8296756744384766, + "max_kl": 5.98471212387085, + "mean_kl": 2.37324595451355, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.935692071914673, + "code_tokens": 76, + "cvar95_kl": 6.532207489013672, + "delta_nll": 2.153007984161377, + "max_kl": 7.176009654998779, + "mean_kl": 2.1139304637908936, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.931753635406494, + "code_tokens": 63, + "cvar95_kl": 7.798240661621094, + "delta_nll": 2.980659008026123, + "max_kl": 9.915923118591309, + "mean_kl": 2.988330364227295, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.501583099365234, + "code_tokens": 107, + "cvar95_kl": 9.319769859313965, + "delta_nll": 3.6368489265441895, + "max_kl": 11.278051376342773, + "mean_kl": 3.750504732131958, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.21495327353477478 + }, + { + "candidate_nll": 3.3651561737060547, + "code_tokens": 108, + "cvar95_kl": 8.91918659210205, + "delta_nll": 2.749983549118042, + "max_kl": 10.046130180358887, + "mean_kl": 2.9141993522644043, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.37037035822868347 + }, + { + "candidate_nll": 4.465773105621338, + "code_tokens": 46, + "cvar95_kl": 9.34549617767334, + "delta_nll": 3.4924416542053223, + "max_kl": 10.790182113647461, + "mean_kl": 3.7745041847229004, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 4.46919584274292, + "code_tokens": 37, + "cvar95_kl": 8.743431091308594, + "delta_nll": 3.539937973022461, + "max_kl": 9.803023338317871, + "mean_kl": 3.731076717376709, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 3.758559226989746, + "code_tokens": 29, + "cvar95_kl": 6.066854953765869, + "delta_nll": 2.606403112411499, + "max_kl": 6.594526290893555, + "mean_kl": 2.828886032104492, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.3448275923728943 + }, + { + "candidate_nll": 3.845865488052368, + "code_tokens": 13, + "cvar95_kl": 5.527911186218262, + "delta_nll": 2.231736183166504, + "max_kl": 5.527911186218262, + "mean_kl": 2.2754855155944824, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.108590602874756, + "code_tokens": 51, + "cvar95_kl": 10.439778327941895, + "delta_nll": 3.4854986667633057, + "max_kl": 12.571571350097656, + "mean_kl": 3.2587289810180664, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.242371559143066, + "code_tokens": 29, + "cvar95_kl": 10.11370849609375, + "delta_nll": 3.1119465827941895, + "max_kl": 12.94186782836914, + "mean_kl": 3.094527244567871, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.009357929229736, + "code_tokens": 30, + "cvar95_kl": 8.696613311767578, + "delta_nll": 2.9797627925872803, + "max_kl": 8.776618003845215, + "mean_kl": 3.292167901992798, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.9267165660858154, + "code_tokens": 103, + "cvar95_kl": 8.294153213500977, + "delta_nll": 3.1359786987304688, + "max_kl": 10.233006477355957, + "mean_kl": 3.0440011024475098, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.26213592290878296 + }, + { + "candidate_nll": 3.704770088195801, + "code_tokens": 119, + "cvar95_kl": 8.722784996032715, + "delta_nll": 2.8042261600494385, + "max_kl": 10.385194778442383, + "mean_kl": 2.9434738159179688, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.3025210201740265 + }, + { + "candidate_nll": 5.215209007263184, + "code_tokens": 22, + "cvar95_kl": 11.108297348022461, + "delta_nll": 3.9183382987976074, + "max_kl": 11.60781192779541, + "mean_kl": 3.9047701358795166, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.5182719230651855, + "code_tokens": 51, + "cvar95_kl": 8.140857696533203, + "delta_nll": 2.8803939819335938, + "max_kl": 9.099769592285156, + "mean_kl": 2.945007085800171, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 3.697047233581543, + "code_tokens": 32, + "cvar95_kl": 6.403913497924805, + "delta_nll": 1.9374985694885254, + "max_kl": 6.5802130699157715, + "mean_kl": 2.2637979984283447, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.348736047744751, + "code_tokens": 116, + "cvar95_kl": 7.225802898406982, + "delta_nll": 2.285435199737549, + "max_kl": 9.466883659362793, + "mean_kl": 2.5584213733673096, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 4.895378589630127, + "code_tokens": 56, + "cvar95_kl": 8.236470222473145, + "delta_nll": 4.110193729400635, + "max_kl": 8.323480606079102, + "mean_kl": 3.8441383838653564, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 3.667224884033203, + "code_tokens": 53, + "cvar95_kl": 7.279455661773682, + "delta_nll": 2.719350814819336, + "max_kl": 8.805008888244629, + "mean_kl": 2.9116549491882324, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.30188679695129395 + }, + { + "candidate_nll": 2.777848720550537, + "code_tokens": 127, + "cvar95_kl": 6.660499572753906, + "delta_nll": 2.166627883911133, + "max_kl": 7.96805477142334, + "mean_kl": 2.2158710956573486, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.4724409580230713 + }, + { + "candidate_nll": 4.3212571144104, + "code_tokens": 61, + "cvar95_kl": 7.692111015319824, + "delta_nll": 3.108722686767578, + "max_kl": 8.83173942565918, + "mean_kl": 3.3077516555786133, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.26229506731033325 + }, + { + "candidate_nll": 2.972508430480957, + "code_tokens": 147, + "cvar95_kl": 7.957058429718018, + "delta_nll": 2.4265127182006836, + "max_kl": 13.002021789550781, + "mean_kl": 2.6187710762023926, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.378654956817627, + "code_tokens": 28, + "cvar95_kl": 7.046989440917969, + "delta_nll": 2.9006686210632324, + "max_kl": 7.206603050231934, + "mean_kl": 3.0986921787261963, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.3899879455566406, + "code_tokens": 66, + "cvar95_kl": 10.152843475341797, + "delta_nll": 2.5536179542541504, + "max_kl": 12.683954238891602, + "mean_kl": 2.5802462100982666, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 4.334560871124268, + "code_tokens": 33, + "cvar95_kl": 6.612712860107422, + "delta_nll": 2.507756233215332, + "max_kl": 6.891603946685791, + "mean_kl": 2.663567543029785, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 4.278843402862549, + "code_tokens": 41, + "cvar95_kl": 7.9475417137146, + "delta_nll": 3.141144275665283, + "max_kl": 10.184948921203613, + "mean_kl": 3.1884512901306152, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.24390244483947754 + }, + { + "candidate_nll": 3.8878183364868164, + "code_tokens": 36, + "cvar95_kl": 8.466215133666992, + "delta_nll": 2.2116405963897705, + "max_kl": 8.519674301147461, + "mean_kl": 2.2101969718933105, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 5.25413179397583, + "code_tokens": 70, + "cvar95_kl": 9.639850616455078, + "delta_nll": 4.336997032165527, + "max_kl": 10.549513816833496, + "mean_kl": 4.159963130950928, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.18571428954601288 + }, + { + "candidate_nll": 3.8966495990753174, + "code_tokens": 17, + "cvar95_kl": 9.264664649963379, + "delta_nll": 2.5971851348876953, + "max_kl": 9.264664649963379, + "mean_kl": 2.7347936630249023, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 4.133302211761475, + "code_tokens": 23, + "cvar95_kl": 7.061160087585449, + "delta_nll": 2.9480948448181152, + "max_kl": 7.570152759552002, + "mean_kl": 2.623774290084839, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 3.938676357269287, + "code_tokens": 101, + "cvar95_kl": 8.775199890136719, + "delta_nll": 3.560802698135376, + "max_kl": 9.5596342086792, + "mean_kl": 3.609135627746582, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.2772277295589447 + }, + { + "candidate_nll": 3.9133996963500977, + "code_tokens": 36, + "cvar95_kl": 6.5380353927612305, + "delta_nll": 2.498522996902466, + "max_kl": 6.9769206047058105, + "mean_kl": 2.655571222305298, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 4.901784896850586, + "code_tokens": 46, + "cvar95_kl": 9.580039024353027, + "delta_nll": 3.696031093597412, + "max_kl": 11.27694034576416, + "mean_kl": 3.82131290435791, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.28260868787765503 + }, + { + "candidate_nll": 3.754990816116333, + "code_tokens": 24, + "cvar95_kl": 6.546494960784912, + "delta_nll": 2.224144220352173, + "max_kl": 8.354422569274902, + "mean_kl": 2.1605818271636963, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 3.9186851978302, + "code_tokens": 22, + "cvar95_kl": 7.1200408935546875, + "delta_nll": 2.543355941772461, + "max_kl": 7.673630714416504, + "mean_kl": 2.936840295791626, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.930544853210449, + "code_tokens": 58, + "cvar95_kl": 6.9891486167907715, + "delta_nll": 3.2416722774505615, + "max_kl": 7.433948516845703, + "mean_kl": 3.177865743637085, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.2931034564971924 + }, + { + "candidate_nll": 4.102241039276123, + "code_tokens": 15, + "cvar95_kl": 6.997026443481445, + "delta_nll": 2.789370536804199, + "max_kl": 6.997026443481445, + "mean_kl": 2.7769691944122314, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 3.6888604164123535, + "code_tokens": 40, + "cvar95_kl": 8.34863567352295, + "delta_nll": 3.2492728233337402, + "max_kl": 8.671638488769531, + "mean_kl": 3.028557777404785, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 3.506209135055542, + "code_tokens": 37, + "cvar95_kl": 7.029004096984863, + "delta_nll": 2.9016594886779785, + "max_kl": 7.13161563873291, + "mean_kl": 2.8082969188690186, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.4324324429035187 + }, + { + "candidate_nll": 3.7418980598449707, + "code_tokens": 21, + "cvar95_kl": 8.413207054138184, + "delta_nll": 2.721652030944824, + "max_kl": 8.645007133483887, + "mean_kl": 3.0520172119140625, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 3.699713945388794, + "code_tokens": 21, + "cvar95_kl": 5.3067216873168945, + "delta_nll": 2.387197971343994, + "max_kl": 6.03485631942749, + "mean_kl": 2.2554309368133545, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 4.3210859298706055, + "code_tokens": 66, + "cvar95_kl": 9.73927116394043, + "delta_nll": 3.3127212524414062, + "max_kl": 11.960338592529297, + "mean_kl": 3.6889100074768066, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.24242424964904785 + }, + { + "candidate_nll": 2.5135810375213623, + "code_tokens": 138, + "cvar95_kl": 5.694031238555908, + "delta_nll": 1.869920253753662, + "max_kl": 6.071230888366699, + "mean_kl": 1.8617287874221802, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.5072463750839233 + }, + { + "candidate_nll": 2.9271812438964844, + "code_tokens": 22, + "cvar95_kl": 5.806823253631592, + "delta_nll": 2.1929385662078857, + "max_kl": 5.939021587371826, + "mean_kl": 2.239217758178711, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 4.087815761566162, + "code_tokens": 37, + "cvar95_kl": 6.983528137207031, + "delta_nll": 3.342231512069702, + "max_kl": 7.215757369995117, + "mean_kl": 3.0345892906188965, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.6902265548706055, + "code_tokens": 41, + "cvar95_kl": 7.7539753913879395, + "delta_nll": 2.739468812942505, + "max_kl": 9.440179824829102, + "mean_kl": 2.7441816329956055, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.3658536672592163 + }, + { + "candidate_nll": 4.680761337280273, + "code_tokens": 63, + "cvar95_kl": 7.932281494140625, + "delta_nll": 3.522270679473877, + "max_kl": 8.534872055053711, + "mean_kl": 3.577409029006958, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.421654224395752, + "code_tokens": 99, + "cvar95_kl": 6.280488014221191, + "delta_nll": 2.5749025344848633, + "max_kl": 6.718188762664795, + "mean_kl": 2.678363561630249, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.2626262605190277 + }, + { + "candidate_nll": 2.894335985183716, + "code_tokens": 65, + "cvar95_kl": 5.073101997375488, + "delta_nll": 1.9642422199249268, + "max_kl": 5.587606906890869, + "mean_kl": 2.054035186767578, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.4307692348957062 + }, + { + "candidate_nll": 3.9153997898101807, + "code_tokens": 39, + "cvar95_kl": 9.319923400878906, + "delta_nll": 2.997870683670044, + "max_kl": 9.374503135681152, + "mean_kl": 3.1657862663269043, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 3.5643134117126465, + "code_tokens": 40, + "cvar95_kl": 7.100107192993164, + "delta_nll": 2.405552625656128, + "max_kl": 7.378698825836182, + "mean_kl": 2.6848645210266113, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.170555591583252, + "code_tokens": 97, + "cvar95_kl": 8.325114250183105, + "delta_nll": 2.875603437423706, + "max_kl": 8.779688835144043, + "mean_kl": 2.9695825576782227, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.3298968970775604 + }, + { + "candidate_nll": 4.744788646697998, + "code_tokens": 23, + "cvar95_kl": 7.163745880126953, + "delta_nll": 2.8297598361968994, + "max_kl": 7.840836524963379, + "mean_kl": 3.032799005508423, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 2.763859272003174, + "code_tokens": 25, + "cvar95_kl": 5.0966291427612305, + "delta_nll": 1.7299896478652954, + "max_kl": 5.256781578063965, + "mean_kl": 1.7494726181030273, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.5600000023841858 + }, + { + "candidate_nll": 4.274447441101074, + "code_tokens": 57, + "cvar95_kl": 7.591884613037109, + "delta_nll": 3.4189188480377197, + "max_kl": 9.348488807678223, + "mean_kl": 2.984360694885254, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.345283031463623, + "code_tokens": 50, + "cvar95_kl": 10.873604774475098, + "delta_nll": 4.302983283996582, + "max_kl": 12.023853302001953, + "mean_kl": 4.507862091064453, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 4.611503601074219, + "code_tokens": 35, + "cvar95_kl": 10.089046478271484, + "delta_nll": 3.7829198837280273, + "max_kl": 12.222404479980469, + "mean_kl": 3.9497122764587402, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 4.312295436859131, + "code_tokens": 29, + "cvar95_kl": 5.601853370666504, + "delta_nll": 1.9678826332092285, + "max_kl": 6.354243278503418, + "mean_kl": 2.5828449726104736, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 3.798854112625122, + "code_tokens": 20, + "cvar95_kl": 7.498744010925293, + "delta_nll": 2.4654359817504883, + "max_kl": 7.498744010925293, + "mean_kl": 2.1545662879943848, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.32600998878479, + "code_tokens": 19, + "cvar95_kl": 5.864838600158691, + "delta_nll": 2.209367036819458, + "max_kl": 5.864838600158691, + "mean_kl": 2.3131606578826904, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 4.585622310638428, + "code_tokens": 71, + "cvar95_kl": 9.901905059814453, + "delta_nll": 3.639338731765747, + "max_kl": 10.528801918029785, + "mean_kl": 3.6890645027160645, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.26760563254356384 + }, + { + "candidate_nll": 4.671967029571533, + "code_tokens": 47, + "cvar95_kl": 9.889126777648926, + "delta_nll": 2.9370741844177246, + "max_kl": 11.056506156921387, + "mean_kl": 3.150179386138916, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.3617021143436432 + }, + { + "candidate_nll": 3.9211578369140625, + "code_tokens": 50, + "cvar95_kl": 8.81220531463623, + "delta_nll": 3.1191353797912598, + "max_kl": 10.41869831085205, + "mean_kl": 3.0652265548706055, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.0319085121154785, + "code_tokens": 113, + "cvar95_kl": 7.2763190269470215, + "delta_nll": 2.979947566986084, + "max_kl": 9.634774208068848, + "mean_kl": 2.852854013442993, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.32743361592292786 + }, + { + "candidate_nll": 4.294144630432129, + "code_tokens": 26, + "cvar95_kl": 6.985459327697754, + "delta_nll": 2.8539867401123047, + "max_kl": 7.9070892333984375, + "mean_kl": 2.92113995552063, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 3.6717538833618164, + "code_tokens": 62, + "cvar95_kl": 9.21405029296875, + "delta_nll": 3.1034951210021973, + "max_kl": 11.597027778625488, + "mean_kl": 3.065289258956909, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.4032258093357086 + }, + { + "candidate_nll": 3.7507710456848145, + "code_tokens": 79, + "cvar95_kl": 7.715055465698242, + "delta_nll": 2.9980525970458984, + "max_kl": 8.741482734680176, + "mean_kl": 2.913745641708374, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.26582279801368713 + }, + { + "candidate_nll": 3.1047306060791016, + "code_tokens": 20, + "cvar95_kl": 5.047817707061768, + "delta_nll": 1.4760818481445312, + "max_kl": 5.047817707061768, + "mean_kl": 1.774153709411621, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 2.92397403717041, + "code_tokens": 186, + "cvar95_kl": 7.393351078033447, + "delta_nll": 2.520829677581787, + "max_kl": 9.977835655212402, + "mean_kl": 2.5448453426361084, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.37634408473968506 + }, + { + "candidate_nll": 4.1862101554870605, + "code_tokens": 99, + "cvar95_kl": 8.814756393432617, + "delta_nll": 3.343062400817871, + "max_kl": 9.986809730529785, + "mean_kl": 3.463512897491455, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.35353535413742065 + }, + { + "candidate_nll": 3.990878105163574, + "code_tokens": 43, + "cvar95_kl": 7.061052322387695, + "delta_nll": 3.179069995880127, + "max_kl": 7.791849136352539, + "mean_kl": 3.3107683658599854, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 2.867126703262329, + "code_tokens": 35, + "cvar95_kl": 3.702925443649292, + "delta_nll": 1.3933329582214355, + "max_kl": 3.7882983684539795, + "mean_kl": 1.3437188863754272, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.5428571701049805 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l13_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 13 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 4.973710536956787, + "median": 2.890531301498413, + "minimum": 1.3933329582214355, + "q25": 2.3999544978141785, + "q75": 3.3113454580307007 + }, + "task_macro": { + "candidate_nll": 3.9106804211934407, + "delta_nll": 2.8539562238587273, + "mean_kl": 2.9360894375377233, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.34725166228082444 + }, + "token_weighted": { + "candidate_nll": 3.8670032024383545, + "cvar95_kl": 8.899869918823242, + "delta_nll": 2.939344644546509, + "max_kl": 17.399648666381836, + "mean_kl": 3.0172276496887207, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.33997103571891785 + } + }, + "random_l18_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.641707016074139, + "macro_mean_kl": 2.7284703928491343, + "macro_top1_agreement": 0.3467181668333385, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 2.854993757994279, + "macro_mean_kl": 2.9660153803618057, + "macro_top1_agreement": 0.3496245741844177, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 3.2779311700300737, + "macro_mean_kl": 3.303581649606878, + "macro_top1_agreement": 0.3006396849047054, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 3.0546410408886997, + "macro_mean_kl": 3.1166290749203074, + "macro_top1_agreement": 0.32958549802953546, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 4.005151748657227, + "code_tokens": 63, + "cvar95_kl": 7.954463481903076, + "delta_nll": 3.41410493850708, + "max_kl": 9.177360534667969, + "mean_kl": 3.378725051879883, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.264232635498047, + "code_tokens": 92, + "cvar95_kl": 8.675922393798828, + "delta_nll": 3.2585089206695557, + "max_kl": 10.53847599029541, + "mean_kl": 3.105114459991455, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.33695653080940247 + }, + { + "candidate_nll": 4.885852813720703, + "code_tokens": 36, + "cvar95_kl": 6.758638381958008, + "delta_nll": 2.9723634719848633, + "max_kl": 7.632549285888672, + "mean_kl": 3.33390212059021, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.8175230026245117, + "code_tokens": 24, + "cvar95_kl": 6.185256004333496, + "delta_nll": 2.0001139640808105, + "max_kl": 6.365230083465576, + "mean_kl": 2.471590995788574, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.238961219787598, + "code_tokens": 153, + "cvar95_kl": 8.314323425292969, + "delta_nll": 3.322009563446045, + "max_kl": 9.984254837036133, + "mean_kl": 3.477700710296631, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.2679738700389862 + }, + { + "candidate_nll": 6.236992835998535, + "code_tokens": 133, + "cvar95_kl": 13.20501708984375, + "delta_nll": 5.170459270477295, + "max_kl": 16.74625587463379, + "mean_kl": 5.201615810394287, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 3.4540202617645264, + "code_tokens": 39, + "cvar95_kl": 8.61186408996582, + "delta_nll": 2.5859873294830322, + "max_kl": 9.777901649475098, + "mean_kl": 2.5971872806549072, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 3.5842936038970947, + "code_tokens": 26, + "cvar95_kl": 6.4350690841674805, + "delta_nll": 2.323338031768799, + "max_kl": 6.4971418380737305, + "mean_kl": 2.5883982181549072, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.168341159820557, + "code_tokens": 38, + "cvar95_kl": 9.845409393310547, + "delta_nll": 2.852668285369873, + "max_kl": 10.026833534240723, + "mean_kl": 2.9765725135803223, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 4.625321865081787, + "code_tokens": 173, + "cvar95_kl": 10.42373275756836, + "delta_nll": 3.7715296745300293, + "max_kl": 13.403803825378418, + "mean_kl": 3.8637807369232178, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.2947976887226105 + }, + { + "candidate_nll": 3.071793556213379, + "code_tokens": 37, + "cvar95_kl": 7.032258987426758, + "delta_nll": 2.15497088432312, + "max_kl": 8.554535865783691, + "mean_kl": 2.29811692237854, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 3.390855073928833, + "code_tokens": 205, + "cvar95_kl": 9.15014934539795, + "delta_nll": 2.578665018081665, + "max_kl": 13.042898178100586, + "mean_kl": 2.70104718208313, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.3804877996444702 + }, + { + "candidate_nll": 4.3649773597717285, + "code_tokens": 162, + "cvar95_kl": 9.611539840698242, + "delta_nll": 3.4659814834594727, + "max_kl": 11.505186080932617, + "mean_kl": 3.770433187484741, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.34567901492118835 + }, + { + "candidate_nll": 3.1061298847198486, + "code_tokens": 132, + "cvar95_kl": 6.800552845001221, + "delta_nll": 2.443772315979004, + "max_kl": 7.325592517852783, + "mean_kl": 2.424379348754883, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.42424243688583374 + }, + { + "candidate_nll": 2.1438941955566406, + "code_tokens": 33, + "cvar95_kl": 4.510168552398682, + "delta_nll": 1.4767005443572998, + "max_kl": 4.691793441772461, + "mean_kl": 1.592424750328064, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.5151515007019043 + }, + { + "candidate_nll": 4.728078365325928, + "code_tokens": 60, + "cvar95_kl": 10.687092781066895, + "delta_nll": 3.3863437175750732, + "max_kl": 12.407981872558594, + "mean_kl": 3.4870615005493164, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.28333333134651184 + }, + { + "candidate_nll": 3.9496991634368896, + "code_tokens": 63, + "cvar95_kl": 6.728694915771484, + "delta_nll": 3.542893648147583, + "max_kl": 7.257243633270264, + "mean_kl": 3.5791866779327393, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 3.4445037841796875, + "code_tokens": 48, + "cvar95_kl": 5.908298015594482, + "delta_nll": 1.8283729553222656, + "max_kl": 6.198225498199463, + "mean_kl": 2.4228429794311523, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.3125 + }, + { + "candidate_nll": 3.1790425777435303, + "code_tokens": 76, + "cvar95_kl": 7.235908508300781, + "delta_nll": 2.3963584899902344, + "max_kl": 8.257939338684082, + "mean_kl": 2.2865636348724365, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.4868420958518982 + }, + { + "candidate_nll": 3.9344747066497803, + "code_tokens": 63, + "cvar95_kl": 7.66597843170166, + "delta_nll": 2.98337984085083, + "max_kl": 8.868157386779785, + "mean_kl": 2.965226888656616, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 4.636326789855957, + "code_tokens": 107, + "cvar95_kl": 9.70175552368164, + "delta_nll": 3.771592617034912, + "max_kl": 10.805517196655273, + "mean_kl": 3.924198627471924, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.21495327353477478 + }, + { + "candidate_nll": 3.4858431816101074, + "code_tokens": 108, + "cvar95_kl": 9.26733112335205, + "delta_nll": 2.8706705570220947, + "max_kl": 10.589630126953125, + "mean_kl": 3.0551915168762207, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.35185185074806213 + }, + { + "candidate_nll": 4.823246002197266, + "code_tokens": 46, + "cvar95_kl": 10.483067512512207, + "delta_nll": 3.84991455078125, + "max_kl": 11.291032791137695, + "mean_kl": 4.057025909423828, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 4.352104187011719, + "code_tokens": 37, + "cvar95_kl": 9.127447128295898, + "delta_nll": 3.4228463172912598, + "max_kl": 10.653403282165527, + "mean_kl": 3.651339054107666, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.840498208999634, + "code_tokens": 29, + "cvar95_kl": 5.3795881271362305, + "delta_nll": 2.6883420944213867, + "max_kl": 5.579110145568848, + "mean_kl": 2.8448832035064697, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.3448275923728943 + }, + { + "candidate_nll": 3.826472759246826, + "code_tokens": 13, + "cvar95_kl": 4.992818832397461, + "delta_nll": 2.212343215942383, + "max_kl": 4.992818832397461, + "mean_kl": 2.3076400756835938, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.004868030548096, + "code_tokens": 51, + "cvar95_kl": 9.594825744628906, + "delta_nll": 3.3817760944366455, + "max_kl": 10.634407997131348, + "mean_kl": 3.3020875453948975, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.985068321228027, + "code_tokens": 29, + "cvar95_kl": 11.032707214355469, + "delta_nll": 3.8546433448791504, + "max_kl": 13.911893844604492, + "mean_kl": 3.640986919403076, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 4.00562047958374, + "code_tokens": 30, + "cvar95_kl": 8.482833862304688, + "delta_nll": 2.976025342941284, + "max_kl": 8.54317569732666, + "mean_kl": 3.1826789379119873, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.099822044372559, + "code_tokens": 103, + "cvar95_kl": 9.054378509521484, + "delta_nll": 3.309083938598633, + "max_kl": 12.789204597473145, + "mean_kl": 3.2119579315185547, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.27184465527534485 + }, + { + "candidate_nll": 3.698732852935791, + "code_tokens": 119, + "cvar95_kl": 8.637478828430176, + "delta_nll": 2.7981889247894287, + "max_kl": 10.183870315551758, + "mean_kl": 2.8671226501464844, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.3025210201740265 + }, + { + "candidate_nll": 5.039762020111084, + "code_tokens": 22, + "cvar95_kl": 10.05474853515625, + "delta_nll": 3.742891311645508, + "max_kl": 10.16331672668457, + "mean_kl": 3.8154280185699463, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 3.629509925842285, + "code_tokens": 51, + "cvar95_kl": 8.296566009521484, + "delta_nll": 2.9916319847106934, + "max_kl": 9.405204772949219, + "mean_kl": 3.0259482860565186, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9084131717681885, + "code_tokens": 32, + "cvar95_kl": 7.360495567321777, + "delta_nll": 2.148864507675171, + "max_kl": 7.672651290893555, + "mean_kl": 2.539900541305542, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.40625 + }, + { + "candidate_nll": 3.19670033454895, + "code_tokens": 116, + "cvar95_kl": 7.235874176025391, + "delta_nll": 2.133399486541748, + "max_kl": 9.783172607421875, + "mean_kl": 2.407454013824463, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.3965517282485962 + }, + { + "candidate_nll": 5.091984272003174, + "code_tokens": 56, + "cvar95_kl": 7.686209201812744, + "delta_nll": 4.306799411773682, + "max_kl": 8.187712669372559, + "mean_kl": 3.96901273727417, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 3.671884536743164, + "code_tokens": 53, + "cvar95_kl": 6.647227764129639, + "delta_nll": 2.724010467529297, + "max_kl": 7.282204627990723, + "mean_kl": 2.852264404296875, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.30188679695129395 + }, + { + "candidate_nll": 2.8667824268341064, + "code_tokens": 127, + "cvar95_kl": 6.91283655166626, + "delta_nll": 2.255561590194702, + "max_kl": 7.473535537719727, + "mean_kl": 2.268324613571167, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.5039370059967041 + }, + { + "candidate_nll": 4.550559997558594, + "code_tokens": 61, + "cvar95_kl": 8.477123260498047, + "delta_nll": 3.3380255699157715, + "max_kl": 10.47312068939209, + "mean_kl": 3.5029284954071045, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.2786885201931 + }, + { + "candidate_nll": 2.9565579891204834, + "code_tokens": 147, + "cvar95_kl": 7.690785884857178, + "delta_nll": 2.41056227684021, + "max_kl": 9.207847595214844, + "mean_kl": 2.621450901031494, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.36054420471191406 + }, + { + "candidate_nll": 4.409075736999512, + "code_tokens": 28, + "cvar95_kl": 6.0275797843933105, + "delta_nll": 2.931089401245117, + "max_kl": 6.061062812805176, + "mean_kl": 3.092386245727539, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.574826717376709, + "code_tokens": 66, + "cvar95_kl": 9.61070442199707, + "delta_nll": 2.7384567260742188, + "max_kl": 11.834867477416992, + "mean_kl": 2.8289198875427246, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 4.244800090789795, + "code_tokens": 33, + "cvar95_kl": 6.896455764770508, + "delta_nll": 2.4179954528808594, + "max_kl": 7.336021423339844, + "mean_kl": 2.714496612548828, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 4.303705215454102, + "code_tokens": 41, + "cvar95_kl": 7.98281717300415, + "delta_nll": 3.166006088256836, + "max_kl": 9.918049812316895, + "mean_kl": 3.126479387283325, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.24390244483947754 + }, + { + "candidate_nll": 4.062335014343262, + "code_tokens": 36, + "cvar95_kl": 8.879289627075195, + "delta_nll": 2.386157274246216, + "max_kl": 8.901636123657227, + "mean_kl": 2.3358702659606934, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 5.420560836791992, + "code_tokens": 70, + "cvar95_kl": 10.571828842163086, + "delta_nll": 4.5034260749816895, + "max_kl": 11.863275527954102, + "mean_kl": 4.347377777099609, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.18571428954601288 + }, + { + "candidate_nll": 3.75815486907959, + "code_tokens": 17, + "cvar95_kl": 7.779269218444824, + "delta_nll": 2.4586904048919678, + "max_kl": 7.779269218444824, + "mean_kl": 2.540537118911743, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.29411765933036804 + }, + { + "candidate_nll": 4.285154342651367, + "code_tokens": 23, + "cvar95_kl": 6.849611282348633, + "delta_nll": 3.099946975708008, + "max_kl": 7.386782646179199, + "mean_kl": 2.705467462539673, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 3.929530382156372, + "code_tokens": 101, + "cvar95_kl": 9.1815767288208, + "delta_nll": 3.551656723022461, + "max_kl": 10.06834602355957, + "mean_kl": 3.6587095260620117, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.32673266530036926 + }, + { + "candidate_nll": 3.8784120082855225, + "code_tokens": 36, + "cvar95_kl": 7.43076229095459, + "delta_nll": 2.4635353088378906, + "max_kl": 7.578914165496826, + "mean_kl": 2.8453168869018555, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.3888888955116272 + }, + { + "candidate_nll": 5.026328086853027, + "code_tokens": 46, + "cvar95_kl": 9.923810005187988, + "delta_nll": 3.8205742835998535, + "max_kl": 11.17995548248291, + "mean_kl": 3.9141228199005127, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.28260868787765503 + }, + { + "candidate_nll": 3.7889792919158936, + "code_tokens": 24, + "cvar95_kl": 6.668489456176758, + "delta_nll": 2.2581326961517334, + "max_kl": 8.2333984375, + "mean_kl": 2.231490135192871, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.4166666567325592 + }, + { + "candidate_nll": 4.131546497344971, + "code_tokens": 22, + "cvar95_kl": 7.076000213623047, + "delta_nll": 2.7562172412872314, + "max_kl": 7.844458103179932, + "mean_kl": 3.1479623317718506, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.052419662475586, + "code_tokens": 58, + "cvar95_kl": 7.381044864654541, + "delta_nll": 3.3635470867156982, + "max_kl": 7.7256388664245605, + "mean_kl": 3.259134292602539, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.36206895112991333 + }, + { + "candidate_nll": 4.198153018951416, + "code_tokens": 15, + "cvar95_kl": 6.7000346183776855, + "delta_nll": 2.885282516479492, + "max_kl": 6.7000346183776855, + "mean_kl": 2.8210508823394775, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.746147871017456, + "code_tokens": 40, + "cvar95_kl": 8.106391906738281, + "delta_nll": 3.3065602779388428, + "max_kl": 8.221272468566895, + "mean_kl": 3.102933406829834, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.6216893196105957, + "code_tokens": 37, + "cvar95_kl": 7.452920913696289, + "delta_nll": 3.0171396732330322, + "max_kl": 7.933398723602295, + "mean_kl": 2.961198091506958, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.791175127029419, + "code_tokens": 21, + "cvar95_kl": 8.361068725585938, + "delta_nll": 2.7709293365478516, + "max_kl": 8.536450386047363, + "mean_kl": 3.0884053707122803, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.523809552192688 + }, + { + "candidate_nll": 3.723337173461914, + "code_tokens": 21, + "cvar95_kl": 4.995732307434082, + "delta_nll": 2.4108214378356934, + "max_kl": 5.018786430358887, + "mean_kl": 2.1784117221832275, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.313911437988281, + "code_tokens": 66, + "cvar95_kl": 9.6868314743042, + "delta_nll": 3.305546760559082, + "max_kl": 12.043803215026855, + "mean_kl": 3.69488263130188, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.28787878155708313 + }, + { + "candidate_nll": 2.69309663772583, + "code_tokens": 138, + "cvar95_kl": 5.764339923858643, + "delta_nll": 2.04943585395813, + "max_kl": 7.54143762588501, + "mean_kl": 1.9886363744735718, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.4492753744125366 + }, + { + "candidate_nll": 3.1775460243225098, + "code_tokens": 22, + "cvar95_kl": 7.479392051696777, + "delta_nll": 2.443303346633911, + "max_kl": 7.690463542938232, + "mean_kl": 2.467205762863159, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 4.36911153793335, + "code_tokens": 37, + "cvar95_kl": 8.19649887084961, + "delta_nll": 3.6235272884368896, + "max_kl": 8.282401084899902, + "mean_kl": 3.295907497406006, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.6810848712921143, + "code_tokens": 41, + "cvar95_kl": 6.676408290863037, + "delta_nll": 2.7303271293640137, + "max_kl": 8.196612358093262, + "mean_kl": 2.6587765216827393, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.3658536672592163 + }, + { + "candidate_nll": 4.660767555236816, + "code_tokens": 63, + "cvar95_kl": 7.992784023284912, + "delta_nll": 3.50227689743042, + "max_kl": 8.16236400604248, + "mean_kl": 3.6311123371124268, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 3.508429527282715, + "code_tokens": 99, + "cvar95_kl": 6.61444091796875, + "delta_nll": 2.661677837371826, + "max_kl": 7.090278148651123, + "mean_kl": 2.7080235481262207, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 2.947138786315918, + "code_tokens": 65, + "cvar95_kl": 4.752860069274902, + "delta_nll": 2.017045021057129, + "max_kl": 5.525080680847168, + "mean_kl": 2.091765880584717, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.4153846204280853 + }, + { + "candidate_nll": 4.070461750030518, + "code_tokens": 39, + "cvar95_kl": 9.05270004272461, + "delta_nll": 3.152932643890381, + "max_kl": 9.19739055633545, + "mean_kl": 3.331286668777466, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 3.4052066802978516, + "code_tokens": 40, + "cvar95_kl": 7.024418354034424, + "delta_nll": 2.246445894241333, + "max_kl": 7.078471660614014, + "mean_kl": 2.634132146835327, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.269992828369141, + "code_tokens": 97, + "cvar95_kl": 8.042820930480957, + "delta_nll": 2.9750406742095947, + "max_kl": 8.589386940002441, + "mean_kl": 3.108980178833008, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.2886597812175751 + }, + { + "candidate_nll": 4.835409164428711, + "code_tokens": 23, + "cvar95_kl": 7.13053035736084, + "delta_nll": 2.9203803539276123, + "max_kl": 7.905058860778809, + "mean_kl": 3.14805269241333, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 3.106496810913086, + "code_tokens": 25, + "cvar95_kl": 5.126241683959961, + "delta_nll": 2.072627067565918, + "max_kl": 5.534509181976318, + "mean_kl": 1.9974502325057983, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.716454029083252, + "code_tokens": 57, + "cvar95_kl": 7.965877532958984, + "delta_nll": 3.8609254360198975, + "max_kl": 9.531194686889648, + "mean_kl": 3.3645288944244385, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 5.463010787963867, + "code_tokens": 50, + "cvar95_kl": 11.723233222961426, + "delta_nll": 4.420711040496826, + "max_kl": 13.295049667358398, + "mean_kl": 4.6639299392700195, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.7459845542907715, + "code_tokens": 35, + "cvar95_kl": 10.142020225524902, + "delta_nll": 3.91740083694458, + "max_kl": 11.724349021911621, + "mean_kl": 4.071507930755615, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.472626209259033, + "code_tokens": 29, + "cvar95_kl": 6.4032392501831055, + "delta_nll": 2.128213405609131, + "max_kl": 6.715555667877197, + "mean_kl": 2.712581157684326, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 3.9944138526916504, + "code_tokens": 20, + "cvar95_kl": 5.6532182693481445, + "delta_nll": 2.6609959602355957, + "max_kl": 5.6532182693481445, + "mean_kl": 2.42169189453125, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 3.7215077877044678, + "code_tokens": 19, + "cvar95_kl": 6.15032434463501, + "delta_nll": 2.6048648357391357, + "max_kl": 6.15032434463501, + "mean_kl": 2.600567579269409, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 4.596343040466309, + "code_tokens": 71, + "cvar95_kl": 8.998541831970215, + "delta_nll": 3.650059461593628, + "max_kl": 9.46657657623291, + "mean_kl": 3.669914960861206, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.2535211145877838 + }, + { + "candidate_nll": 5.02692174911499, + "code_tokens": 47, + "cvar95_kl": 9.982539176940918, + "delta_nll": 3.2920289039611816, + "max_kl": 10.66815185546875, + "mean_kl": 3.4332313537597656, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.3404255211353302 + }, + { + "candidate_nll": 4.093407154083252, + "code_tokens": 50, + "cvar95_kl": 8.516829490661621, + "delta_nll": 3.291384696960449, + "max_kl": 9.146936416625977, + "mean_kl": 3.2351837158203125, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 4.235611438751221, + "code_tokens": 113, + "cvar95_kl": 7.80162239074707, + "delta_nll": 3.183650493621826, + "max_kl": 9.15034008026123, + "mean_kl": 2.979755163192749, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.3451327383518219 + }, + { + "candidate_nll": 4.45747184753418, + "code_tokens": 26, + "cvar95_kl": 7.062280654907227, + "delta_nll": 3.0173139572143555, + "max_kl": 7.782673358917236, + "mean_kl": 3.003892421722412, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.3461538553237915 + }, + { + "candidate_nll": 3.7356679439544678, + "code_tokens": 62, + "cvar95_kl": 8.978543281555176, + "delta_nll": 3.1674094200134277, + "max_kl": 10.75926399230957, + "mean_kl": 3.1201977729797363, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 3.9478647708892822, + "code_tokens": 79, + "cvar95_kl": 6.984984397888184, + "delta_nll": 3.195146322250366, + "max_kl": 7.375044345855713, + "mean_kl": 3.1004698276519775, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.2531645596027374 + }, + { + "candidate_nll": 3.1714038848876953, + "code_tokens": 20, + "cvar95_kl": 4.542719841003418, + "delta_nll": 1.542755126953125, + "max_kl": 4.542719841003418, + "mean_kl": 1.7460596561431885, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.0677952766418457, + "code_tokens": 186, + "cvar95_kl": 7.707716464996338, + "delta_nll": 2.6646509170532227, + "max_kl": 10.125898361206055, + "mean_kl": 2.6993465423583984, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.3817204236984253 + }, + { + "candidate_nll": 4.204006195068359, + "code_tokens": 99, + "cvar95_kl": 8.401692390441895, + "delta_nll": 3.36085844039917, + "max_kl": 8.812019348144531, + "mean_kl": 3.422146797180176, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.3232323229312897 + }, + { + "candidate_nll": 4.040437698364258, + "code_tokens": 43, + "cvar95_kl": 6.967243194580078, + "delta_nll": 3.2286295890808105, + "max_kl": 7.672679424285889, + "mean_kl": 3.309424877166748, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 2.905073642730713, + "code_tokens": 35, + "cvar95_kl": 4.08913516998291, + "delta_nll": 1.4312798976898193, + "max_kl": 4.123841285705566, + "mean_kl": 1.4372001886367798, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.48571428656578064 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l18_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 18 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 5.170459270477295, + "median": 2.973702073097229, + "minimum": 1.4312798976898193, + "q25": 2.4434205889701843, + "q75": 3.362874925136566 + }, + "task_macro": { + "candidate_nll": 4.009398712052239, + "delta_nll": 2.9526745160420735, + "mean_kl": 3.024642319149441, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.3320093007551299 + }, + "token_weighted": { + "candidate_nll": 3.9642653465270996, + "cvar95_kl": 9.03290843963623, + "delta_nll": 3.036606788635254, + "max_kl": 16.74625587463379, + "mean_kl": 3.1047608852386475, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.32965242862701416 + } + }, + "random_l4_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.4215252192124077, + "macro_mean_kl": 2.522120766017748, + "macro_top1_agreement": 0.38480358020119043, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 2.6151484380597654, + "macro_mean_kl": 2.7683726134507554, + "macro_top1_agreement": 0.3658274755529735, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 3.043630426580256, + "macro_mean_kl": 3.0864297368309717, + "macro_top1_agreement": 0.32594900171865115, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 2.8507063388824463, + "macro_mean_kl": 2.9217781370336358, + "macro_top1_agreement": 0.3501553372903304, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 3.697962522506714, + "code_tokens": 63, + "cvar95_kl": 8.268806457519531, + "delta_nll": 3.1069157123565674, + "max_kl": 8.786489486694336, + "mean_kl": 3.1522433757781982, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.30158731341362 + }, + { + "candidate_nll": 3.8818089962005615, + "code_tokens": 92, + "cvar95_kl": 8.200223922729492, + "delta_nll": 2.8760852813720703, + "max_kl": 12.013046264648438, + "mean_kl": 2.758474826812744, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.3586956560611725 + }, + { + "candidate_nll": 4.3893513679504395, + "code_tokens": 36, + "cvar95_kl": 6.050378322601318, + "delta_nll": 2.4758620262145996, + "max_kl": 6.075777530670166, + "mean_kl": 2.718343734741211, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.2777777910232544 + }, + { + "candidate_nll": 3.4056074619293213, + "code_tokens": 24, + "cvar95_kl": 6.004136085510254, + "delta_nll": 1.5881984233856201, + "max_kl": 6.2719926834106445, + "mean_kl": 2.156606435775757, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 4.163101673126221, + "code_tokens": 153, + "cvar95_kl": 8.402976989746094, + "delta_nll": 3.246150016784668, + "max_kl": 11.444881439208984, + "mean_kl": 3.4502265453338623, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.2875817120075226 + }, + { + "candidate_nll": 5.917141914367676, + "code_tokens": 133, + "cvar95_kl": 12.343687057495117, + "delta_nll": 4.8506083488464355, + "max_kl": 16.917163848876953, + "mean_kl": 5.019104480743408, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.18796992301940918 + }, + { + "candidate_nll": 3.480668306350708, + "code_tokens": 39, + "cvar95_kl": 6.980378150939941, + "delta_nll": 2.612635374069214, + "max_kl": 7.151417255401611, + "mean_kl": 2.629307270050049, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.3589743673801422 + }, + { + "candidate_nll": 2.7650046348571777, + "code_tokens": 26, + "cvar95_kl": 5.451140403747559, + "delta_nll": 1.5040489435195923, + "max_kl": 6.124221324920654, + "mean_kl": 1.774647831916809, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 4.458333492279053, + "code_tokens": 38, + "cvar95_kl": 10.491903305053711, + "delta_nll": 3.142660617828369, + "max_kl": 10.530671119689941, + "mean_kl": 3.1844642162323, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 4.209057331085205, + "code_tokens": 173, + "cvar95_kl": 9.686699867248535, + "delta_nll": 3.3552651405334473, + "max_kl": 13.725617408752441, + "mean_kl": 3.3974108695983887, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.3583815097808838 + }, + { + "candidate_nll": 2.7377755641937256, + "code_tokens": 37, + "cvar95_kl": 8.133733749389648, + "delta_nll": 1.8209528923034668, + "max_kl": 10.812057495117188, + "mean_kl": 1.975081205368042, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.4054054021835327 + }, + { + "candidate_nll": 3.1254591941833496, + "code_tokens": 205, + "cvar95_kl": 8.414073944091797, + "delta_nll": 2.3132691383361816, + "max_kl": 10.213601112365723, + "mean_kl": 2.4278552532196045, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.4292683005332947 + }, + { + "candidate_nll": 4.1708269119262695, + "code_tokens": 162, + "cvar95_kl": 10.160523414611816, + "delta_nll": 3.2718310356140137, + "max_kl": 12.04229736328125, + "mean_kl": 3.579373598098755, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.3024691343307495 + }, + { + "candidate_nll": 2.956092596054077, + "code_tokens": 132, + "cvar95_kl": 6.858893871307373, + "delta_nll": 2.2937350273132324, + "max_kl": 8.177632331848145, + "mean_kl": 2.349871873855591, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.43939393758773804 + }, + { + "candidate_nll": 1.6626943349838257, + "code_tokens": 33, + "cvar95_kl": 2.8994269371032715, + "delta_nll": 0.9955006241798401, + "max_kl": 3.221698760986328, + "mean_kl": 1.1687219142913818, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.6060606241226196 + }, + { + "candidate_nll": 4.671377658843994, + "code_tokens": 60, + "cvar95_kl": 11.078923225402832, + "delta_nll": 3.3296430110931396, + "max_kl": 14.185201644897461, + "mean_kl": 3.435004234313965, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.8637301921844482, + "code_tokens": 63, + "cvar95_kl": 7.6096649169921875, + "delta_nll": 3.4569246768951416, + "max_kl": 9.668323516845703, + "mean_kl": 3.429563522338867, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.2698412835597992 + }, + { + "candidate_nll": 3.0435428619384766, + "code_tokens": 48, + "cvar95_kl": 5.704840183258057, + "delta_nll": 1.4274120330810547, + "max_kl": 6.268909454345703, + "mean_kl": 2.1296274662017822, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.3541666567325592 + }, + { + "candidate_nll": 2.9133572578430176, + "code_tokens": 76, + "cvar95_kl": 6.110639572143555, + "delta_nll": 2.1306731700897217, + "max_kl": 6.620903968811035, + "mean_kl": 2.0929172039031982, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.46052631735801697 + }, + { + "candidate_nll": 3.6327054500579834, + "code_tokens": 63, + "cvar95_kl": 7.161320209503174, + "delta_nll": 2.681610584259033, + "max_kl": 8.370038032531738, + "mean_kl": 2.597550392150879, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.3650793731212616 + }, + { + "candidate_nll": 4.446523189544678, + "code_tokens": 107, + "cvar95_kl": 9.403797149658203, + "delta_nll": 3.581789016723633, + "max_kl": 11.951190948486328, + "mean_kl": 3.6053929328918457, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.21495327353477478 + }, + { + "candidate_nll": 3.549755573272705, + "code_tokens": 108, + "cvar95_kl": 8.002754211425781, + "delta_nll": 2.9345829486846924, + "max_kl": 8.951117515563965, + "mean_kl": 3.1028614044189453, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.3055555522441864 + }, + { + "candidate_nll": 4.1172871589660645, + "code_tokens": 46, + "cvar95_kl": 8.667256355285645, + "delta_nll": 3.143955707550049, + "max_kl": 11.205277442932129, + "mean_kl": 3.302581787109375, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 3.73848032951355, + "code_tokens": 37, + "cvar95_kl": 8.917360305786133, + "delta_nll": 2.8092222213745117, + "max_kl": 10.175028800964355, + "mean_kl": 3.0325968265533447, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.3243243098258972 + }, + { + "candidate_nll": 3.8052852153778076, + "code_tokens": 29, + "cvar95_kl": 7.255312442779541, + "delta_nll": 2.6531291007995605, + "max_kl": 7.981550693511963, + "mean_kl": 2.7044785022735596, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 4.182766437530518, + "code_tokens": 13, + "cvar95_kl": 5.566764831542969, + "delta_nll": 2.568636894226074, + "max_kl": 5.566764831542969, + "mean_kl": 2.3950605392456055, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 4.281055927276611, + "code_tokens": 51, + "cvar95_kl": 9.780045509338379, + "delta_nll": 3.657963991165161, + "max_kl": 11.079684257507324, + "mean_kl": 3.5019311904907227, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 3.9918103218078613, + "code_tokens": 29, + "cvar95_kl": 10.508709907531738, + "delta_nll": 2.8613853454589844, + "max_kl": 14.217609405517578, + "mean_kl": 2.712496042251587, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.4482758641242981 + }, + { + "candidate_nll": 3.795807361602783, + "code_tokens": 30, + "cvar95_kl": 8.182945251464844, + "delta_nll": 2.766212224960327, + "max_kl": 8.217219352722168, + "mean_kl": 3.040471076965332, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.36666667461395264 + }, + { + "candidate_nll": 3.71636700630188, + "code_tokens": 103, + "cvar95_kl": 8.36789608001709, + "delta_nll": 2.925629138946533, + "max_kl": 9.470518112182617, + "mean_kl": 2.768789291381836, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.35922330617904663 + }, + { + "candidate_nll": 3.649885416030884, + "code_tokens": 119, + "cvar95_kl": 8.7811861038208, + "delta_nll": 2.7493414878845215, + "max_kl": 11.061680793762207, + "mean_kl": 2.821239709854126, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.3361344635486603 + }, + { + "candidate_nll": 4.724217414855957, + "code_tokens": 22, + "cvar95_kl": 10.132523536682129, + "delta_nll": 3.427346706390381, + "max_kl": 10.828780174255371, + "mean_kl": 3.6297523975372314, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 3.234102725982666, + "code_tokens": 51, + "cvar95_kl": 7.854490280151367, + "delta_nll": 2.596224784851074, + "max_kl": 8.595439910888672, + "mean_kl": 2.6509766578674316, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 3.614658832550049, + "code_tokens": 32, + "cvar95_kl": 6.3545331954956055, + "delta_nll": 1.8551101684570312, + "max_kl": 6.844962120056152, + "mean_kl": 2.2415356636047363, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.46875 + }, + { + "candidate_nll": 3.0880966186523438, + "code_tokens": 116, + "cvar95_kl": 6.620830535888672, + "delta_nll": 2.0247960090637207, + "max_kl": 7.520521640777588, + "mean_kl": 2.274684190750122, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.43103447556495667 + }, + { + "candidate_nll": 4.227554798126221, + "code_tokens": 56, + "cvar95_kl": 7.640956878662109, + "delta_nll": 3.4423699378967285, + "max_kl": 8.56278133392334, + "mean_kl": 3.2516233921051025, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.3035714328289032 + }, + { + "candidate_nll": 3.570824146270752, + "code_tokens": 53, + "cvar95_kl": 7.049681186676025, + "delta_nll": 2.6229500770568848, + "max_kl": 8.067105293273926, + "mean_kl": 2.6916561126708984, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.3207547068595886 + }, + { + "candidate_nll": 2.8575150966644287, + "code_tokens": 127, + "cvar95_kl": 6.769360065460205, + "delta_nll": 2.2462942600250244, + "max_kl": 7.381105899810791, + "mean_kl": 2.30710506439209, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.4251968562602997 + }, + { + "candidate_nll": 4.14841365814209, + "code_tokens": 61, + "cvar95_kl": 9.10840129852295, + "delta_nll": 2.9358792304992676, + "max_kl": 12.245336532592773, + "mean_kl": 3.311856985092163, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.24590164422988892 + }, + { + "candidate_nll": 2.7802045345306396, + "code_tokens": 147, + "cvar95_kl": 8.02630615234375, + "delta_nll": 2.234208822250366, + "max_kl": 12.661044120788574, + "mean_kl": 2.3771650791168213, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.40816327929496765 + }, + { + "candidate_nll": 4.02643346786499, + "code_tokens": 28, + "cvar95_kl": 6.808594703674316, + "delta_nll": 2.5484471321105957, + "max_kl": 6.980624675750732, + "mean_kl": 2.943204164505005, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.3214285671710968 + }, + { + "candidate_nll": 3.200775623321533, + "code_tokens": 66, + "cvar95_kl": 9.116652488708496, + "delta_nll": 2.364405632019043, + "max_kl": 10.341926574707031, + "mean_kl": 2.5066845417022705, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.42424243688583374 + }, + { + "candidate_nll": 3.763701915740967, + "code_tokens": 33, + "cvar95_kl": 6.258485317230225, + "delta_nll": 1.9368971586227417, + "max_kl": 7.079742908477783, + "mean_kl": 2.4266905784606934, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.39393940567970276 + }, + { + "candidate_nll": 4.178773880004883, + "code_tokens": 41, + "cvar95_kl": 8.03918170928955, + "delta_nll": 3.041074752807617, + "max_kl": 10.933012962341309, + "mean_kl": 2.920809030532837, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.26829269528388977 + }, + { + "candidate_nll": 3.9070358276367188, + "code_tokens": 36, + "cvar95_kl": 7.654692649841309, + "delta_nll": 2.230858087539673, + "max_kl": 7.7864990234375, + "mean_kl": 2.268319845199585, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.4444444477558136 + }, + { + "candidate_nll": 4.9901628494262695, + "code_tokens": 70, + "cvar95_kl": 8.357059478759766, + "delta_nll": 4.073028087615967, + "max_kl": 9.38172435760498, + "mean_kl": 4.011991024017334, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 3.1423420906066895, + "code_tokens": 17, + "cvar95_kl": 8.031208992004395, + "delta_nll": 1.8428776264190674, + "max_kl": 8.031208992004395, + "mean_kl": 1.958680272102356, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.47058823704719543 + }, + { + "candidate_nll": 4.470083236694336, + "code_tokens": 23, + "cvar95_kl": 7.4791741371154785, + "delta_nll": 3.2848758697509766, + "max_kl": 8.735645294189453, + "mean_kl": 2.9010298252105713, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.3478260934352875 + }, + { + "candidate_nll": 3.7634918689727783, + "code_tokens": 101, + "cvar95_kl": 9.647727966308594, + "delta_nll": 3.385618209838867, + "max_kl": 13.03792953491211, + "mean_kl": 3.4400994777679443, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.3465346395969391 + }, + { + "candidate_nll": 3.735504150390625, + "code_tokens": 36, + "cvar95_kl": 7.105195045471191, + "delta_nll": 2.320627450942993, + "max_kl": 7.184623718261719, + "mean_kl": 2.5888848304748535, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 4.980473518371582, + "code_tokens": 46, + "cvar95_kl": 11.550544738769531, + "delta_nll": 3.774719715118408, + "max_kl": 13.3289213180542, + "mean_kl": 3.951132297515869, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 3.2688636779785156, + "code_tokens": 24, + "cvar95_kl": 6.461575508117676, + "delta_nll": 1.7380170822143555, + "max_kl": 8.430036544799805, + "mean_kl": 1.7850607633590698, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.5416666865348816 + }, + { + "candidate_nll": 4.010259628295898, + "code_tokens": 22, + "cvar95_kl": 7.13347864151001, + "delta_nll": 2.634930372238159, + "max_kl": 7.7957305908203125, + "mean_kl": 2.971846103668213, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 3.749924421310425, + "code_tokens": 58, + "cvar95_kl": 7.565622806549072, + "delta_nll": 3.061051845550537, + "max_kl": 8.179715156555176, + "mean_kl": 2.9905312061309814, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.36206895112991333 + }, + { + "candidate_nll": 3.6460466384887695, + "code_tokens": 15, + "cvar95_kl": 5.250370025634766, + "delta_nll": 2.3331761360168457, + "max_kl": 5.250370025634766, + "mean_kl": 2.2857918739318848, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.46666666865348816 + }, + { + "candidate_nll": 3.42097544670105, + "code_tokens": 40, + "cvar95_kl": 9.171025276184082, + "delta_nll": 2.9813878536224365, + "max_kl": 10.795073509216309, + "mean_kl": 2.9622931480407715, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.483046293258667, + "code_tokens": 37, + "cvar95_kl": 7.891551971435547, + "delta_nll": 2.8784966468811035, + "max_kl": 8.350440979003906, + "mean_kl": 2.8231513500213623, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.37837839126586914 + }, + { + "candidate_nll": 3.2573423385620117, + "code_tokens": 21, + "cvar95_kl": 8.519705772399902, + "delta_nll": 2.2370963096618652, + "max_kl": 9.252758026123047, + "mean_kl": 2.262068510055542, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.523809552192688 + }, + { + "candidate_nll": 3.822953462600708, + "code_tokens": 21, + "cvar95_kl": 5.46238899230957, + "delta_nll": 2.510437488555908, + "max_kl": 5.796491622924805, + "mean_kl": 2.275078058242798, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.380952388048172 + }, + { + "candidate_nll": 5.166333198547363, + "code_tokens": 66, + "cvar95_kl": 9.754901885986328, + "delta_nll": 4.157968521118164, + "max_kl": 10.717887878417969, + "mean_kl": 4.371059417724609, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 2.560255765914917, + "code_tokens": 138, + "cvar95_kl": 5.558399200439453, + "delta_nll": 1.9165949821472168, + "max_kl": 6.572728633880615, + "mean_kl": 1.8730778694152832, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.4420289993286133 + }, + { + "candidate_nll": 3.1922807693481445, + "code_tokens": 22, + "cvar95_kl": 7.961118698120117, + "delta_nll": 2.458038091659546, + "max_kl": 9.595173835754395, + "mean_kl": 2.4657483100891113, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 4.076706886291504, + "code_tokens": 37, + "cvar95_kl": 7.585383892059326, + "delta_nll": 3.331122636795044, + "max_kl": 7.961958885192871, + "mean_kl": 3.1960701942443848, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 3.418212413787842, + "code_tokens": 41, + "cvar95_kl": 6.278361797332764, + "delta_nll": 2.467454671859741, + "max_kl": 6.995954513549805, + "mean_kl": 2.5837364196777344, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.3414634168148041 + }, + { + "candidate_nll": 4.389913558959961, + "code_tokens": 63, + "cvar95_kl": 6.7890143394470215, + "delta_nll": 3.2314229011535645, + "max_kl": 6.970921039581299, + "mean_kl": 3.151583671569824, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.1110944747924805, + "code_tokens": 99, + "cvar95_kl": 7.078920841217041, + "delta_nll": 2.264342784881592, + "max_kl": 7.526179790496826, + "mean_kl": 2.477461814880371, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 2.576974868774414, + "code_tokens": 65, + "cvar95_kl": 4.946934700012207, + "delta_nll": 1.646881103515625, + "max_kl": 5.733255863189697, + "mean_kl": 1.7378549575805664, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.4769230782985687 + }, + { + "candidate_nll": 3.4752888679504395, + "code_tokens": 39, + "cvar95_kl": 10.32791519165039, + "delta_nll": 2.5577597618103027, + "max_kl": 10.809345245361328, + "mean_kl": 3.0320489406585693, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.43589743971824646 + }, + { + "candidate_nll": 3.633431911468506, + "code_tokens": 40, + "cvar95_kl": 7.772727966308594, + "delta_nll": 2.4746711254119873, + "max_kl": 8.252426147460938, + "mean_kl": 2.748398780822754, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.32499998807907104 + }, + { + "candidate_nll": 4.3469085693359375, + "code_tokens": 97, + "cvar95_kl": 8.136125564575195, + "delta_nll": 3.0519564151763916, + "max_kl": 11.335161209106445, + "mean_kl": 3.0809473991394043, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.2989690601825714 + }, + { + "candidate_nll": 4.6397857666015625, + "code_tokens": 23, + "cvar95_kl": 7.660138130187988, + "delta_nll": 2.724756956100464, + "max_kl": 7.767531394958496, + "mean_kl": 2.9769718647003174, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 2.9211981296539307, + "code_tokens": 25, + "cvar95_kl": 5.229940891265869, + "delta_nll": 1.8873285055160522, + "max_kl": 5.43509578704834, + "mean_kl": 1.7526065111160278, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.5199999809265137 + }, + { + "candidate_nll": 4.015822887420654, + "code_tokens": 57, + "cvar95_kl": 8.112438201904297, + "delta_nll": 3.1602942943573, + "max_kl": 9.893936157226562, + "mean_kl": 2.7752768993377686, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 5.156582832336426, + "code_tokens": 50, + "cvar95_kl": 11.486775398254395, + "delta_nll": 4.114283084869385, + "max_kl": 13.105146408081055, + "mean_kl": 4.365245819091797, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.2199999988079071 + }, + { + "candidate_nll": 4.618897914886475, + "code_tokens": 35, + "cvar95_kl": 9.599714279174805, + "delta_nll": 3.790314197540283, + "max_kl": 10.145642280578613, + "mean_kl": 3.9593095779418945, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.34285715222358704 + }, + { + "candidate_nll": 4.77828311920166, + "code_tokens": 29, + "cvar95_kl": 6.799556255340576, + "delta_nll": 2.433870315551758, + "max_kl": 7.544467926025391, + "mean_kl": 3.059868335723877, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 4.1150102615356445, + "code_tokens": 20, + "cvar95_kl": 5.759528636932373, + "delta_nll": 2.78159236907959, + "max_kl": 5.759528636932373, + "mean_kl": 2.5942587852478027, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.3499999940395355 + }, + { + "candidate_nll": 3.7515060901641846, + "code_tokens": 19, + "cvar95_kl": 7.706749439239502, + "delta_nll": 2.6348631381988525, + "max_kl": 7.706749439239502, + "mean_kl": 2.7409801483154297, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 4.490461826324463, + "code_tokens": 71, + "cvar95_kl": 10.419889450073242, + "delta_nll": 3.5441782474517822, + "max_kl": 12.931192398071289, + "mean_kl": 3.5728073120117188, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.26760563254356384 + }, + { + "candidate_nll": 4.997596263885498, + "code_tokens": 47, + "cvar95_kl": 10.813481330871582, + "delta_nll": 3.2627034187316895, + "max_kl": 12.707353591918945, + "mean_kl": 3.5969464778900146, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.3617021143436432 + }, + { + "candidate_nll": 3.6889655590057373, + "code_tokens": 50, + "cvar95_kl": 8.628803253173828, + "delta_nll": 2.8869431018829346, + "max_kl": 10.253931045532227, + "mean_kl": 2.850801706314087, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.36000001430511475 + }, + { + "candidate_nll": 3.72497296333313, + "code_tokens": 113, + "cvar95_kl": 8.274931907653809, + "delta_nll": 2.6730120182037354, + "max_kl": 10.832941055297852, + "mean_kl": 2.542893409729004, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.37168142199516296 + }, + { + "candidate_nll": 3.715372323989868, + "code_tokens": 26, + "cvar95_kl": 5.479803085327148, + "delta_nll": 2.275214672088623, + "max_kl": 5.507267475128174, + "mean_kl": 2.420628070831299, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 3.899103879928589, + "code_tokens": 62, + "cvar95_kl": 8.358048439025879, + "delta_nll": 3.330845355987549, + "max_kl": 10.328207015991211, + "mean_kl": 3.322667121887207, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.33870968222618103 + }, + { + "candidate_nll": 3.8559226989746094, + "code_tokens": 79, + "cvar95_kl": 8.606897354125977, + "delta_nll": 3.1032042503356934, + "max_kl": 9.11029052734375, + "mean_kl": 3.038228750228882, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.27848100662231445 + }, + { + "candidate_nll": 3.629249095916748, + "code_tokens": 20, + "cvar95_kl": 7.805404186248779, + "delta_nll": 2.0006003379821777, + "max_kl": 7.805404186248779, + "mean_kl": 2.2014431953430176, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.44999998807907104 + }, + { + "candidate_nll": 2.7643003463745117, + "code_tokens": 186, + "cvar95_kl": 6.7617974281311035, + "delta_nll": 2.3611559867858887, + "max_kl": 8.563138008117676, + "mean_kl": 2.374577283859253, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.42473119497299194 + }, + { + "candidate_nll": 3.899216890335083, + "code_tokens": 99, + "cvar95_kl": 8.836236953735352, + "delta_nll": 3.0560691356658936, + "max_kl": 10.290447235107422, + "mean_kl": 3.212277889251709, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 3.469623327255249, + "code_tokens": 43, + "cvar95_kl": 6.806684970855713, + "delta_nll": 2.6578149795532227, + "max_kl": 6.971786975860596, + "mean_kl": 2.7964682579040527, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.302325576543808 + }, + { + "candidate_nll": 3.060405731201172, + "code_tokens": 35, + "cvar95_kl": 4.0680832862854, + "delta_nll": 1.5866119861602783, + "max_kl": 4.71359395980835, + "mean_kl": 1.5656777620315552, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.4285714328289032 + } + ], + "policy": { + "default_bits": 4, + "name": "random_l4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 4 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 4.8506083488464355, + "median": 2.6773113012313843, + "minimum": 0.9955006241798401, + "q25": 2.2986185550689697, + "q75": 3.156209647655487 + }, + "task_macro": { + "candidate_nll": 3.7847120112842982, + "delta_nll": 2.727987810638216, + "mean_kl": 2.8206880105866325, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.35709788600603737 + }, + "token_weighted": { + "candidate_nll": 3.7429590225219727, + "cvar95_kl": 8.914681434631348, + "delta_nll": 2.815300464630127, + "max_kl": 16.917163848876953, + "mean_kl": 2.90035343170166, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.35228094458580017 + } + }, + "read_risk_l0_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 0.5179035793180051, + "macro_mean_kl": 0.5404429111791693, + "macro_top1_agreement": 0.7802550585373588, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 0.6800926509110824, + "macro_mean_kl": 0.7177636649297632, + "macro_top1_agreement": 0.7299536233362944, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 0.8626517951488495, + "macro_mean_kl": 0.8604557893492959, + "macro_top1_agreement": 0.6792999749833887, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 1.020532862706618, + "macro_mean_kl": 1.0432947765697131, + "macro_top1_agreement": 0.6560033803636377, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 1.732251524925232, + "code_tokens": 63, + "cvar95_kl": 3.474870443344116, + "delta_nll": 1.141204833984375, + "max_kl": 3.741807460784912, + "mean_kl": 0.9814460277557373, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.5873016119003296 + }, + { + "candidate_nll": 2.248811721801758, + "code_tokens": 92, + "cvar95_kl": 4.760335922241211, + "delta_nll": 1.2430880069732666, + "max_kl": 5.1050190925598145, + "mean_kl": 1.1677846908569336, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.6630434989929199 + }, + { + "candidate_nll": 2.7197532653808594, + "code_tokens": 36, + "cvar95_kl": 4.589881420135498, + "delta_nll": 0.8062640428543091, + "max_kl": 5.348085880279541, + "mean_kl": 0.9984854459762573, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.6111111044883728 + }, + { + "candidate_nll": 2.2496488094329834, + "code_tokens": 24, + "cvar95_kl": 3.2688493728637695, + "delta_nll": 0.4322397708892822, + "max_kl": 4.011963367462158, + "mean_kl": 0.7629708647727966, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.8724886178970337, + "code_tokens": 153, + "cvar95_kl": 6.034920692443848, + "delta_nll": 0.9555368423461914, + "max_kl": 7.151743412017822, + "mean_kl": 1.1941282749176025, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.6405228972434998 + }, + { + "candidate_nll": 2.5365004539489746, + "code_tokens": 133, + "cvar95_kl": 8.025484085083008, + "delta_nll": 1.4699668884277344, + "max_kl": 11.771374702453613, + "mean_kl": 1.4831041097640991, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.5639097690582275 + }, + { + "candidate_nll": 1.7924917936325073, + "code_tokens": 39, + "cvar95_kl": 3.900834560394287, + "delta_nll": 0.924458920955658, + "max_kl": 4.592340469360352, + "mean_kl": 0.7618184089660645, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.692307710647583 + }, + { + "candidate_nll": 1.7271370887756348, + "code_tokens": 26, + "cvar95_kl": 2.3618905544281006, + "delta_nll": 0.4661813974380493, + "max_kl": 2.5585296154022217, + "mean_kl": 0.4578700661659241, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.9001494646072388, + "code_tokens": 38, + "cvar95_kl": 2.387478828430176, + "delta_nll": 0.5844765901565552, + "max_kl": 2.5290989875793457, + "mean_kl": 0.5287707448005676, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.9828054904937744, + "code_tokens": 173, + "cvar95_kl": 5.090429782867432, + "delta_nll": 1.129013180732727, + "max_kl": 5.9992475509643555, + "mean_kl": 1.149426817893982, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.6358381509780884 + }, + { + "candidate_nll": 1.4193477630615234, + "code_tokens": 37, + "cvar95_kl": 2.053830623626709, + "delta_nll": 0.5025250911712646, + "max_kl": 2.103363037109375, + "mean_kl": 0.5058015584945679, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.7837837934494019 + }, + { + "candidate_nll": 2.055155038833618, + "code_tokens": 205, + "cvar95_kl": 5.599775791168213, + "delta_nll": 1.2429649829864502, + "max_kl": 6.802768230438232, + "mean_kl": 1.2813493013381958, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.6146341562271118 + }, + { + "candidate_nll": 1.7530744075775146, + "code_tokens": 162, + "cvar95_kl": 5.451320648193359, + "delta_nll": 0.854078471660614, + "max_kl": 6.91378927230835, + "mean_kl": 1.0569254159927368, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.6913580298423767 + }, + { + "candidate_nll": 1.5984687805175781, + "code_tokens": 132, + "cvar95_kl": 3.898916721343994, + "delta_nll": 0.9361112117767334, + "max_kl": 4.909263610839844, + "mean_kl": 0.8876777291297913, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.6439393758773804 + }, + { + "candidate_nll": 1.1327953338623047, + "code_tokens": 33, + "cvar95_kl": 2.5398707389831543, + "delta_nll": 0.4656016230583191, + "max_kl": 2.936553478240967, + "mean_kl": 0.44754263758659363, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.8181818127632141 + }, + { + "candidate_nll": 2.111821174621582, + "code_tokens": 60, + "cvar95_kl": 4.277328968048096, + "delta_nll": 0.7700865268707275, + "max_kl": 5.780797004699707, + "mean_kl": 0.7908816933631897, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.699999988079071 + }, + { + "candidate_nll": 1.5509346723556519, + "code_tokens": 63, + "cvar95_kl": 4.594099998474121, + "delta_nll": 1.1441290378570557, + "max_kl": 5.859574317932129, + "mean_kl": 1.0865414142608643, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.6666666865348816 + }, + { + "candidate_nll": 1.9959763288497925, + "code_tokens": 48, + "cvar95_kl": 3.144836664199829, + "delta_nll": 0.3798454999923706, + "max_kl": 3.6520800590515137, + "mean_kl": 0.7425467371940613, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.7083333134651184 + }, + { + "candidate_nll": 1.6196094751358032, + "code_tokens": 76, + "cvar95_kl": 4.821486473083496, + "delta_nll": 0.8369253873825073, + "max_kl": 6.931477069854736, + "mean_kl": 0.8121975064277649, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.7236841917037964 + }, + { + "candidate_nll": 1.6314399242401123, + "code_tokens": 63, + "cvar95_kl": 3.1793127059936523, + "delta_nll": 0.6803451776504517, + "max_kl": 3.800250768661499, + "mean_kl": 0.729357123374939, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.7936508059501648 + }, + { + "candidate_nll": 1.940843939781189, + "code_tokens": 107, + "cvar95_kl": 5.487821102142334, + "delta_nll": 1.0761098861694336, + "max_kl": 9.730924606323242, + "mean_kl": 1.0419647693634033, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.6261682510375977 + }, + { + "candidate_nll": 1.4738730192184448, + "code_tokens": 108, + "cvar95_kl": 5.022552967071533, + "delta_nll": 0.8587003946304321, + "max_kl": 5.433715343475342, + "mean_kl": 0.9610470533370972, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.6851851940155029 + }, + { + "candidate_nll": 1.5193557739257812, + "code_tokens": 46, + "cvar95_kl": 3.1560914516448975, + "delta_nll": 0.5460243821144104, + "max_kl": 4.615480899810791, + "mean_kl": 0.6608195304870605, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.760869562625885 + }, + { + "candidate_nll": 1.7791805267333984, + "code_tokens": 37, + "cvar95_kl": 4.812079429626465, + "delta_nll": 0.8499225378036499, + "max_kl": 5.208279609680176, + "mean_kl": 0.8442744016647339, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.7297297120094299 + }, + { + "candidate_nll": 1.9646399021148682, + "code_tokens": 29, + "cvar95_kl": 3.024362564086914, + "delta_nll": 0.8124837875366211, + "max_kl": 3.2402076721191406, + "mean_kl": 0.7503215074539185, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.7586206793785095 + }, + { + "candidate_nll": 1.6979126930236816, + "code_tokens": 13, + "cvar95_kl": 0.7281028628349304, + "delta_nll": 0.08378326892852783, + "max_kl": 0.7281028628349304, + "mean_kl": 0.31850001215934753, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.9230769276618958 + }, + { + "candidate_nll": 1.7905513048171997, + "code_tokens": 51, + "cvar95_kl": 4.860567569732666, + "delta_nll": 1.16745924949646, + "max_kl": 6.935393333435059, + "mean_kl": 0.9186844229698181, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.686274528503418 + }, + { + "candidate_nll": 1.8261600732803345, + "code_tokens": 29, + "cvar95_kl": 1.67676842212677, + "delta_nll": 0.6957350969314575, + "max_kl": 2.1295695304870605, + "mean_kl": 0.41461825370788574, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.8620689511299133 + }, + { + "candidate_nll": 1.5178250074386597, + "code_tokens": 30, + "cvar95_kl": 3.6664505004882812, + "delta_nll": 0.4882298707962036, + "max_kl": 4.343649387359619, + "mean_kl": 0.6858417391777039, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.800000011920929 + }, + { + "candidate_nll": 1.9049439430236816, + "code_tokens": 103, + "cvar95_kl": 4.801912784576416, + "delta_nll": 1.1142059564590454, + "max_kl": 5.96905517578125, + "mean_kl": 1.0409752130508423, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.5631067752838135 + }, + { + "candidate_nll": 1.7872135639190674, + "code_tokens": 119, + "cvar95_kl": 3.8967525959014893, + "delta_nll": 0.8866696357727051, + "max_kl": 4.89632511138916, + "mean_kl": 0.9269121289253235, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.6470588445663452 + }, + { + "candidate_nll": 1.8620305061340332, + "code_tokens": 22, + "cvar95_kl": 2.795938014984131, + "delta_nll": 0.565159797668457, + "max_kl": 2.9193925857543945, + "mean_kl": 0.7613252401351929, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.6363636255264282 + }, + { + "candidate_nll": 1.2055070400238037, + "code_tokens": 51, + "cvar95_kl": 2.6508543491363525, + "delta_nll": 0.5676289796829224, + "max_kl": 5.191623210906982, + "mean_kl": 0.5249396562576294, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.7647058963775635 + }, + { + "candidate_nll": 2.443270444869995, + "code_tokens": 32, + "cvar95_kl": 2.4545235633850098, + "delta_nll": 0.6837217807769775, + "max_kl": 2.8483693599700928, + "mean_kl": 0.5845952033996582, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.625 + }, + { + "candidate_nll": 1.7185503244400024, + "code_tokens": 116, + "cvar95_kl": 4.748171329498291, + "delta_nll": 0.6552495956420898, + "max_kl": 5.980818271636963, + "mean_kl": 0.7927326560020447, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.7241379022598267 + }, + { + "candidate_nll": 1.7016364336013794, + "code_tokens": 56, + "cvar95_kl": 3.491488218307495, + "delta_nll": 0.9164514541625977, + "max_kl": 4.3265061378479, + "mean_kl": 0.6824652552604675, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.7321428656578064 + }, + { + "candidate_nll": 1.8299857378005981, + "code_tokens": 53, + "cvar95_kl": 3.5960323810577393, + "delta_nll": 0.8821116089820862, + "max_kl": 3.892441987991333, + "mean_kl": 0.9184355735778809, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.6037735939025879 + }, + { + "candidate_nll": 1.8149334192276, + "code_tokens": 127, + "cvar95_kl": 6.560326099395752, + "delta_nll": 1.2037125825881958, + "max_kl": 9.002546310424805, + "mean_kl": 1.1901766061782837, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.6299212574958801 + }, + { + "candidate_nll": 2.148423194885254, + "code_tokens": 61, + "cvar95_kl": 3.989513635635376, + "delta_nll": 0.9358888864517212, + "max_kl": 5.814340591430664, + "mean_kl": 1.063145399093628, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.5573770403862 + }, + { + "candidate_nll": 1.450118899345398, + "code_tokens": 147, + "cvar95_kl": 5.18900728225708, + "delta_nll": 0.9041231274604797, + "max_kl": 8.659774780273438, + "mean_kl": 0.9226810336112976, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.7482993006706238 + }, + { + "candidate_nll": 1.9640789031982422, + "code_tokens": 28, + "cvar95_kl": 2.654557704925537, + "delta_nll": 0.4860924482345581, + "max_kl": 3.840364694595337, + "mean_kl": 0.579189658164978, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.6785714030265808 + }, + { + "candidate_nll": 1.4519189596176147, + "code_tokens": 66, + "cvar95_kl": 4.233325481414795, + "delta_nll": 0.6155489683151245, + "max_kl": 8.123661041259766, + "mean_kl": 0.7767921090126038, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 2.3811237812042236, + "code_tokens": 33, + "cvar95_kl": 5.815465450286865, + "delta_nll": 0.5543190240859985, + "max_kl": 6.136956691741943, + "mean_kl": 0.7374077439308167, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.7575757503509521 + }, + { + "candidate_nll": 2.0118210315704346, + "code_tokens": 41, + "cvar95_kl": 3.478135347366333, + "delta_nll": 0.8741217851638794, + "max_kl": 3.7473278045654297, + "mean_kl": 0.7758055925369263, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.7317073345184326 + }, + { + "candidate_nll": 2.2031726837158203, + "code_tokens": 36, + "cvar95_kl": 2.3229804039001465, + "delta_nll": 0.5269949436187744, + "max_kl": 2.323989152908325, + "mean_kl": 0.6381937861442566, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.6388888955116272 + }, + { + "candidate_nll": 2.0672099590301514, + "code_tokens": 70, + "cvar95_kl": 3.522871971130371, + "delta_nll": 1.1500751972198486, + "max_kl": 4.446450710296631, + "mean_kl": 0.9686630368232727, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.6000000238418579 + }, + { + "candidate_nll": 1.8653390407562256, + "code_tokens": 17, + "cvar95_kl": 2.0200142860412598, + "delta_nll": 0.5658745765686035, + "max_kl": 2.0200142860412598, + "mean_kl": 0.4210987389087677, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.8823529481887817 + }, + { + "candidate_nll": 1.5949524641036987, + "code_tokens": 23, + "cvar95_kl": 1.5197210311889648, + "delta_nll": 0.4097452163696289, + "max_kl": 1.6185063123703003, + "mean_kl": 0.41086018085479736, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.602938175201416, + "code_tokens": 101, + "cvar95_kl": 5.3965067863464355, + "delta_nll": 1.2250646352767944, + "max_kl": 6.981556415557861, + "mean_kl": 1.259110689163208, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.5742574334144592 + }, + { + "candidate_nll": 1.9242193698883057, + "code_tokens": 36, + "cvar95_kl": 5.365108966827393, + "delta_nll": 0.5093426704406738, + "max_kl": 6.128144264221191, + "mean_kl": 0.7643920183181763, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.69818913936615, + "code_tokens": 46, + "cvar95_kl": 3.960514783859253, + "delta_nll": 0.4924354553222656, + "max_kl": 5.271826267242432, + "mean_kl": 0.8176872134208679, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.739130437374115 + }, + { + "candidate_nll": 2.0165300369262695, + "code_tokens": 24, + "cvar95_kl": 2.4906840324401855, + "delta_nll": 0.4856834411621094, + "max_kl": 3.6217143535614014, + "mean_kl": 0.4304535388946533, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.875 + }, + { + "candidate_nll": 1.8642551898956299, + "code_tokens": 22, + "cvar95_kl": 2.5553269386291504, + "delta_nll": 0.4889259338378906, + "max_kl": 2.8784468173980713, + "mean_kl": 0.5888121128082275, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.5909090638160706 + }, + { + "candidate_nll": 1.5638474225997925, + "code_tokens": 58, + "cvar95_kl": 3.671574354171753, + "delta_nll": 0.8749749064445496, + "max_kl": 3.9545581340789795, + "mean_kl": 0.7755099534988403, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.6551724076271057 + }, + { + "candidate_nll": 1.6432263851165771, + "code_tokens": 15, + "cvar95_kl": 2.7998435497283936, + "delta_nll": 0.3303558826446533, + "max_kl": 2.7998435497283936, + "mean_kl": 0.4439100921154022, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.9333333373069763 + }, + { + "candidate_nll": 1.2727930545806885, + "code_tokens": 40, + "cvar95_kl": 5.503000259399414, + "delta_nll": 0.8332054615020752, + "max_kl": 5.872326374053955, + "mean_kl": 0.7590802907943726, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.7749999761581421 + }, + { + "candidate_nll": 1.475277304649353, + "code_tokens": 37, + "cvar95_kl": 5.659841060638428, + "delta_nll": 0.8707277178764343, + "max_kl": 7.1288909912109375, + "mean_kl": 0.892333447933197, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.7027027010917664 + }, + { + "candidate_nll": 1.535710334777832, + "code_tokens": 21, + "cvar95_kl": 3.0329627990722656, + "delta_nll": 0.5154644250869751, + "max_kl": 4.079394340515137, + "mean_kl": 0.4050963222980499, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.8571428656578064 + }, + { + "candidate_nll": 1.8222547769546509, + "code_tokens": 21, + "cvar95_kl": 1.8580646514892578, + "delta_nll": 0.5097389221191406, + "max_kl": 2.379983425140381, + "mean_kl": 0.37436068058013916, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 2.17185378074646, + "code_tokens": 66, + "cvar95_kl": 6.8722310066223145, + "delta_nll": 1.1634891033172607, + "max_kl": 11.786860466003418, + "mean_kl": 1.2447997331619263, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.6060606241226196 + }, + { + "candidate_nll": 1.6720259189605713, + "code_tokens": 138, + "cvar95_kl": 4.463130950927734, + "delta_nll": 1.028365135192871, + "max_kl": 5.386237621307373, + "mean_kl": 0.9903431534767151, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.6086956262588501 + }, + { + "candidate_nll": 1.4151910543441772, + "code_tokens": 22, + "cvar95_kl": 3.607597827911377, + "delta_nll": 0.6809484362602234, + "max_kl": 5.018783092498779, + "mean_kl": 0.7517454028129578, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.7727272510528564 + }, + { + "candidate_nll": 1.4702600240707397, + "code_tokens": 37, + "cvar95_kl": 4.183772563934326, + "delta_nll": 0.724675714969635, + "max_kl": 4.6542649269104, + "mean_kl": 0.6333199739456177, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.7027027010917664 + }, + { + "candidate_nll": 1.7952399253845215, + "code_tokens": 41, + "cvar95_kl": 3.1770858764648438, + "delta_nll": 0.8444822430610657, + "max_kl": 4.2851715087890625, + "mean_kl": 0.7267717719078064, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.707317054271698 + }, + { + "candidate_nll": 1.962184190750122, + "code_tokens": 63, + "cvar95_kl": 3.191837787628174, + "delta_nll": 0.803693413734436, + "max_kl": 4.139280319213867, + "mean_kl": 0.8838675618171692, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.5873016119003296 + }, + { + "candidate_nll": 1.5951545238494873, + "code_tokens": 99, + "cvar95_kl": 3.0328381061553955, + "delta_nll": 0.7484027147293091, + "max_kl": 3.474276304244995, + "mean_kl": 0.7106393575668335, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.7272727489471436 + }, + { + "candidate_nll": 1.3291159868240356, + "code_tokens": 65, + "cvar95_kl": 1.767317771911621, + "delta_nll": 0.39902228116989136, + "max_kl": 1.9927705526351929, + "mean_kl": 0.5239065289497375, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.7384615540504456 + }, + { + "candidate_nll": 1.5504562854766846, + "code_tokens": 39, + "cvar95_kl": 4.638823986053467, + "delta_nll": 0.6329271793365479, + "max_kl": 5.931685924530029, + "mean_kl": 0.6130590438842773, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.7179487347602844 + }, + { + "candidate_nll": 1.8587223291397095, + "code_tokens": 40, + "cvar95_kl": 5.913382530212402, + "delta_nll": 0.6999615430831909, + "max_kl": 8.238985061645508, + "mean_kl": 0.8137434720993042, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 2.4069576263427734, + "code_tokens": 97, + "cvar95_kl": 4.830368995666504, + "delta_nll": 1.1120054721832275, + "max_kl": 6.578426837921143, + "mean_kl": 1.095460057258606, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.6391752362251282 + }, + { + "candidate_nll": 2.215336799621582, + "code_tokens": 23, + "cvar95_kl": 1.4431116580963135, + "delta_nll": 0.3003079891204834, + "max_kl": 1.4984580278396606, + "mean_kl": 0.5395223498344421, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.5652173757553101 + }, + { + "candidate_nll": 1.5997490882873535, + "code_tokens": 25, + "cvar95_kl": 1.6611899137496948, + "delta_nll": 0.5658794641494751, + "max_kl": 1.7832651138305664, + "mean_kl": 0.4811854064464569, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.8799999952316284 + }, + { + "candidate_nll": 1.9063208103179932, + "code_tokens": 57, + "cvar95_kl": 4.0191121101379395, + "delta_nll": 1.0507922172546387, + "max_kl": 5.517393589019775, + "mean_kl": 0.8299940228462219, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.6842105388641357 + }, + { + "candidate_nll": 1.913771629333496, + "code_tokens": 50, + "cvar95_kl": 3.6857547760009766, + "delta_nll": 0.8714720010757446, + "max_kl": 3.9767940044403076, + "mean_kl": 1.0066502094268799, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.6200000047683716 + }, + { + "candidate_nll": 1.771997332572937, + "code_tokens": 35, + "cvar95_kl": 3.831244468688965, + "delta_nll": 0.9434136152267456, + "max_kl": 3.856947898864746, + "mean_kl": 0.9803882241249084, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.7142857313156128 + }, + { + "candidate_nll": 2.775933265686035, + "code_tokens": 29, + "cvar95_kl": 3.468865394592285, + "delta_nll": 0.4315204620361328, + "max_kl": 4.713822841644287, + "mean_kl": 0.6695300936698914, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.5862069129943848 + }, + { + "candidate_nll": 2.14585542678833, + "code_tokens": 20, + "cvar95_kl": 2.4081037044525146, + "delta_nll": 0.8124374151229858, + "max_kl": 2.4081037044525146, + "mean_kl": 0.5416251420974731, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.8500000238418579 + }, + { + "candidate_nll": 1.506408452987671, + "code_tokens": 19, + "cvar95_kl": 1.9445544481277466, + "delta_nll": 0.38976550102233887, + "max_kl": 1.9445544481277466, + "mean_kl": 0.3775123655796051, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.8947368264198303 + }, + { + "candidate_nll": 2.0468297004699707, + "code_tokens": 71, + "cvar95_kl": 7.276410102844238, + "delta_nll": 1.10054612159729, + "max_kl": 8.970968246459961, + "mean_kl": 1.1704801321029663, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.6338028311729431 + }, + { + "candidate_nll": 2.611591339111328, + "code_tokens": 47, + "cvar95_kl": 5.950962543487549, + "delta_nll": 0.8766984939575195, + "max_kl": 7.460782527923584, + "mean_kl": 1.1716316938400269, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.7021276354789734 + }, + { + "candidate_nll": 1.6417804956436157, + "code_tokens": 50, + "cvar95_kl": 4.791374683380127, + "delta_nll": 0.839758038520813, + "max_kl": 6.7698259353637695, + "mean_kl": 0.7675440907478333, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.7599999904632568 + }, + { + "candidate_nll": 2.321031093597412, + "code_tokens": 113, + "cvar95_kl": 4.14279317855835, + "delta_nll": 1.2690701484680176, + "max_kl": 4.709221363067627, + "mean_kl": 1.1301106214523315, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.6194690465927124 + }, + { + "candidate_nll": 2.601745128631592, + "code_tokens": 26, + "cvar95_kl": 5.0964765548706055, + "delta_nll": 1.1615873575210571, + "max_kl": 5.340967655181885, + "mean_kl": 0.8116962909698486, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.7692307829856873 + }, + { + "candidate_nll": 1.255149245262146, + "code_tokens": 62, + "cvar95_kl": 5.216206073760986, + "delta_nll": 0.6868906021118164, + "max_kl": 10.479104042053223, + "mean_kl": 0.7311791777610779, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.8387096524238586 + }, + { + "candidate_nll": 1.4854358434677124, + "code_tokens": 79, + "cvar95_kl": 4.315554141998291, + "delta_nll": 0.7327173352241516, + "max_kl": 6.2377777099609375, + "mean_kl": 0.6723738312721252, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.7848101258277893 + }, + { + "candidate_nll": 1.862290620803833, + "code_tokens": 20, + "cvar95_kl": 1.6864235401153564, + "delta_nll": 0.2336418628692627, + "max_kl": 1.6864235401153564, + "mean_kl": 0.4521408975124359, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.75 + }, + { + "candidate_nll": 1.1973932981491089, + "code_tokens": 186, + "cvar95_kl": 4.050142765045166, + "delta_nll": 0.7942488789558411, + "max_kl": 6.547865390777588, + "mean_kl": 0.8819687366485596, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.7043010592460632 + }, + { + "candidate_nll": 1.8554657697677612, + "code_tokens": 99, + "cvar95_kl": 5.436880111694336, + "delta_nll": 1.0123178958892822, + "max_kl": 6.03115701675415, + "mean_kl": 1.1155928373336792, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.6969696879386902 + }, + { + "candidate_nll": 1.2948133945465088, + "code_tokens": 43, + "cvar95_kl": 2.5436809062957764, + "delta_nll": 0.483005166053772, + "max_kl": 4.766280651092529, + "mean_kl": 0.4108973741531372, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.8372092843055725 + }, + { + "candidate_nll": 1.886618733406067, + "code_tokens": 35, + "cvar95_kl": 2.3529393672943115, + "delta_nll": 0.41282498836517334, + "max_kl": 3.4648971557617188, + "mean_kl": 0.44174471497535706, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.8571428656578064 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 1.4699668884277344, + "median": 0.7989711463451385, + "minimum": 0.08378326892852783, + "q25": 0.5183470547199249, + "q75": 0.9360556304454803 + }, + "task_macro": { + "candidate_nll": 1.8232128222783406, + "delta_nll": 0.7664886196454366, + "mean_kl": 0.7869029300080406, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.7123497055636512 + }, + "token_weighted": { + "candidate_nll": 1.8130292892456055, + "cvar95_kl": 4.917716026306152, + "delta_nll": 0.885370671749115, + "max_kl": 11.786860466003418, + "mean_kl": 0.9094873070716858, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.6833816170692444 + } + }, + "read_risk_l0_stochastic_seed_2339": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.370992339175681, + "macro_mean_kl": 4.568015461382658, + "macro_top1_agreement": 0.23231710234413977, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 5.254188527231631, + "macro_mean_kl": 5.596312636914461, + "macro_top1_agreement": 0.20654311257859934, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 6.060611941597679, + "macro_mean_kl": 6.174874392422763, + "macro_top1_agreement": 0.16659201072020965, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 7.636982549320567, + "macro_mean_kl": 7.732887788252397, + "macro_top1_agreement": 0.10166361860253594, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 6.974740028381348, + "code_tokens": 63, + "cvar95_kl": 12.802448272705078, + "delta_nll": 6.383693218231201, + "max_kl": 14.10097885131836, + "mean_kl": 6.029170989990234, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.1587301641702652 + }, + { + "candidate_nll": 8.445570945739746, + "code_tokens": 92, + "cvar95_kl": 17.25725746154785, + "delta_nll": 7.439846992492676, + "max_kl": 18.46240234375, + "mean_kl": 7.603812217712402, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.09782608598470688 + }, + { + "candidate_nll": 7.005684852600098, + "code_tokens": 36, + "cvar95_kl": 13.947959899902344, + "delta_nll": 5.092195510864258, + "max_kl": 14.10556697845459, + "mean_kl": 5.891890048980713, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 5.837184429168701, + "code_tokens": 24, + "cvar95_kl": 12.917991638183594, + "delta_nll": 4.019775390625, + "max_kl": 13.543244361877441, + "mean_kl": 4.995212078094482, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 8.685683250427246, + "code_tokens": 153, + "cvar95_kl": 15.874530792236328, + "delta_nll": 7.768731594085693, + "max_kl": 16.872344970703125, + "mean_kl": 7.869211673736572, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.0784313753247261 + }, + { + "candidate_nll": 9.564866065979004, + "code_tokens": 133, + "cvar95_kl": 19.081384658813477, + "delta_nll": 8.498332977294922, + "max_kl": 20.712242126464844, + "mean_kl": 8.813499450683594, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.07518796622753143 + }, + { + "candidate_nll": 6.093593120574951, + "code_tokens": 39, + "cvar95_kl": 10.571945190429688, + "delta_nll": 5.225560188293457, + "max_kl": 10.575294494628906, + "mean_kl": 5.1774420738220215, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 4.307559490203857, + "code_tokens": 26, + "cvar95_kl": 10.135454177856445, + "delta_nll": 3.0466036796569824, + "max_kl": 11.344375610351562, + "mean_kl": 3.3723526000976562, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 5.760018348693848, + "code_tokens": 38, + "cvar95_kl": 11.736849784851074, + "delta_nll": 4.444345474243164, + "max_kl": 12.70501708984375, + "mean_kl": 4.077847480773926, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.34210526943206787 + }, + { + "candidate_nll": 9.715704917907715, + "code_tokens": 173, + "cvar95_kl": 18.29750633239746, + "delta_nll": 8.861912727355957, + "max_kl": 21.15022087097168, + "mean_kl": 9.007447242736816, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.080924853682518 + }, + { + "candidate_nll": 6.313807487487793, + "code_tokens": 37, + "cvar95_kl": 13.716251373291016, + "delta_nll": 5.396985054016113, + "max_kl": 14.471479415893555, + "mean_kl": 5.533100605010986, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 8.85506534576416, + "code_tokens": 205, + "cvar95_kl": 16.620920181274414, + "delta_nll": 8.042875289916992, + "max_kl": 20.251230239868164, + "mean_kl": 7.9236907958984375, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.039024390280246735 + }, + { + "candidate_nll": 9.382599830627441, + "code_tokens": 162, + "cvar95_kl": 19.46416664123535, + "delta_nll": 8.483603477478027, + "max_kl": 23.92094612121582, + "mean_kl": 8.516987800598145, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.08024691045284271 + }, + { + "candidate_nll": 9.194616317749023, + "code_tokens": 132, + "cvar95_kl": 18.25509262084961, + "delta_nll": 8.532258987426758, + "max_kl": 23.40381622314453, + "mean_kl": 8.568408012390137, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.09090909361839294 + }, + { + "candidate_nll": 5.684486389160156, + "code_tokens": 33, + "cvar95_kl": 11.564269065856934, + "delta_nll": 5.017292499542236, + "max_kl": 11.603829383850098, + "mean_kl": 5.344631671905518, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 7.323878765106201, + "code_tokens": 60, + "cvar95_kl": 14.112614631652832, + "delta_nll": 5.982144355773926, + "max_kl": 15.465890884399414, + "mean_kl": 5.960351467132568, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.942822456359863, + "code_tokens": 63, + "cvar95_kl": 15.735878944396973, + "delta_nll": 6.536016941070557, + "max_kl": 17.41824722290039, + "mean_kl": 6.499839782714844, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 7.327298641204834, + "code_tokens": 48, + "cvar95_kl": 12.691093444824219, + "delta_nll": 5.711167812347412, + "max_kl": 15.336591720581055, + "mean_kl": 6.287228107452393, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.0625 + }, + { + "candidate_nll": 6.482539176940918, + "code_tokens": 76, + "cvar95_kl": 14.939459800720215, + "delta_nll": 5.699854850769043, + "max_kl": 18.65948486328125, + "mean_kl": 5.916039943695068, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.17105263471603394 + }, + { + "candidate_nll": 5.510316371917725, + "code_tokens": 63, + "cvar95_kl": 12.563594818115234, + "delta_nll": 4.5592217445373535, + "max_kl": 15.750612258911133, + "mean_kl": 4.699466705322266, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 8.27934741973877, + "code_tokens": 107, + "cvar95_kl": 17.045854568481445, + "delta_nll": 7.414613246917725, + "max_kl": 22.147605895996094, + "mean_kl": 7.771185874938965, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.1214953288435936 + }, + { + "candidate_nll": 7.3728814125061035, + "code_tokens": 108, + "cvar95_kl": 16.551931381225586, + "delta_nll": 6.757708549499512, + "max_kl": 21.055421829223633, + "mean_kl": 7.016513347625732, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 7.35414457321167, + "code_tokens": 46, + "cvar95_kl": 15.36780834197998, + "delta_nll": 6.380813121795654, + "max_kl": 15.988103866577148, + "mean_kl": 6.7955217361450195, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.19565217196941376 + }, + { + "candidate_nll": 6.731426239013672, + "code_tokens": 37, + "cvar95_kl": 12.777986526489258, + "delta_nll": 5.802168369293213, + "max_kl": 13.356475830078125, + "mean_kl": 6.189980983734131, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 5.564114093780518, + "code_tokens": 29, + "cvar95_kl": 10.533136367797852, + "delta_nll": 4.411957740783691, + "max_kl": 10.79262638092041, + "mean_kl": 4.209768295288086, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 5.567591667175293, + "code_tokens": 13, + "cvar95_kl": 8.787561416625977, + "delta_nll": 3.9534621238708496, + "max_kl": 8.787561416625977, + "mean_kl": 3.373650074005127, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.38461539149284363 + }, + { + "candidate_nll": 7.16248083114624, + "code_tokens": 51, + "cvar95_kl": 14.627189636230469, + "delta_nll": 6.539388656616211, + "max_kl": 15.472834587097168, + "mean_kl": 6.397191524505615, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.13725490868091583 + }, + { + "candidate_nll": 6.505219459533691, + "code_tokens": 29, + "cvar95_kl": 12.453593254089355, + "delta_nll": 5.3747944831848145, + "max_kl": 13.186201095581055, + "mean_kl": 5.578799724578857, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 6.3857622146606445, + "code_tokens": 30, + "cvar95_kl": 15.38247013092041, + "delta_nll": 5.356166839599609, + "max_kl": 15.9933500289917, + "mean_kl": 5.228220462799072, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 8.333529472351074, + "code_tokens": 103, + "cvar95_kl": 15.695256233215332, + "delta_nll": 7.542791366577148, + "max_kl": 16.990705490112305, + "mean_kl": 7.428241729736328, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.08737864345312119 + }, + { + "candidate_nll": 9.269323348999023, + "code_tokens": 119, + "cvar95_kl": 19.805410385131836, + "delta_nll": 8.368779182434082, + "max_kl": 21.939929962158203, + "mean_kl": 8.466765403747559, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.08403361588716507 + }, + { + "candidate_nll": 7.214044570922852, + "code_tokens": 22, + "cvar95_kl": 13.806503295898438, + "delta_nll": 5.917173862457275, + "max_kl": 13.84778881072998, + "mean_kl": 6.241672039031982, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 5.43965482711792, + "code_tokens": 51, + "cvar95_kl": 11.842211723327637, + "delta_nll": 4.801776885986328, + "max_kl": 13.113347053527832, + "mean_kl": 4.89779806137085, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.21568627655506134 + }, + { + "candidate_nll": 6.441482067108154, + "code_tokens": 32, + "cvar95_kl": 10.368082046508789, + "delta_nll": 4.681933403015137, + "max_kl": 10.374910354614258, + "mean_kl": 5.391016960144043, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.125 + }, + { + "candidate_nll": 8.242984771728516, + "code_tokens": 116, + "cvar95_kl": 17.231117248535156, + "delta_nll": 7.179684162139893, + "max_kl": 19.540054321289062, + "mean_kl": 7.120629787445068, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.08620689809322357 + }, + { + "candidate_nll": 6.593114376068115, + "code_tokens": 56, + "cvar95_kl": 14.429523468017578, + "delta_nll": 5.807929515838623, + "max_kl": 15.538570404052734, + "mean_kl": 5.489715099334717, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 6.119375705718994, + "code_tokens": 53, + "cvar95_kl": 10.032809257507324, + "delta_nll": 5.171501636505127, + "max_kl": 10.062739372253418, + "mean_kl": 5.347361087799072, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.18867924809455872 + }, + { + "candidate_nll": 8.419652938842773, + "code_tokens": 127, + "cvar95_kl": 16.86922264099121, + "delta_nll": 7.808432102203369, + "max_kl": 20.14916229248047, + "mean_kl": 7.618187427520752, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.09448818862438202 + }, + { + "candidate_nll": 6.779396057128906, + "code_tokens": 61, + "cvar95_kl": 13.054197311401367, + "delta_nll": 5.566861629486084, + "max_kl": 14.11196231842041, + "mean_kl": 5.741044044494629, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.13114753365516663 + }, + { + "candidate_nll": 8.78656005859375, + "code_tokens": 147, + "cvar95_kl": 19.084016799926758, + "delta_nll": 8.240564346313477, + "max_kl": 21.641212463378906, + "mean_kl": 8.185507774353027, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 6.701731204986572, + "code_tokens": 28, + "cvar95_kl": 12.652978897094727, + "delta_nll": 5.223744869232178, + "max_kl": 12.813732147216797, + "mean_kl": 5.504700183868408, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 8.291099548339844, + "code_tokens": 66, + "cvar95_kl": 18.172409057617188, + "delta_nll": 7.4547295570373535, + "max_kl": 19.26842498779297, + "mean_kl": 7.6973137855529785, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.13636364042758942 + }, + { + "candidate_nll": 6.758686542510986, + "code_tokens": 33, + "cvar95_kl": 14.740278244018555, + "delta_nll": 4.931881904602051, + "max_kl": 16.258787155151367, + "mean_kl": 5.750136375427246, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 7.303677558898926, + "code_tokens": 41, + "cvar95_kl": 17.261552810668945, + "delta_nll": 6.16597843170166, + "max_kl": 18.213891983032227, + "mean_kl": 6.453349590301514, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.17073170840740204 + }, + { + "candidate_nll": 5.571554660797119, + "code_tokens": 36, + "cvar95_kl": 12.276771545410156, + "delta_nll": 3.8953769207000732, + "max_kl": 12.342561721801758, + "mean_kl": 4.4551849365234375, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.797791481018066, + "code_tokens": 70, + "cvar95_kl": 18.275856018066406, + "delta_nll": 6.880656719207764, + "max_kl": 22.739904403686523, + "mean_kl": 6.973044395446777, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.0714285746216774 + }, + { + "candidate_nll": 5.8224263191223145, + "code_tokens": 17, + "cvar95_kl": 17.220600128173828, + "delta_nll": 4.522961616516113, + "max_kl": 17.220600128173828, + "mean_kl": 4.957215785980225, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.1764705926179886 + }, + { + "candidate_nll": 5.249005317687988, + "code_tokens": 23, + "cvar95_kl": 10.005476951599121, + "delta_nll": 4.063797950744629, + "max_kl": 10.875011444091797, + "mean_kl": 3.795933961868286, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 8.359569549560547, + "code_tokens": 101, + "cvar95_kl": 15.643603324890137, + "delta_nll": 7.981696128845215, + "max_kl": 17.37759017944336, + "mean_kl": 8.152217864990234, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.15841583907604218 + }, + { + "candidate_nll": 5.776579856872559, + "code_tokens": 36, + "cvar95_kl": 12.377686500549316, + "delta_nll": 4.361702919006348, + "max_kl": 14.136775970458984, + "mean_kl": 5.204257488250732, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.351516246795654, + "code_tokens": 46, + "cvar95_kl": 14.092377662658691, + "delta_nll": 6.1457624435424805, + "max_kl": 17.106611251831055, + "mean_kl": 6.328570365905762, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 7.13072395324707, + "code_tokens": 24, + "cvar95_kl": 14.637588500976562, + "delta_nll": 5.59987735748291, + "max_kl": 15.497237205505371, + "mean_kl": 5.740052700042725, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.0833333358168602 + }, + { + "candidate_nll": 5.794450759887695, + "code_tokens": 22, + "cvar95_kl": 11.546947479248047, + "delta_nll": 4.419121742248535, + "max_kl": 12.618914604187012, + "mean_kl": 5.1935272216796875, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 6.284912109375, + "code_tokens": 58, + "cvar95_kl": 16.171802520751953, + "delta_nll": 5.596039772033691, + "max_kl": 19.08001136779785, + "mean_kl": 5.798869609832764, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.1551724076271057 + }, + { + "candidate_nll": 5.161379337310791, + "code_tokens": 15, + "cvar95_kl": 10.648674011230469, + "delta_nll": 3.848508834838867, + "max_kl": 10.648674011230469, + "mean_kl": 4.260432243347168, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 5.221661567687988, + "code_tokens": 40, + "cvar95_kl": 17.39031219482422, + "delta_nll": 4.782073974609375, + "max_kl": 18.058700561523438, + "mean_kl": 4.699553489685059, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 7.327016353607178, + "code_tokens": 37, + "cvar95_kl": 16.03681182861328, + "delta_nll": 6.722466945648193, + "max_kl": 16.843130111694336, + "mean_kl": 6.793019771575928, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 6.013007640838623, + "code_tokens": 21, + "cvar95_kl": 10.127534866333008, + "delta_nll": 4.992761611938477, + "max_kl": 10.457286834716797, + "mean_kl": 5.405475616455078, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.095238097012043 + }, + { + "candidate_nll": 3.925049304962158, + "code_tokens": 21, + "cvar95_kl": 7.481966972351074, + "delta_nll": 2.6125335693359375, + "max_kl": 8.45840835571289, + "mean_kl": 3.1037814617156982, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 7.525839328765869, + "code_tokens": 66, + "cvar95_kl": 15.085143089294434, + "delta_nll": 6.51747465133667, + "max_kl": 17.116424560546875, + "mean_kl": 6.98826265335083, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.13636364042758942 + }, + { + "candidate_nll": 7.5206990242004395, + "code_tokens": 138, + "cvar95_kl": 15.353676795959473, + "delta_nll": 6.87703800201416, + "max_kl": 17.036378860473633, + "mean_kl": 6.9324259757995605, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.15942029654979706 + }, + { + "candidate_nll": 5.834319591522217, + "code_tokens": 22, + "cvar95_kl": 10.876739501953125, + "delta_nll": 5.100077152252197, + "max_kl": 12.123966217041016, + "mean_kl": 4.525371074676514, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 4.7748870849609375, + "code_tokens": 37, + "cvar95_kl": 10.60107135772705, + "delta_nll": 4.029302597045898, + "max_kl": 11.445956230163574, + "mean_kl": 3.97615122795105, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 5.2497358322143555, + "code_tokens": 41, + "cvar95_kl": 11.216687202453613, + "delta_nll": 4.298978328704834, + "max_kl": 13.283617973327637, + "mean_kl": 4.597160339355469, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.2195121943950653 + }, + { + "candidate_nll": 6.330481052398682, + "code_tokens": 63, + "cvar95_kl": 15.503747940063477, + "delta_nll": 5.171990394592285, + "max_kl": 17.817529678344727, + "mean_kl": 5.822525501251221, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 6.7657012939453125, + "code_tokens": 99, + "cvar95_kl": 14.30577564239502, + "delta_nll": 5.918949604034424, + "max_kl": 15.163710594177246, + "mean_kl": 6.104728698730469, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.638106822967529, + "code_tokens": 65, + "cvar95_kl": 15.131366729736328, + "delta_nll": 5.70801305770874, + "max_kl": 17.055986404418945, + "mean_kl": 5.726099967956543, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.2153846174478531 + }, + { + "candidate_nll": 5.888869285583496, + "code_tokens": 39, + "cvar95_kl": 13.11184310913086, + "delta_nll": 4.971340179443359, + "max_kl": 13.301127433776855, + "mean_kl": 5.811803340911865, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.1538461595773697 + }, + { + "candidate_nll": 6.974963188171387, + "code_tokens": 40, + "cvar95_kl": 15.968128204345703, + "delta_nll": 5.816202163696289, + "max_kl": 16.213741302490234, + "mean_kl": 6.393324851989746, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 10.153322219848633, + "code_tokens": 97, + "cvar95_kl": 18.91643714904785, + "delta_nll": 8.858369827270508, + "max_kl": 19.421646118164062, + "mean_kl": 9.139683723449707, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.07216494530439377 + }, + { + "candidate_nll": 4.478241920471191, + "code_tokens": 23, + "cvar95_kl": 6.948012351989746, + "delta_nll": 2.5632131099700928, + "max_kl": 7.615723609924316, + "mean_kl": 3.2948601245880127, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 4.7450785636901855, + "code_tokens": 25, + "cvar95_kl": 12.193367004394531, + "delta_nll": 3.7112088203430176, + "max_kl": 14.339950561523438, + "mean_kl": 3.8905043601989746, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.265447616577148, + "code_tokens": 57, + "cvar95_kl": 15.214072227478027, + "delta_nll": 6.409918785095215, + "max_kl": 15.99849796295166, + "mean_kl": 5.999303817749023, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.28070175647735596 + }, + { + "candidate_nll": 7.236759662628174, + "code_tokens": 50, + "cvar95_kl": 15.813087463378906, + "delta_nll": 6.194459915161133, + "max_kl": 17.94374656677246, + "mean_kl": 6.384695053100586, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.18000000715255737 + }, + { + "candidate_nll": 7.234143257141113, + "code_tokens": 35, + "cvar95_kl": 13.709182739257812, + "delta_nll": 6.405559539794922, + "max_kl": 14.20266056060791, + "mean_kl": 6.3567633628845215, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + }, + { + "candidate_nll": 6.0271148681640625, + "code_tokens": 29, + "cvar95_kl": 11.39857292175293, + "delta_nll": 3.68270206451416, + "max_kl": 11.538177490234375, + "mean_kl": 4.352133750915527, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.3103448152542114 + }, + { + "candidate_nll": 5.793717861175537, + "code_tokens": 20, + "cvar95_kl": 11.94158935546875, + "delta_nll": 4.460299968719482, + "max_kl": 11.94158935546875, + "mean_kl": 4.9092302322387695, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 5.948282718658447, + "code_tokens": 19, + "cvar95_kl": 12.107385635375977, + "delta_nll": 4.831639766693115, + "max_kl": 12.107385635375977, + "mean_kl": 4.600917816162109, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.653994083404541, + "code_tokens": 71, + "cvar95_kl": 16.429914474487305, + "delta_nll": 6.707710266113281, + "max_kl": 17.059598922729492, + "mean_kl": 7.0038228034973145, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.1267605572938919 + }, + { + "candidate_nll": 7.16719388961792, + "code_tokens": 47, + "cvar95_kl": 15.803241729736328, + "delta_nll": 5.432301044464111, + "max_kl": 18.912620544433594, + "mean_kl": 6.09286642074585, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.1702127605676651 + }, + { + "candidate_nll": 6.835486888885498, + "code_tokens": 50, + "cvar95_kl": 18.57679557800293, + "delta_nll": 6.033464431762695, + "max_kl": 20.51227378845215, + "mean_kl": 6.210640907287598, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.9128217697143555, + "code_tokens": 113, + "cvar95_kl": 16.67354965209961, + "delta_nll": 6.860860824584961, + "max_kl": 20.793630599975586, + "mean_kl": 6.80762243270874, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.1150442510843277 + }, + { + "candidate_nll": 6.141636848449707, + "code_tokens": 26, + "cvar95_kl": 10.297872543334961, + "delta_nll": 4.701478958129883, + "max_kl": 11.079755783081055, + "mean_kl": 4.21248722076416, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.26923078298568726 + }, + { + "candidate_nll": 8.467706680297852, + "code_tokens": 62, + "cvar95_kl": 16.677066802978516, + "delta_nll": 7.899447917938232, + "max_kl": 18.61310577392578, + "mean_kl": 7.977451324462891, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 7.6418070793151855, + "code_tokens": 79, + "cvar95_kl": 16.674461364746094, + "delta_nll": 6.8890886306762695, + "max_kl": 17.941280364990234, + "mean_kl": 7.174304008483887, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.10126582533121109 + }, + { + "candidate_nll": 5.747611045837402, + "code_tokens": 20, + "cvar95_kl": 13.418828010559082, + "delta_nll": 4.118962287902832, + "max_kl": 13.418828010559082, + "mean_kl": 4.318056583404541, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.088084697723389, + "code_tokens": 186, + "cvar95_kl": 18.574426651000977, + "delta_nll": 6.684940338134766, + "max_kl": 23.649112701416016, + "mean_kl": 6.8278584480285645, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.16129031777381897 + }, + { + "candidate_nll": 7.8456854820251465, + "code_tokens": 99, + "cvar95_kl": 15.933067321777344, + "delta_nll": 7.002537727355957, + "max_kl": 18.954248428344727, + "mean_kl": 7.074601650238037, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.12121212482452393 + }, + { + "candidate_nll": 6.720214366912842, + "code_tokens": 43, + "cvar95_kl": 16.15057945251465, + "delta_nll": 5.9084062576293945, + "max_kl": 16.952425003051758, + "mean_kl": 6.130138397216797, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.27906978130340576 + }, + { + "candidate_nll": 6.411502361297607, + "code_tokens": 35, + "cvar95_kl": 15.284128189086914, + "delta_nll": 4.937708854675293, + "max_kl": 17.2886962890625, + "mean_kl": 5.27147912979126, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2339", + "rounding": "stochastic", + "seed": 2339, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 8.861912727355957, + "median": 5.709590435028076, + "minimum": 2.5632131099700928, + "q25": 4.786999702453613, + "q75": 6.748898148536682 + }, + "task_macro": { + "candidate_nll": 6.864793533749051, + "delta_nll": 5.8080693191952175, + "mean_kl": 5.997225713729859, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.17772676431470447 + }, + "token_weighted": { + "candidate_nll": 7.539523601531982, + "cvar95_kl": 16.9124698638916, + "delta_nll": 6.611865043640137, + "max_kl": 23.92094612121582, + "mean_kl": 6.755039691925049, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.1448225975036621 + } + }, + "read_risk_l0_stochastic_seed_2340": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.463914073031882, + "macro_mean_kl": 4.633196996605915, + "macro_top1_agreement": 0.22921773402587228, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 5.413156913674396, + "macro_mean_kl": 5.6958610700524375, + "macro_top1_agreement": 0.18633922513412393, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 5.894749056209218, + "macro_mean_kl": 5.898994727568193, + "macro_top1_agreement": 0.16680739684538406, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 7.573567888953469, + "macro_mean_kl": 7.698049697009, + "macro_top1_agreement": 0.099289046600461, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 6.544124126434326, + "code_tokens": 63, + "cvar95_kl": 13.862344741821289, + "delta_nll": 5.95307731628418, + "max_kl": 15.61821174621582, + "mean_kl": 5.617649555206299, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 8.455533027648926, + "code_tokens": 92, + "cvar95_kl": 15.570950508117676, + "delta_nll": 7.4498090744018555, + "max_kl": 17.293659210205078, + "mean_kl": 7.50670862197876, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 7.025112628936768, + "code_tokens": 36, + "cvar95_kl": 11.026506423950195, + "delta_nll": 5.111623287200928, + "max_kl": 11.186152458190918, + "mean_kl": 5.554076194763184, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.030893325805664, + "code_tokens": 24, + "cvar95_kl": 12.02951431274414, + "delta_nll": 4.213484287261963, + "max_kl": 12.894341468811035, + "mean_kl": 5.324285984039307, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 8.910778999328613, + "code_tokens": 153, + "cvar95_kl": 17.31267547607422, + "delta_nll": 7.9938273429870605, + "max_kl": 20.348461151123047, + "mean_kl": 8.310751914978027, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.05882352963089943 + }, + { + "candidate_nll": 8.86522388458252, + "code_tokens": 133, + "cvar95_kl": 16.75077247619629, + "delta_nll": 7.798690319061279, + "max_kl": 17.613000869750977, + "mean_kl": 7.798819065093994, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.06766917556524277 + }, + { + "candidate_nll": 5.639773368835449, + "code_tokens": 39, + "cvar95_kl": 11.75048828125, + "delta_nll": 4.771740436553955, + "max_kl": 11.967806816101074, + "mean_kl": 4.5174102783203125, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 5.320810794830322, + "code_tokens": 26, + "cvar95_kl": 8.44737434387207, + "delta_nll": 4.059854984283447, + "max_kl": 8.551806449890137, + "mean_kl": 4.050045967102051, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.11538461595773697 + }, + { + "candidate_nll": 7.425384521484375, + "code_tokens": 38, + "cvar95_kl": 15.52159309387207, + "delta_nll": 6.109711647033691, + "max_kl": 15.94216251373291, + "mean_kl": 5.703563690185547, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.1315789520740509 + }, + { + "candidate_nll": 9.594624519348145, + "code_tokens": 173, + "cvar95_kl": 18.5823917388916, + "delta_nll": 8.740832328796387, + "max_kl": 22.00394630432129, + "mean_kl": 8.933487892150879, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.08670520037412643 + }, + { + "candidate_nll": 6.328518867492676, + "code_tokens": 37, + "cvar95_kl": 12.630167007446289, + "delta_nll": 5.411696434020996, + "max_kl": 14.225987434387207, + "mean_kl": 5.796911716461182, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 10.119414329528809, + "code_tokens": 205, + "cvar95_kl": 19.166893005371094, + "delta_nll": 9.30722427368164, + "max_kl": 20.818923950195312, + "mean_kl": 9.399170875549316, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.04878048598766327 + }, + { + "candidate_nll": 8.721358299255371, + "code_tokens": 162, + "cvar95_kl": 15.158294677734375, + "delta_nll": 7.822362422943115, + "max_kl": 16.16020393371582, + "mean_kl": 7.826670169830322, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.09876543283462524 + }, + { + "candidate_nll": 7.872762203216553, + "code_tokens": 132, + "cvar95_kl": 16.935070037841797, + "delta_nll": 7.210404396057129, + "max_kl": 18.978012084960938, + "mean_kl": 7.083163738250732, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.10606060922145844 + }, + { + "candidate_nll": 5.061439514160156, + "code_tokens": 33, + "cvar95_kl": 12.481465339660645, + "delta_nll": 4.394245624542236, + "max_kl": 13.154863357543945, + "mean_kl": 4.829336166381836, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 6.76341438293457, + "code_tokens": 60, + "cvar95_kl": 12.376751899719238, + "delta_nll": 5.421679496765137, + "max_kl": 12.400494575500488, + "mean_kl": 5.626031875610352, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.037529945373535, + "code_tokens": 63, + "cvar95_kl": 15.650035858154297, + "delta_nll": 6.6307244300842285, + "max_kl": 16.742094039916992, + "mean_kl": 6.577803611755371, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.1269841343164444 + }, + { + "candidate_nll": 6.745548248291016, + "code_tokens": 48, + "cvar95_kl": 13.477058410644531, + "delta_nll": 5.129417419433594, + "max_kl": 15.207723617553711, + "mean_kl": 5.621490955352783, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 6.530641078948975, + "code_tokens": 76, + "cvar95_kl": 17.6773681640625, + "delta_nll": 5.747957229614258, + "max_kl": 19.057653427124023, + "mean_kl": 5.746732234954834, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.19736842811107635 + }, + { + "candidate_nll": 5.570438861846924, + "code_tokens": 63, + "cvar95_kl": 11.855996131896973, + "delta_nll": 4.619344234466553, + "max_kl": 13.805729866027832, + "mean_kl": 4.618216037750244, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 8.898385047912598, + "code_tokens": 107, + "cvar95_kl": 17.935684204101562, + "delta_nll": 8.033651351928711, + "max_kl": 21.866119384765625, + "mean_kl": 8.581483840942383, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.13084112107753754 + }, + { + "candidate_nll": 8.434900283813477, + "code_tokens": 108, + "cvar95_kl": 19.93597984313965, + "delta_nll": 7.819727897644043, + "max_kl": 23.768007278442383, + "mean_kl": 8.15883731842041, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.10185185074806213 + }, + { + "candidate_nll": 7.218331813812256, + "code_tokens": 46, + "cvar95_kl": 15.737205505371094, + "delta_nll": 6.24500036239624, + "max_kl": 18.03778839111328, + "mean_kl": 6.830062389373779, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.19565217196941376 + }, + { + "candidate_nll": 6.5906596183776855, + "code_tokens": 37, + "cvar95_kl": 16.156002044677734, + "delta_nll": 5.661401748657227, + "max_kl": 16.18041229248047, + "mean_kl": 6.199398517608643, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.18918919563293457 + }, + { + "candidate_nll": 6.477595806121826, + "code_tokens": 29, + "cvar95_kl": 15.06338882446289, + "delta_nll": 5.325439453125, + "max_kl": 16.452280044555664, + "mean_kl": 5.246098518371582, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 4.3433074951171875, + "code_tokens": 13, + "cvar95_kl": 6.465516090393066, + "delta_nll": 2.729177951812744, + "max_kl": 6.465516090393066, + "mean_kl": 2.9933958053588867, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 6.567824840545654, + "code_tokens": 51, + "cvar95_kl": 12.952701568603516, + "delta_nll": 5.944732666015625, + "max_kl": 14.26248836517334, + "mean_kl": 5.884134292602539, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.13725490868091583 + }, + { + "candidate_nll": 6.4205732345581055, + "code_tokens": 29, + "cvar95_kl": 16.533966064453125, + "delta_nll": 5.2901482582092285, + "max_kl": 18.695096969604492, + "mean_kl": 5.720642566680908, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 6.835206985473633, + "code_tokens": 30, + "cvar95_kl": 11.594429969787598, + "delta_nll": 5.805611610412598, + "max_kl": 12.57662582397461, + "mean_kl": 5.49409294128418, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.23333333432674408 + }, + { + "candidate_nll": 7.6988019943237305, + "code_tokens": 103, + "cvar95_kl": 16.842988967895508, + "delta_nll": 6.908063888549805, + "max_kl": 20.97039222717285, + "mean_kl": 7.190357208251953, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.10679611563682556 + }, + { + "candidate_nll": 8.375335693359375, + "code_tokens": 119, + "cvar95_kl": 16.79197120666504, + "delta_nll": 7.474791526794434, + "max_kl": 19.50111198425293, + "mean_kl": 7.536040782928467, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.07563025504350662 + }, + { + "candidate_nll": 5.001574516296387, + "code_tokens": 22, + "cvar95_kl": 10.717607498168945, + "delta_nll": 3.7047038078308105, + "max_kl": 11.278826713562012, + "mean_kl": 3.997922658920288, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.22727273404598236 + }, + { + "candidate_nll": 5.772824287414551, + "code_tokens": 51, + "cvar95_kl": 13.502449035644531, + "delta_nll": 5.134946346282959, + "max_kl": 14.244786262512207, + "mean_kl": 5.160031795501709, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.11764705926179886 + }, + { + "candidate_nll": 7.105396747589111, + "code_tokens": 32, + "cvar95_kl": 14.361690521240234, + "delta_nll": 5.345848083496094, + "max_kl": 14.859716415405273, + "mean_kl": 5.968695640563965, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.1875 + }, + { + "candidate_nll": 8.240540504455566, + "code_tokens": 116, + "cvar95_kl": 17.30885124206543, + "delta_nll": 7.177239894866943, + "max_kl": 18.501216888427734, + "mean_kl": 7.573566913604736, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.07758620381355286 + }, + { + "candidate_nll": 6.006852626800537, + "code_tokens": 56, + "cvar95_kl": 14.111733436584473, + "delta_nll": 5.221667766571045, + "max_kl": 16.211132049560547, + "mean_kl": 5.035096168518066, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 6.930572032928467, + "code_tokens": 53, + "cvar95_kl": 12.677935600280762, + "delta_nll": 5.9826979637146, + "max_kl": 13.73661994934082, + "mean_kl": 5.644200801849365, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.16981132328510284 + }, + { + "candidate_nll": 9.192941665649414, + "code_tokens": 127, + "cvar95_kl": 19.304113388061523, + "delta_nll": 8.581720352172852, + "max_kl": 21.178003311157227, + "mean_kl": 8.427864074707031, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.12598425149917603 + }, + { + "candidate_nll": 6.652094841003418, + "code_tokens": 61, + "cvar95_kl": 12.399677276611328, + "delta_nll": 5.439560413360596, + "max_kl": 14.179699897766113, + "mean_kl": 5.705773830413818, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.13114753365516663 + }, + { + "candidate_nll": 9.8593168258667, + "code_tokens": 147, + "cvar95_kl": 20.010303497314453, + "delta_nll": 9.313321113586426, + "max_kl": 21.414592742919922, + "mean_kl": 9.44106674194336, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.054421767592430115 + }, + { + "candidate_nll": 6.15283203125, + "code_tokens": 28, + "cvar95_kl": 11.102852821350098, + "delta_nll": 4.6748456954956055, + "max_kl": 11.409460067749023, + "mean_kl": 5.0397539138793945, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.2142857164144516 + }, + { + "candidate_nll": 7.221923828125, + "code_tokens": 66, + "cvar95_kl": 16.699186325073242, + "delta_nll": 6.38555383682251, + "max_kl": 17.707305908203125, + "mean_kl": 6.727912902832031, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.206398963928223, + "code_tokens": 33, + "cvar95_kl": 15.503710746765137, + "delta_nll": 5.379594326019287, + "max_kl": 17.050498962402344, + "mean_kl": 6.483284950256348, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.1515151560306549 + }, + { + "candidate_nll": 6.634546756744385, + "code_tokens": 41, + "cvar95_kl": 15.142973899841309, + "delta_nll": 5.496847629547119, + "max_kl": 16.565521240234375, + "mean_kl": 5.4499006271362305, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.19512194395065308 + }, + { + "candidate_nll": 5.86331844329834, + "code_tokens": 36, + "cvar95_kl": 13.104535102844238, + "delta_nll": 4.187140464782715, + "max_kl": 13.271939277648926, + "mean_kl": 4.528860092163086, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 8.615726470947266, + "code_tokens": 70, + "cvar95_kl": 20.188676834106445, + "delta_nll": 7.698591709136963, + "max_kl": 24.250560760498047, + "mean_kl": 7.2894463539123535, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.10000000149011612 + }, + { + "candidate_nll": 4.647222518920898, + "code_tokens": 17, + "cvar95_kl": 10.942117691040039, + "delta_nll": 3.3477580547332764, + "max_kl": 10.942117691040039, + "mean_kl": 4.300841331481934, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 6.772756099700928, + "code_tokens": 23, + "cvar95_kl": 13.145456314086914, + "delta_nll": 5.587548732757568, + "max_kl": 13.311027526855469, + "mean_kl": 5.211615562438965, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 7.178734302520752, + "code_tokens": 101, + "cvar95_kl": 14.722602844238281, + "delta_nll": 6.80086088180542, + "max_kl": 16.050806045532227, + "mean_kl": 6.830381393432617, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.11881188303232193 + }, + { + "candidate_nll": 7.182323932647705, + "code_tokens": 36, + "cvar95_kl": 13.253531455993652, + "delta_nll": 5.767447471618652, + "max_kl": 13.452710151672363, + "mean_kl": 6.340400695800781, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 6.9880690574646, + "code_tokens": 46, + "cvar95_kl": 14.47664737701416, + "delta_nll": 5.782315254211426, + "max_kl": 15.050168991088867, + "mean_kl": 5.996487617492676, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.1304347813129425 + }, + { + "candidate_nll": 6.012237071990967, + "code_tokens": 24, + "cvar95_kl": 11.188422203063965, + "delta_nll": 4.481390476226807, + "max_kl": 12.187289237976074, + "mean_kl": 4.976369380950928, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 5.909796237945557, + "code_tokens": 22, + "cvar95_kl": 10.684364318847656, + "delta_nll": 4.534466743469238, + "max_kl": 11.256509780883789, + "mean_kl": 4.646027565002441, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 6.376093864440918, + "code_tokens": 58, + "cvar95_kl": 15.074664115905762, + "delta_nll": 5.687221527099609, + "max_kl": 15.691648483276367, + "mean_kl": 5.6468305587768555, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 4.81688928604126, + "code_tokens": 15, + "cvar95_kl": 9.629343032836914, + "delta_nll": 3.504018783569336, + "max_kl": 9.629343032836914, + "mean_kl": 3.6989402770996094, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.324522972106934, + "code_tokens": 40, + "cvar95_kl": 13.64777946472168, + "delta_nll": 3.8849353790283203, + "max_kl": 14.735072135925293, + "mean_kl": 3.864968776702881, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 6.567628383636475, + "code_tokens": 37, + "cvar95_kl": 13.623196601867676, + "delta_nll": 5.96307897567749, + "max_kl": 13.887578010559082, + "mean_kl": 6.067342758178711, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.13513512909412384 + }, + { + "candidate_nll": 5.615818023681641, + "code_tokens": 21, + "cvar95_kl": 9.288825988769531, + "delta_nll": 4.595571994781494, + "max_kl": 10.013643264770508, + "mean_kl": 4.543464660644531, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 5.588283061981201, + "code_tokens": 21, + "cvar95_kl": 9.833072662353516, + "delta_nll": 4.2757673263549805, + "max_kl": 11.227319717407227, + "mean_kl": 3.823406457901001, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 7.255352020263672, + "code_tokens": 66, + "cvar95_kl": 15.36340045928955, + "delta_nll": 6.246987342834473, + "max_kl": 16.776866912841797, + "mean_kl": 6.564784049987793, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.1515151560306549 + }, + { + "candidate_nll": 8.356356620788574, + "code_tokens": 138, + "cvar95_kl": 16.952184677124023, + "delta_nll": 7.712696075439453, + "max_kl": 18.173086166381836, + "mean_kl": 7.616617679595947, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.08695652335882187 + }, + { + "candidate_nll": 6.052218437194824, + "code_tokens": 22, + "cvar95_kl": 14.503508567810059, + "delta_nll": 5.317975997924805, + "max_kl": 15.947765350341797, + "mean_kl": 4.692487716674805, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 6.124895095825195, + "code_tokens": 37, + "cvar95_kl": 11.807100296020508, + "delta_nll": 5.379310607910156, + "max_kl": 12.681122779846191, + "mean_kl": 5.1365742683410645, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.29729729890823364 + }, + { + "candidate_nll": 6.2969069480896, + "code_tokens": 41, + "cvar95_kl": 12.030006408691406, + "delta_nll": 5.346149444580078, + "max_kl": 12.931090354919434, + "mean_kl": 5.396070957183838, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.19512194395065308 + }, + { + "candidate_nll": 6.036737442016602, + "code_tokens": 63, + "cvar95_kl": 12.719291687011719, + "delta_nll": 4.878246784210205, + "max_kl": 16.127567291259766, + "mean_kl": 4.97087287902832, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 6.9714202880859375, + "code_tokens": 99, + "cvar95_kl": 14.812997817993164, + "delta_nll": 6.124668598175049, + "max_kl": 15.189360618591309, + "mean_kl": 6.181728363037109, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.1515151560306549 + }, + { + "candidate_nll": 6.770676612854004, + "code_tokens": 65, + "cvar95_kl": 14.433212280273438, + "delta_nll": 5.840582847595215, + "max_kl": 18.473146438598633, + "mean_kl": 5.7059454917907715, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.1846153885126114 + }, + { + "candidate_nll": 7.533798694610596, + "code_tokens": 39, + "cvar95_kl": 17.107776641845703, + "delta_nll": 6.616269588470459, + "max_kl": 18.62160873413086, + "mean_kl": 6.800552845001221, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.12820513546466827 + }, + { + "candidate_nll": 7.271509647369385, + "code_tokens": 40, + "cvar95_kl": 17.776500701904297, + "delta_nll": 6.112749099731445, + "max_kl": 18.40268898010254, + "mean_kl": 6.285770416259766, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.15000000596046448 + }, + { + "candidate_nll": 8.756146430969238, + "code_tokens": 97, + "cvar95_kl": 17.268871307373047, + "delta_nll": 7.461194038391113, + "max_kl": 19.2891845703125, + "mean_kl": 7.633471488952637, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.10309278219938278 + }, + { + "candidate_nll": 7.0157790184021, + "code_tokens": 23, + "cvar95_kl": 14.014749526977539, + "delta_nll": 5.100749969482422, + "max_kl": 14.061195373535156, + "mean_kl": 5.506932735443115, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 5.364053726196289, + "code_tokens": 25, + "cvar95_kl": 10.611160278320312, + "delta_nll": 4.330183982849121, + "max_kl": 10.844087600708008, + "mean_kl": 4.641603469848633, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.332075119018555, + "code_tokens": 57, + "cvar95_kl": 16.121124267578125, + "delta_nll": 6.476546287536621, + "max_kl": 17.34490966796875, + "mean_kl": 6.045762538909912, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.3763251304626465, + "code_tokens": 50, + "cvar95_kl": 15.217066764831543, + "delta_nll": 6.3340253829956055, + "max_kl": 17.083465576171875, + "mean_kl": 6.595113754272461, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.1599999964237213 + }, + { + "candidate_nll": 7.739337921142578, + "code_tokens": 35, + "cvar95_kl": 16.167438507080078, + "delta_nll": 6.910754203796387, + "max_kl": 16.674245834350586, + "mean_kl": 6.815474987030029, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.17142857611179352 + }, + { + "candidate_nll": 6.171323776245117, + "code_tokens": 29, + "cvar95_kl": 9.441194534301758, + "delta_nll": 3.826910972595215, + "max_kl": 9.691383361816406, + "mean_kl": 4.329409599304199, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.17241379618644714 + }, + { + "candidate_nll": 5.680327415466309, + "code_tokens": 20, + "cvar95_kl": 14.343515396118164, + "delta_nll": 4.346909523010254, + "max_kl": 14.343515396118164, + "mean_kl": 4.493745803833008, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 6.3522114753723145, + "code_tokens": 19, + "cvar95_kl": 13.624517440795898, + "delta_nll": 5.235568523406982, + "max_kl": 13.624517440795898, + "mean_kl": 5.338731288909912, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.15789473056793213 + }, + { + "candidate_nll": 7.005715847015381, + "code_tokens": 71, + "cvar95_kl": 15.510259628295898, + "delta_nll": 6.059432029724121, + "max_kl": 17.93260955810547, + "mean_kl": 6.188913822174072, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.1690140813589096 + }, + { + "candidate_nll": 7.047274112701416, + "code_tokens": 47, + "cvar95_kl": 14.95253849029541, + "delta_nll": 5.312381267547607, + "max_kl": 16.213239669799805, + "mean_kl": 5.787436008453369, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.1702127605676651 + }, + { + "candidate_nll": 6.289481163024902, + "code_tokens": 50, + "cvar95_kl": 17.490097045898438, + "delta_nll": 5.4874587059021, + "max_kl": 18.819889068603516, + "mean_kl": 5.451233386993408, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 7.73956298828125, + "code_tokens": 113, + "cvar95_kl": 17.125280380249023, + "delta_nll": 6.6876020431518555, + "max_kl": 19.34027671813965, + "mean_kl": 6.738039493560791, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.10619468986988068 + }, + { + "candidate_nll": 6.444066524505615, + "code_tokens": 26, + "cvar95_kl": 11.19009017944336, + "delta_nll": 5.003908634185791, + "max_kl": 13.027620315551758, + "mean_kl": 4.917722225189209, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.11538461595773697 + }, + { + "candidate_nll": 7.932286262512207, + "code_tokens": 62, + "cvar95_kl": 14.676658630371094, + "delta_nll": 7.364027500152588, + "max_kl": 15.060540199279785, + "mean_kl": 7.353907108306885, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.08064515888690948 + }, + { + "candidate_nll": 7.0430073738098145, + "code_tokens": 79, + "cvar95_kl": 14.108237266540527, + "delta_nll": 6.290288925170898, + "max_kl": 16.09196662902832, + "mean_kl": 6.464272975921631, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.10126582533121109 + }, + { + "candidate_nll": 5.00667667388916, + "code_tokens": 20, + "cvar95_kl": 8.104409217834473, + "delta_nll": 3.37802791595459, + "max_kl": 8.104409217834473, + "mean_kl": 3.5759944915771484, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.807407379150391, + "code_tokens": 186, + "cvar95_kl": 18.384748458862305, + "delta_nll": 7.404263019561768, + "max_kl": 21.049745559692383, + "mean_kl": 7.556021213531494, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.14516128599643707 + }, + { + "candidate_nll": 7.3484015464782715, + "code_tokens": 99, + "cvar95_kl": 15.291760444641113, + "delta_nll": 6.505253791809082, + "max_kl": 16.367382049560547, + "mean_kl": 6.5685715675354, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.10101009905338287 + }, + { + "candidate_nll": 7.056305408477783, + "code_tokens": 43, + "cvar95_kl": 17.826223373413086, + "delta_nll": 6.244497299194336, + "max_kl": 18.362083435058594, + "mean_kl": 6.610915660858154, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.20930232107639313 + }, + { + "candidate_nll": 4.541664123535156, + "code_tokens": 35, + "cvar95_kl": 12.745595932006836, + "delta_nll": 3.0678703784942627, + "max_kl": 13.448343276977539, + "mean_kl": 4.041309356689453, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.3142857253551483 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2340", + "rounding": "stochastic", + "seed": 2340, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 9.313321113586426, + "median": 5.717589378356934, + "minimum": 2.729177951812744, + "q25": 5.028118968009949, + "q75": 6.627110719680786 + }, + "task_macro": { + "candidate_nll": 6.873119825787015, + "delta_nll": 5.816395616531372, + "mean_kl": 5.96337014304267, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.17124368684987226 + }, + "token_weighted": { + "candidate_nll": 7.550448894500732, + "cvar95_kl": 17.00991439819336, + "delta_nll": 6.622790336608887, + "max_kl": 24.250560760498047, + "mean_kl": 6.742859840393066, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.13939175009727478 + } + }, + "read_risk_l0_stochastic_seed_2341": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 4.317051224086596, + "macro_mean_kl": 4.3623347489730175, + "macro_top1_agreement": 0.2924489689909894, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 5.3211900565935215, + "macro_mean_kl": 5.58789993369061, + "macro_top1_agreement": 0.21104334165220676, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 6.005554480986162, + "macro_mean_kl": 6.045758225701072, + "macro_top1_agreement": 0.16381050578572534, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 7.75425910949707, + "macro_mean_kl": 7.788081190802834, + "macro_top1_agreement": 0.09302508373829452, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 6.419129371643066, + "code_tokens": 63, + "cvar95_kl": 12.40610408782959, + "delta_nll": 5.82808256149292, + "max_kl": 12.984889030456543, + "mean_kl": 5.608553409576416, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 8.238696098327637, + "code_tokens": 92, + "cvar95_kl": 15.682500839233398, + "delta_nll": 7.232972145080566, + "max_kl": 17.2144718170166, + "mean_kl": 7.1876983642578125, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.06521739065647125 + }, + { + "candidate_nll": 6.54990291595459, + "code_tokens": 36, + "cvar95_kl": 12.479930877685547, + "delta_nll": 4.63641357421875, + "max_kl": 12.500532150268555, + "mean_kl": 5.3236985206604, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 5.2655792236328125, + "code_tokens": 24, + "cvar95_kl": 11.513696670532227, + "delta_nll": 3.4481701850891113, + "max_kl": 12.046982765197754, + "mean_kl": 4.169482707977295, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 8.986047744750977, + "code_tokens": 153, + "cvar95_kl": 15.601425170898438, + "delta_nll": 8.069095611572266, + "max_kl": 17.951047897338867, + "mean_kl": 8.256019592285156, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.07189542800188065 + }, + { + "candidate_nll": 9.216503143310547, + "code_tokens": 133, + "cvar95_kl": 17.414255142211914, + "delta_nll": 8.149969100952148, + "max_kl": 18.54424476623535, + "mean_kl": 8.294785499572754, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.07518796622753143 + }, + { + "candidate_nll": 5.785990238189697, + "code_tokens": 39, + "cvar95_kl": 10.018486022949219, + "delta_nll": 4.917957305908203, + "max_kl": 10.064337730407715, + "mean_kl": 4.644176959991455, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.1794871836900711 + }, + { + "candidate_nll": 4.6434245109558105, + "code_tokens": 26, + "cvar95_kl": 8.403282165527344, + "delta_nll": 3.3824687004089355, + "max_kl": 8.69671630859375, + "mean_kl": 3.3389971256256104, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.19230769574642181 + }, + { + "candidate_nll": 7.348623275756836, + "code_tokens": 38, + "cvar95_kl": 15.792366027832031, + "delta_nll": 6.032950401306152, + "max_kl": 16.475811004638672, + "mean_kl": 6.082019805908203, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 8.628329277038574, + "code_tokens": 173, + "cvar95_kl": 16.719295501708984, + "delta_nll": 7.774537086486816, + "max_kl": 19.260597229003906, + "mean_kl": 7.942739963531494, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.09826589375734329 + }, + { + "candidate_nll": 6.776033878326416, + "code_tokens": 37, + "cvar95_kl": 14.760622024536133, + "delta_nll": 5.859210968017578, + "max_kl": 15.481344223022461, + "mean_kl": 6.082396984100342, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.1621621549129486 + }, + { + "candidate_nll": 9.768095016479492, + "code_tokens": 205, + "cvar95_kl": 19.914522171020508, + "delta_nll": 8.955904960632324, + "max_kl": 27.61359405517578, + "mean_kl": 8.87096118927002, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.08292683213949203 + }, + { + "candidate_nll": 9.56509780883789, + "code_tokens": 162, + "cvar95_kl": 20.799846649169922, + "delta_nll": 8.666101455688477, + "max_kl": 24.983280181884766, + "mean_kl": 8.966395378112793, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.08641975373029709 + }, + { + "candidate_nll": 8.636388778686523, + "code_tokens": 132, + "cvar95_kl": 16.148151397705078, + "delta_nll": 7.974031448364258, + "max_kl": 22.166091918945312, + "mean_kl": 7.9336419105529785, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.12878787517547607 + }, + { + "candidate_nll": 6.236004829406738, + "code_tokens": 33, + "cvar95_kl": 13.14552116394043, + "delta_nll": 5.568810939788818, + "max_kl": 13.404152870178223, + "mean_kl": 5.517482757568359, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.1818181872367859 + }, + { + "candidate_nll": 7.293123245239258, + "code_tokens": 60, + "cvar95_kl": 17.062591552734375, + "delta_nll": 5.951388359069824, + "max_kl": 20.675092697143555, + "mean_kl": 6.466798782348633, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.18333333730697632 + }, + { + "candidate_nll": 7.2079668045043945, + "code_tokens": 63, + "cvar95_kl": 15.05526351928711, + "delta_nll": 6.801161289215088, + "max_kl": 16.18248176574707, + "mean_kl": 6.734397888183594, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.190476194024086 + }, + { + "candidate_nll": 6.584774017333984, + "code_tokens": 48, + "cvar95_kl": 12.449204444885254, + "delta_nll": 4.9686431884765625, + "max_kl": 13.142480850219727, + "mean_kl": 5.334798812866211, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 6.300167083740234, + "code_tokens": 76, + "cvar95_kl": 13.842445373535156, + "delta_nll": 5.517482757568359, + "max_kl": 15.769675254821777, + "mean_kl": 5.478440761566162, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.18421052396297455 + }, + { + "candidate_nll": 6.012965679168701, + "code_tokens": 63, + "cvar95_kl": 10.926203727722168, + "delta_nll": 5.06187105178833, + "max_kl": 11.964547157287598, + "mean_kl": 4.99922513961792, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.1269841343164444 + }, + { + "candidate_nll": 8.070700645446777, + "code_tokens": 107, + "cvar95_kl": 17.081750869750977, + "delta_nll": 7.205966472625732, + "max_kl": 18.040536880493164, + "mean_kl": 7.4388017654418945, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.09345794469118118 + }, + { + "candidate_nll": 7.978504657745361, + "code_tokens": 108, + "cvar95_kl": 18.399642944335938, + "delta_nll": 7.3633317947387695, + "max_kl": 25.84657096862793, + "mean_kl": 7.647286415100098, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.12037037312984467 + }, + { + "candidate_nll": 6.183456897735596, + "code_tokens": 46, + "cvar95_kl": 13.818303108215332, + "delta_nll": 5.21012544631958, + "max_kl": 15.168720245361328, + "mean_kl": 6.109142780303955, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 6.913670539855957, + "code_tokens": 37, + "cvar95_kl": 15.495148658752441, + "delta_nll": 5.984412670135498, + "max_kl": 15.956964492797852, + "mean_kl": 6.203413486480713, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.2702702581882477 + }, + { + "candidate_nll": 7.215846538543701, + "code_tokens": 29, + "cvar95_kl": 16.59461212158203, + "delta_nll": 6.063690185546875, + "max_kl": 17.804880142211914, + "mean_kl": 6.270417213439941, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 5.119845390319824, + "code_tokens": 13, + "cvar95_kl": 7.626054286956787, + "delta_nll": 3.505715847015381, + "max_kl": 7.626054286956787, + "mean_kl": 2.5961644649505615, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.4615384638309479 + }, + { + "candidate_nll": 5.959805965423584, + "code_tokens": 51, + "cvar95_kl": 14.91170597076416, + "delta_nll": 5.336713790893555, + "max_kl": 18.38939666748047, + "mean_kl": 5.0080060958862305, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.2549019753932953 + }, + { + "candidate_nll": 6.41758394241333, + "code_tokens": 29, + "cvar95_kl": 14.565505027770996, + "delta_nll": 5.287158966064453, + "max_kl": 16.527746200561523, + "mean_kl": 5.647218704223633, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 5.94488525390625, + "code_tokens": 30, + "cvar95_kl": 10.714733123779297, + "delta_nll": 4.915289878845215, + "max_kl": 11.116127014160156, + "mean_kl": 4.208693504333496, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 8.595977783203125, + "code_tokens": 103, + "cvar95_kl": 18.304203033447266, + "delta_nll": 7.805239677429199, + "max_kl": 22.246845245361328, + "mean_kl": 7.792548179626465, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.06796116381883621 + }, + { + "candidate_nll": 7.949257850646973, + "code_tokens": 119, + "cvar95_kl": 16.378711700439453, + "delta_nll": 7.048713684082031, + "max_kl": 18.70162010192871, + "mean_kl": 7.219964504241943, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.10924369841814041 + }, + { + "candidate_nll": 5.785362720489502, + "code_tokens": 22, + "cvar95_kl": 11.748210906982422, + "delta_nll": 4.488492012023926, + "max_kl": 13.247370719909668, + "mean_kl": 4.5318098068237305, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.3636363744735718 + }, + { + "candidate_nll": 5.49396276473999, + "code_tokens": 51, + "cvar95_kl": 11.291810035705566, + "delta_nll": 4.856084823608398, + "max_kl": 11.498564720153809, + "mean_kl": 4.680813789367676, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.21568627655506134 + }, + { + "candidate_nll": 7.30900764465332, + "code_tokens": 32, + "cvar95_kl": 14.452192306518555, + "delta_nll": 5.549458980560303, + "max_kl": 14.98080825805664, + "mean_kl": 5.989202499389648, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.28125 + }, + { + "candidate_nll": 7.654016017913818, + "code_tokens": 116, + "cvar95_kl": 15.452423095703125, + "delta_nll": 6.590715408325195, + "max_kl": 16.782594680786133, + "mean_kl": 6.527286052703857, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.07758620381355286 + }, + { + "candidate_nll": 6.703216552734375, + "code_tokens": 56, + "cvar95_kl": 12.473861694335938, + "delta_nll": 5.918031692504883, + "max_kl": 13.236404418945312, + "mean_kl": 5.6949052810668945, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 6.027978420257568, + "code_tokens": 53, + "cvar95_kl": 11.499445915222168, + "delta_nll": 5.080104351043701, + "max_kl": 13.716387748718262, + "mean_kl": 5.310311317443848, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.1320754736661911 + }, + { + "candidate_nll": 9.451435089111328, + "code_tokens": 127, + "cvar95_kl": 18.212690353393555, + "delta_nll": 8.840213775634766, + "max_kl": 19.68141746520996, + "mean_kl": 8.71788501739502, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.12598425149917603 + }, + { + "candidate_nll": 6.288944721221924, + "code_tokens": 61, + "cvar95_kl": 12.281949996948242, + "delta_nll": 5.076410293579102, + "max_kl": 13.665770530700684, + "mean_kl": 5.085712909698486, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.14754098653793335 + }, + { + "candidate_nll": 8.979199409484863, + "code_tokens": 147, + "cvar95_kl": 19.402860641479492, + "delta_nll": 8.43320369720459, + "max_kl": 22.260225296020508, + "mean_kl": 8.459488868713379, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.08163265138864517 + }, + { + "candidate_nll": 6.637240886688232, + "code_tokens": 28, + "cvar95_kl": 11.963064193725586, + "delta_nll": 5.159254550933838, + "max_kl": 12.958780288696289, + "mean_kl": 5.508810997009277, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 7.082012176513672, + "code_tokens": 66, + "cvar95_kl": 17.194538116455078, + "delta_nll": 6.245642185211182, + "max_kl": 18.042171478271484, + "mean_kl": 6.610511302947998, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.1666666716337204 + }, + { + "candidate_nll": 7.3039631843566895, + "code_tokens": 33, + "cvar95_kl": 16.319076538085938, + "delta_nll": 5.477158546447754, + "max_kl": 17.36894416809082, + "mean_kl": 6.339298248291016, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.21212121844291687 + }, + { + "candidate_nll": 7.320053577423096, + "code_tokens": 41, + "cvar95_kl": 15.70119571685791, + "delta_nll": 6.18235445022583, + "max_kl": 16.21238136291504, + "mean_kl": 6.332803249359131, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.1463414579629898 + }, + { + "candidate_nll": 6.577307224273682, + "code_tokens": 36, + "cvar95_kl": 11.269817352294922, + "delta_nll": 4.901129722595215, + "max_kl": 11.774816513061523, + "mean_kl": 4.681329250335693, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 7.184139728546143, + "code_tokens": 70, + "cvar95_kl": 14.130422592163086, + "delta_nll": 6.26700496673584, + "max_kl": 16.412437438964844, + "mean_kl": 6.097937107086182, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.15714286267757416 + }, + { + "candidate_nll": 5.103997230529785, + "code_tokens": 17, + "cvar95_kl": 8.961750030517578, + "delta_nll": 3.804532766342163, + "max_kl": 8.961750030517578, + "mean_kl": 3.524629831314087, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.3529411852359772 + }, + { + "candidate_nll": 6.189363479614258, + "code_tokens": 23, + "cvar95_kl": 11.204545974731445, + "delta_nll": 5.004156112670898, + "max_kl": 12.607990264892578, + "mean_kl": 5.020361423492432, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.21739129722118378 + }, + { + "candidate_nll": 9.296629905700684, + "code_tokens": 101, + "cvar95_kl": 18.42713165283203, + "delta_nll": 8.918756484985352, + "max_kl": 21.393810272216797, + "mean_kl": 8.86591911315918, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.09900990128517151 + }, + { + "candidate_nll": 5.809181213378906, + "code_tokens": 36, + "cvar95_kl": 12.30173397064209, + "delta_nll": 4.394304275512695, + "max_kl": 13.769633293151855, + "mean_kl": 4.983697891235352, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.1388888955116272 + }, + { + "candidate_nll": 6.716891288757324, + "code_tokens": 46, + "cvar95_kl": 15.525394439697266, + "delta_nll": 5.51113748550415, + "max_kl": 15.957758903503418, + "mean_kl": 5.663649082183838, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.19565217196941376 + }, + { + "candidate_nll": 5.577081680297852, + "code_tokens": 24, + "cvar95_kl": 11.011754989624023, + "delta_nll": 4.046235084533691, + "max_kl": 13.544897079467773, + "mean_kl": 3.879046678543091, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.2083333283662796 + }, + { + "candidate_nll": 5.952527046203613, + "code_tokens": 22, + "cvar95_kl": 12.153968811035156, + "delta_nll": 4.577198028564453, + "max_kl": 12.168365478515625, + "mean_kl": 4.820488452911377, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.3181818127632141 + }, + { + "candidate_nll": 7.871956825256348, + "code_tokens": 58, + "cvar95_kl": 15.272567749023438, + "delta_nll": 7.183084487915039, + "max_kl": 17.663747787475586, + "mean_kl": 7.176721572875977, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.13793103396892548 + }, + { + "candidate_nll": 4.900137424468994, + "code_tokens": 15, + "cvar95_kl": 9.127665519714355, + "delta_nll": 3.5872669219970703, + "max_kl": 9.127665519714355, + "mean_kl": 3.4128148555755615, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.907089710235596, + "code_tokens": 40, + "cvar95_kl": 12.722107887268066, + "delta_nll": 4.467502117156982, + "max_kl": 13.448787689208984, + "mean_kl": 4.35037088394165, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.2750000059604645 + }, + { + "candidate_nll": 6.4558424949646, + "code_tokens": 37, + "cvar95_kl": 14.366979598999023, + "delta_nll": 5.851293087005615, + "max_kl": 14.742392539978027, + "mean_kl": 5.755148887634277, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 5.3979620933532715, + "code_tokens": 21, + "cvar95_kl": 10.962966918945312, + "delta_nll": 4.377716064453125, + "max_kl": 12.15998649597168, + "mean_kl": 4.420843601226807, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.565592288970947, + "code_tokens": 21, + "cvar95_kl": 10.510967254638672, + "delta_nll": 3.2530765533447266, + "max_kl": 12.185391426086426, + "mean_kl": 3.389596462249756, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 8.174182891845703, + "code_tokens": 66, + "cvar95_kl": 16.33708381652832, + "delta_nll": 7.165818214416504, + "max_kl": 19.805688858032227, + "mean_kl": 7.546159744262695, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.12121212482452393 + }, + { + "candidate_nll": 8.787983894348145, + "code_tokens": 138, + "cvar95_kl": 16.437694549560547, + "delta_nll": 8.144323348999023, + "max_kl": 18.114715576171875, + "mean_kl": 8.131980895996094, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.050724636763334274 + }, + { + "candidate_nll": 5.4471211433410645, + "code_tokens": 22, + "cvar95_kl": 10.746257781982422, + "delta_nll": 4.712878704071045, + "max_kl": 12.030619621276855, + "mean_kl": 4.264645099639893, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 6.047444820404053, + "code_tokens": 37, + "cvar95_kl": 15.397395133972168, + "delta_nll": 5.301860332489014, + "max_kl": 18.22244644165039, + "mean_kl": 5.36387300491333, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.21621622145175934 + }, + { + "candidate_nll": 6.259530544281006, + "code_tokens": 41, + "cvar95_kl": 14.721848487854004, + "delta_nll": 5.308773040771484, + "max_kl": 15.431343078613281, + "mean_kl": 5.448766231536865, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.2195121943950653 + }, + { + "candidate_nll": 7.537677764892578, + "code_tokens": 63, + "cvar95_kl": 15.07811164855957, + "delta_nll": 6.379187107086182, + "max_kl": 18.303678512573242, + "mean_kl": 6.682464599609375, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.1428571492433548 + }, + { + "candidate_nll": 7.678191661834717, + "code_tokens": 99, + "cvar95_kl": 16.373516082763672, + "delta_nll": 6.831439971923828, + "max_kl": 18.673660278320312, + "mean_kl": 6.9058518409729, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.1111111119389534 + }, + { + "candidate_nll": 5.201188564300537, + "code_tokens": 65, + "cvar95_kl": 11.61126708984375, + "delta_nll": 4.271094799041748, + "max_kl": 13.693753242492676, + "mean_kl": 3.9785714149475098, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.26153847575187683 + }, + { + "candidate_nll": 7.111242771148682, + "code_tokens": 39, + "cvar95_kl": 15.378898620605469, + "delta_nll": 6.193713665008545, + "max_kl": 16.800689697265625, + "mean_kl": 6.392470836639404, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.20512820780277252 + }, + { + "candidate_nll": 6.280364513397217, + "code_tokens": 40, + "cvar95_kl": 17.79937171936035, + "delta_nll": 5.121603965759277, + "max_kl": 19.271589279174805, + "mean_kl": 5.937577724456787, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.17499999701976776 + }, + { + "candidate_nll": 8.892420768737793, + "code_tokens": 97, + "cvar95_kl": 16.90801239013672, + "delta_nll": 7.597468376159668, + "max_kl": 18.71505355834961, + "mean_kl": 7.547543048858643, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.07216494530439377 + }, + { + "candidate_nll": 5.488603591918945, + "code_tokens": 23, + "cvar95_kl": 8.98811149597168, + "delta_nll": 3.5735747814178467, + "max_kl": 9.772472381591797, + "mean_kl": 3.9514544010162354, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 6.016584873199463, + "code_tokens": 25, + "cvar95_kl": 13.413265228271484, + "delta_nll": 4.982715129852295, + "max_kl": 14.869283676147461, + "mean_kl": 5.328226089477539, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.23999999463558197 + }, + { + "candidate_nll": 7.510341644287109, + "code_tokens": 57, + "cvar95_kl": 15.000572204589844, + "delta_nll": 6.654812812805176, + "max_kl": 16.126415252685547, + "mean_kl": 6.3289103507995605, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.17543859779834747 + }, + { + "candidate_nll": 8.988350868225098, + "code_tokens": 50, + "cvar95_kl": 19.393999099731445, + "delta_nll": 7.946051120758057, + "max_kl": 21.469594955444336, + "mean_kl": 8.302715301513672, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 6.60241174697876, + "code_tokens": 35, + "cvar95_kl": 13.74036979675293, + "delta_nll": 5.773828029632568, + "max_kl": 14.810892105102539, + "mean_kl": 5.851193904876709, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 6.1070556640625, + "code_tokens": 29, + "cvar95_kl": 9.973566055297852, + "delta_nll": 3.7626428604125977, + "max_kl": 10.531097412109375, + "mean_kl": 4.432506561279297, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.2068965584039688 + }, + { + "candidate_nll": 5.080870628356934, + "code_tokens": 20, + "cvar95_kl": 13.713451385498047, + "delta_nll": 3.747452735900879, + "max_kl": 13.713451385498047, + "mean_kl": 4.047869682312012, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 5.891643524169922, + "code_tokens": 19, + "cvar95_kl": 14.826623916625977, + "delta_nll": 4.77500057220459, + "max_kl": 14.826623916625977, + "mean_kl": 4.539501190185547, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 6.914092540740967, + "code_tokens": 71, + "cvar95_kl": 14.09136962890625, + "delta_nll": 5.967808723449707, + "max_kl": 16.257076263427734, + "mean_kl": 6.257577896118164, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.11267605423927307 + }, + { + "candidate_nll": 6.659002304077148, + "code_tokens": 47, + "cvar95_kl": 13.38924789428711, + "delta_nll": 4.92410945892334, + "max_kl": 14.236868858337402, + "mean_kl": 5.239963054656982, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.23404255509376526 + }, + { + "candidate_nll": 7.364187240600586, + "code_tokens": 50, + "cvar95_kl": 18.16984748840332, + "delta_nll": 6.562164783477783, + "max_kl": 18.989410400390625, + "mean_kl": 6.551002025604248, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.14000000059604645 + }, + { + "candidate_nll": 8.61365032196045, + "code_tokens": 113, + "cvar95_kl": 16.85106658935547, + "delta_nll": 7.561689376831055, + "max_kl": 19.485919952392578, + "mean_kl": 7.274227619171143, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.09734513610601425 + }, + { + "candidate_nll": 5.73289680480957, + "code_tokens": 26, + "cvar95_kl": 8.691627502441406, + "delta_nll": 4.292738914489746, + "max_kl": 8.756921768188477, + "mean_kl": 4.039400577545166, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.23076923191547394 + }, + { + "candidate_nll": 7.651813983917236, + "code_tokens": 62, + "cvar95_kl": 15.075267791748047, + "delta_nll": 7.083555221557617, + "max_kl": 16.098615646362305, + "mean_kl": 7.072145462036133, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.12903225421905518 + }, + { + "candidate_nll": 7.503262042999268, + "code_tokens": 79, + "cvar95_kl": 17.4106388092041, + "delta_nll": 6.750543594360352, + "max_kl": 24.573444366455078, + "mean_kl": 6.823371410369873, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.1139240488409996 + }, + { + "candidate_nll": 6.173401355743408, + "code_tokens": 20, + "cvar95_kl": 13.624422073364258, + "delta_nll": 4.544752597808838, + "max_kl": 13.624422073364258, + "mean_kl": 4.990719795227051, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.25 + }, + { + "candidate_nll": 8.4579496383667, + "code_tokens": 186, + "cvar95_kl": 17.691829681396484, + "delta_nll": 8.054804801940918, + "max_kl": 19.700414657592773, + "mean_kl": 8.075571060180664, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.08602150529623032 + }, + { + "candidate_nll": 7.467825889587402, + "code_tokens": 99, + "cvar95_kl": 14.344050407409668, + "delta_nll": 6.624678134918213, + "max_kl": 16.078147888183594, + "mean_kl": 6.457818508148193, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.13131313025951385 + }, + { + "candidate_nll": 6.101416110992432, + "code_tokens": 43, + "cvar95_kl": 13.8400239944458, + "delta_nll": 5.289608001708984, + "max_kl": 15.623430252075195, + "mean_kl": 5.649662971496582, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 5.403448581695557, + "code_tokens": 35, + "cvar95_kl": 10.007646560668945, + "delta_nll": 3.929654836654663, + "max_kl": 10.264555931091309, + "mean_kl": 4.58035945892334, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.22857142984867096 + } + ], + "policy": { + "default_bits": 4, + "name": "read_risk_l0_stochastic_seed_2341", + "rounding": "stochastic", + "seed": 2341, + "upgrade_layer": 0 + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2564096, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2564096, + "resident_compression_ratio": 7.361022364217252 + }, + "task_delta_nll_distribution": { + "maximum": 8.955904960632324, + "median": 5.5591349601745605, + "minimum": 3.2530765533447266, + "q25": 4.8673460483551025, + "q75": 6.823870301246643 + }, + "task_macro": { + "candidate_nll": 6.883340332243177, + "delta_nll": 5.826616093847487, + "mean_kl": 5.924442942937215, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.19145229015913273 + }, + "token_weighted": { + "candidate_nll": 7.608199119567871, + "cvar95_kl": 16.938119888305664, + "delta_nll": 6.680540561676025, + "max_kl": 27.61359405517578, + "mean_kl": 6.760253429412842, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.14590875804424286 + } + }, + "uniform_int4_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": 2.6604545012764307, + "macro_mean_kl": 2.7537121461785357, + "macro_top1_agreement": 0.34409933375275653, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 2.8651129162829854, + "macro_mean_kl": 2.977499148120051, + "macro_top1_agreement": 0.35181327358536096, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 3.284889828075062, + "macro_mean_kl": 3.321256984363903, + "macro_top1_agreement": 0.29339692470702256, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 3.065754110162908, + "macro_mean_kl": 3.128426356749101, + "macro_top1_agreement": 0.3274120952595364, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 4.094038009643555, + "code_tokens": 63, + "cvar95_kl": 8.139703750610352, + "delta_nll": 3.502991199493408, + "max_kl": 8.909977912902832, + "mean_kl": 3.4738433361053467, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.2380952388048172 + }, + { + "candidate_nll": 4.2347187995910645, + "code_tokens": 92, + "cvar95_kl": 8.665093421936035, + "delta_nll": 3.2289950847625732, + "max_kl": 10.962203979492188, + "mean_kl": 3.07781982421875, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.32608696818351746 + }, + { + "candidate_nll": 4.983755111694336, + "code_tokens": 36, + "cvar95_kl": 6.7959184646606445, + "delta_nll": 3.070265769958496, + "max_kl": 7.349113941192627, + "mean_kl": 3.4522902965545654, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 0.2222222238779068 + }, + { + "candidate_nll": 3.884222984313965, + "code_tokens": 24, + "cvar95_kl": 6.3897600173950195, + "delta_nll": 2.0668139457702637, + "max_kl": 6.548449993133545, + "mean_kl": 2.519688129425049, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 0.375 + }, + { + "candidate_nll": 4.229776382446289, + "code_tokens": 153, + "cvar95_kl": 8.512266159057617, + "delta_nll": 3.3128247261047363, + "max_kl": 10.516579627990723, + "mean_kl": 3.446185350418091, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.27450981736183167 + }, + { + "candidate_nll": 6.331780910491943, + "code_tokens": 133, + "cvar95_kl": 13.555427551269531, + "delta_nll": 5.265247344970703, + "max_kl": 17.34539794921875, + "mean_kl": 5.2621846199035645, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.15037593245506287 + }, + { + "candidate_nll": 3.373256206512451, + "code_tokens": 39, + "cvar95_kl": 8.381192207336426, + "delta_nll": 2.505223274230957, + "max_kl": 9.552412986755371, + "mean_kl": 2.552400588989258, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 3.6086385250091553, + "code_tokens": 26, + "cvar95_kl": 6.859403133392334, + "delta_nll": 2.3476829528808594, + "max_kl": 7.235695838928223, + "mean_kl": 2.6618640422821045, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.3013997077941895, + "code_tokens": 38, + "cvar95_kl": 10.195242881774902, + "delta_nll": 2.985726833343506, + "max_kl": 10.525386810302734, + "mean_kl": 3.068575143814087, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.3684210479259491 + }, + { + "candidate_nll": 4.664905071258545, + "code_tokens": 173, + "cvar95_kl": 10.665684700012207, + "delta_nll": 3.811112880706787, + "max_kl": 14.241655349731445, + "mean_kl": 3.9172186851501465, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.28901734948158264 + }, + { + "candidate_nll": 3.0656790733337402, + "code_tokens": 37, + "cvar95_kl": 6.781228065490723, + "delta_nll": 2.1488564014434814, + "max_kl": 8.061763763427734, + "mean_kl": 2.300877571105957, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.3513513505458832 + }, + { + "candidate_nll": 3.3881165981292725, + "code_tokens": 205, + "cvar95_kl": 9.056483268737793, + "delta_nll": 2.5759265422821045, + "max_kl": 12.578962326049805, + "mean_kl": 2.7062458992004395, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.39024388790130615 + }, + { + "candidate_nll": 4.416467189788818, + "code_tokens": 162, + "cvar95_kl": 9.979737281799316, + "delta_nll": 3.5174713134765625, + "max_kl": 11.227472305297852, + "mean_kl": 3.8480796813964844, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.3395061790943146 + }, + { + "candidate_nll": 3.103994131088257, + "code_tokens": 132, + "cvar95_kl": 6.621105194091797, + "delta_nll": 2.441636562347412, + "max_kl": 7.982386112213135, + "mean_kl": 2.4110779762268066, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.4469696879386902 + }, + { + "candidate_nll": 2.1626389026641846, + "code_tokens": 33, + "cvar95_kl": 4.8563385009765625, + "delta_nll": 1.4954452514648438, + "max_kl": 5.312148571014404, + "mean_kl": 1.6082534790039062, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 0.5151515007019043 + }, + { + "candidate_nll": 4.792906284332275, + "code_tokens": 60, + "cvar95_kl": 11.012053489685059, + "delta_nll": 3.451171636581421, + "max_kl": 12.786828994750977, + "mean_kl": 3.59126877784729, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.2666666805744171 + }, + { + "candidate_nll": 4.0534443855285645, + "code_tokens": 63, + "cvar95_kl": 6.868353843688965, + "delta_nll": 3.646638870239258, + "max_kl": 7.525240898132324, + "mean_kl": 3.6943318843841553, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 3.4382946491241455, + "code_tokens": 48, + "cvar95_kl": 6.303212642669678, + "delta_nll": 1.8221638202667236, + "max_kl": 6.9882097244262695, + "mean_kl": 2.458308458328247, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.2916666567325592 + }, + { + "candidate_nll": 3.1840713024139404, + "code_tokens": 76, + "cvar95_kl": 7.121147632598877, + "delta_nll": 2.4013872146606445, + "max_kl": 7.858313083648682, + "mean_kl": 2.304766893386841, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.4868420958518982 + }, + { + "candidate_nll": 3.9222664833068848, + "code_tokens": 63, + "cvar95_kl": 7.396600723266602, + "delta_nll": 2.9711718559265137, + "max_kl": 8.736921310424805, + "mean_kl": 2.972618341445923, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.3174603283405304 + }, + { + "candidate_nll": 4.635219573974609, + "code_tokens": 107, + "cvar95_kl": 9.68270492553711, + "delta_nll": 3.7704854011535645, + "max_kl": 11.203866958618164, + "mean_kl": 3.9099655151367188, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.2429906576871872 + }, + { + "candidate_nll": 3.418982982635498, + "code_tokens": 108, + "cvar95_kl": 9.262359619140625, + "delta_nll": 2.8038103580474854, + "max_kl": 10.322102546691895, + "mean_kl": 2.985664129257202, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 4.8202738761901855, + "code_tokens": 46, + "cvar95_kl": 10.161595344543457, + "delta_nll": 3.84694242477417, + "max_kl": 10.3450288772583, + "mean_kl": 4.051937103271484, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 0.17391304671764374 + }, + { + "candidate_nll": 4.382615566253662, + "code_tokens": 37, + "cvar95_kl": 8.815812110900879, + "delta_nll": 3.453357696533203, + "max_kl": 10.132514953613281, + "mean_kl": 3.686544895172119, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.881091356277466, + "code_tokens": 29, + "cvar95_kl": 5.462983131408691, + "delta_nll": 2.7289352416992188, + "max_kl": 5.516096115112305, + "mean_kl": 2.9134726524353027, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.37931033968925476 + }, + { + "candidate_nll": 3.905167818069458, + "code_tokens": 13, + "cvar95_kl": 5.208853721618652, + "delta_nll": 2.2910385131835938, + "max_kl": 5.208853721618652, + "mean_kl": 2.3794093132019043, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 4.02086067199707, + "code_tokens": 51, + "cvar95_kl": 10.204272270202637, + "delta_nll": 3.39776873588562, + "max_kl": 11.921181678771973, + "mean_kl": 3.3472650051116943, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 0.3333333432674408 + }, + { + "candidate_nll": 4.982914924621582, + "code_tokens": 29, + "cvar95_kl": 11.356088638305664, + "delta_nll": 3.852489948272705, + "max_kl": 13.898453712463379, + "mean_kl": 3.6746058464050293, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.24137930572032928 + }, + { + "candidate_nll": 4.020605564117432, + "code_tokens": 30, + "cvar95_kl": 8.857093811035156, + "delta_nll": 2.9910104274749756, + "max_kl": 8.947272300720215, + "mean_kl": 3.247835636138916, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.0817952156066895, + "code_tokens": 103, + "cvar95_kl": 9.088061332702637, + "delta_nll": 3.2910571098327637, + "max_kl": 11.183342933654785, + "mean_kl": 3.187256097793579, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.26213592290878296 + }, + { + "candidate_nll": 3.655787944793701, + "code_tokens": 119, + "cvar95_kl": 8.588122367858887, + "delta_nll": 2.755244016647339, + "max_kl": 10.299145698547363, + "mean_kl": 2.80863094329834, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 5.072477340698242, + "code_tokens": 22, + "cvar95_kl": 10.284576416015625, + "delta_nll": 3.775606632232666, + "max_kl": 10.595476150512695, + "mean_kl": 3.8234000205993652, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 3.5697386264801025, + "code_tokens": 51, + "cvar95_kl": 7.92575216293335, + "delta_nll": 2.9318604469299316, + "max_kl": 9.409361839294434, + "mean_kl": 2.9788990020751953, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.3137255012989044 + }, + { + "candidate_nll": 3.94193696975708, + "code_tokens": 32, + "cvar95_kl": 6.897381782531738, + "delta_nll": 2.1823883056640625, + "max_kl": 7.294264793395996, + "mean_kl": 2.5162291526794434, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.34375 + }, + { + "candidate_nll": 3.2037363052368164, + "code_tokens": 116, + "cvar95_kl": 7.314982891082764, + "delta_nll": 2.1404356956481934, + "max_kl": 10.261045455932617, + "mean_kl": 2.435479164123535, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.4051724076271057 + }, + { + "candidate_nll": 5.112421989440918, + "code_tokens": 56, + "cvar95_kl": 7.792276382446289, + "delta_nll": 4.327237129211426, + "max_kl": 8.173297882080078, + "mean_kl": 3.975388765335083, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.1785714328289032 + }, + { + "candidate_nll": 3.596846103668213, + "code_tokens": 53, + "cvar95_kl": 6.611747741699219, + "delta_nll": 2.6489720344543457, + "max_kl": 7.042396545410156, + "mean_kl": 2.8259031772613525, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.3396226465702057 + }, + { + "candidate_nll": 2.8973004817962646, + "code_tokens": 127, + "cvar95_kl": 7.040439605712891, + "delta_nll": 2.2860796451568604, + "max_kl": 7.8400068283081055, + "mean_kl": 2.305920362472534, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.5039370059967041 + }, + { + "candidate_nll": 4.5556511878967285, + "code_tokens": 61, + "cvar95_kl": 8.403921127319336, + "delta_nll": 3.3431167602539062, + "max_kl": 9.887153625488281, + "mean_kl": 3.5047881603240967, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.26229506731033325 + }, + { + "candidate_nll": 3.0199174880981445, + "code_tokens": 147, + "cvar95_kl": 7.891502857208252, + "delta_nll": 2.473921775817871, + "max_kl": 9.942960739135742, + "mean_kl": 2.6771090030670166, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.35374149680137634 + }, + { + "candidate_nll": 4.415050029754639, + "code_tokens": 28, + "cvar95_kl": 6.4762797355651855, + "delta_nll": 2.937063694000244, + "max_kl": 6.692333698272705, + "mean_kl": 3.145369291305542, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 3.4673099517822266, + "code_tokens": 66, + "cvar95_kl": 9.614180564880371, + "delta_nll": 2.6309399604797363, + "max_kl": 11.286248207092285, + "mean_kl": 2.7361836433410645, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 0.43939393758773804 + }, + { + "candidate_nll": 4.258036136627197, + "code_tokens": 33, + "cvar95_kl": 6.704224586486816, + "delta_nll": 2.4312314987182617, + "max_kl": 6.939979076385498, + "mean_kl": 2.721433639526367, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.4545454680919647 + }, + { + "candidate_nll": 4.2296953201293945, + "code_tokens": 41, + "cvar95_kl": 8.169517517089844, + "delta_nll": 3.091996192932129, + "max_kl": 9.901902198791504, + "mean_kl": 3.098534345626831, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 0.31707316637039185 + }, + { + "candidate_nll": 4.04701042175293, + "code_tokens": 36, + "cvar95_kl": 8.79304313659668, + "delta_nll": 2.370832681655884, + "max_kl": 9.092019081115723, + "mean_kl": 2.363572597503662, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.4722222089767456 + }, + { + "candidate_nll": 5.540225982666016, + "code_tokens": 70, + "cvar95_kl": 10.80223274230957, + "delta_nll": 4.623091220855713, + "max_kl": 11.823500633239746, + "mean_kl": 4.445784091949463, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.15714286267757416 + }, + { + "candidate_nll": 3.850351333618164, + "code_tokens": 17, + "cvar95_kl": 8.070456504821777, + "delta_nll": 2.550886869430542, + "max_kl": 8.070456504821777, + "mean_kl": 2.5514562129974365, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 0.23529411852359772 + }, + { + "candidate_nll": 4.399035453796387, + "code_tokens": 23, + "cvar95_kl": 7.155698299407959, + "delta_nll": 3.2138280868530273, + "max_kl": 7.737255096435547, + "mean_kl": 2.7912063598632812, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 4.101292610168457, + "code_tokens": 101, + "cvar95_kl": 9.285934448242188, + "delta_nll": 3.723418951034546, + "max_kl": 10.052959442138672, + "mean_kl": 3.819678544998169, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.31683167815208435 + }, + { + "candidate_nll": 4.01340913772583, + "code_tokens": 36, + "cvar95_kl": 7.399251937866211, + "delta_nll": 2.5985324382781982, + "max_kl": 7.425439834594727, + "mean_kl": 2.911294937133789, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.3611111044883728 + }, + { + "candidate_nll": 5.019290924072266, + "code_tokens": 46, + "cvar95_kl": 10.171727180480957, + "delta_nll": 3.813537120819092, + "max_kl": 11.108030319213867, + "mean_kl": 3.901594400405884, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.30434781312942505 + }, + { + "candidate_nll": 3.842963218688965, + "code_tokens": 24, + "cvar95_kl": 6.581437110900879, + "delta_nll": 2.3121166229248047, + "max_kl": 8.360477447509766, + "mean_kl": 2.2886388301849365, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 0.4583333432674408 + }, + { + "candidate_nll": 4.144324779510498, + "code_tokens": 22, + "cvar95_kl": 7.377143859863281, + "delta_nll": 2.768995523452759, + "max_kl": 8.146191596984863, + "mean_kl": 3.1282598972320557, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.27272728085517883 + }, + { + "candidate_nll": 4.017523288726807, + "code_tokens": 58, + "cvar95_kl": 7.595575332641602, + "delta_nll": 3.328650712966919, + "max_kl": 7.790223121643066, + "mean_kl": 3.237703800201416, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.32758620381355286 + }, + { + "candidate_nll": 4.098639011383057, + "code_tokens": 15, + "cvar95_kl": 6.756157875061035, + "delta_nll": 2.785768508911133, + "max_kl": 6.756157875061035, + "mean_kl": 2.7613425254821777, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 3.8299973011016846, + "code_tokens": 40, + "cvar95_kl": 8.380942344665527, + "delta_nll": 3.3904097080230713, + "max_kl": 8.551811218261719, + "mean_kl": 3.1498398780822754, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.6272661685943604, + "code_tokens": 37, + "cvar95_kl": 7.375539779663086, + "delta_nll": 3.022716522216797, + "max_kl": 7.765446186065674, + "mean_kl": 2.9628381729125977, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 0.3513513505458832 + }, + { + "candidate_nll": 3.7769641876220703, + "code_tokens": 21, + "cvar95_kl": 8.454581260681152, + "delta_nll": 2.756718158721924, + "max_kl": 8.664769172668457, + "mean_kl": 3.071133852005005, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 0.4761904776096344 + }, + { + "candidate_nll": 3.7339324951171875, + "code_tokens": 21, + "cvar95_kl": 5.0968732833862305, + "delta_nll": 2.421416759490967, + "max_kl": 5.230508327484131, + "mean_kl": 2.194509744644165, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.4285714328289032 + }, + { + "candidate_nll": 4.302450656890869, + "code_tokens": 66, + "cvar95_kl": 9.785279273986816, + "delta_nll": 3.29408597946167, + "max_kl": 11.414859771728516, + "mean_kl": 3.683077335357666, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.25757575035095215 + }, + { + "candidate_nll": 2.759397268295288, + "code_tokens": 138, + "cvar95_kl": 5.887688159942627, + "delta_nll": 2.115736484527588, + "max_kl": 8.12856674194336, + "mean_kl": 2.0383365154266357, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.4057970941066742 + }, + { + "candidate_nll": 3.1431691646575928, + "code_tokens": 22, + "cvar95_kl": 7.311114311218262, + "delta_nll": 2.408926486968994, + "max_kl": 7.450708389282227, + "mean_kl": 2.438551664352417, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 0.40909090638160706 + }, + { + "candidate_nll": 4.249351501464844, + "code_tokens": 37, + "cvar95_kl": 8.109967231750488, + "delta_nll": 3.503767251968384, + "max_kl": 8.470084190368652, + "mean_kl": 3.189560651779175, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 0.2432432472705841 + }, + { + "candidate_nll": 3.68324613571167, + "code_tokens": 41, + "cvar95_kl": 7.3946757316589355, + "delta_nll": 2.7324883937835693, + "max_kl": 8.15935230255127, + "mean_kl": 2.6946754455566406, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.4390243887901306 + }, + { + "candidate_nll": 4.616489887237549, + "code_tokens": 63, + "cvar95_kl": 8.100735664367676, + "delta_nll": 3.4579992294311523, + "max_kl": 8.35948657989502, + "mean_kl": 3.590510129928589, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.1746031790971756 + }, + { + "candidate_nll": 3.478971481323242, + "code_tokens": 99, + "cvar95_kl": 6.677661895751953, + "delta_nll": 2.6322197914123535, + "max_kl": 6.9825758934021, + "mean_kl": 2.708930015563965, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.2525252401828766 + }, + { + "candidate_nll": 2.880063533782959, + "code_tokens": 65, + "cvar95_kl": 4.847674369812012, + "delta_nll": 1.94996976852417, + "max_kl": 5.595011234283447, + "mean_kl": 2.031872510910034, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.4307692348957062 + }, + { + "candidate_nll": 4.074697494506836, + "code_tokens": 39, + "cvar95_kl": 8.78947639465332, + "delta_nll": 3.157168388366699, + "max_kl": 9.020490646362305, + "mean_kl": 3.357948064804077, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.41025641560554504 + }, + { + "candidate_nll": 3.5114810466766357, + "code_tokens": 40, + "cvar95_kl": 7.038084506988525, + "delta_nll": 2.352720260620117, + "max_kl": 7.074913501739502, + "mean_kl": 2.72064208984375, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.244096755981445, + "code_tokens": 97, + "cvar95_kl": 8.235143661499023, + "delta_nll": 2.9491446018218994, + "max_kl": 8.825080871582031, + "mean_kl": 3.0991508960723877, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.30927833914756775 + }, + { + "candidate_nll": 4.754541873931885, + "code_tokens": 23, + "cvar95_kl": 7.197890281677246, + "delta_nll": 2.839513063430786, + "max_kl": 7.99038553237915, + "mean_kl": 3.0580413341522217, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.260869562625885 + }, + { + "candidate_nll": 3.14814829826355, + "code_tokens": 25, + "cvar95_kl": 5.582275390625, + "delta_nll": 2.114278793334961, + "max_kl": 5.784864902496338, + "mean_kl": 2.075026035308838, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 0.4000000059604645 + }, + { + "candidate_nll": 4.700220108032227, + "code_tokens": 57, + "cvar95_kl": 7.953707218170166, + "delta_nll": 3.844691514968872, + "max_kl": 9.453850746154785, + "mean_kl": 3.3731119632720947, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.2982456088066101 + }, + { + "candidate_nll": 5.497686862945557, + "code_tokens": 50, + "cvar95_kl": 11.712023735046387, + "delta_nll": 4.455387115478516, + "max_kl": 12.959553718566895, + "mean_kl": 4.663835525512695, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.20000000298023224 + }, + { + "candidate_nll": 4.782854080200195, + "code_tokens": 35, + "cvar95_kl": 10.282578468322754, + "delta_nll": 3.954270362854004, + "max_kl": 11.657696723937988, + "mean_kl": 4.123940467834473, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 0.2857142984867096 + }, + { + "candidate_nll": 4.455539226531982, + "code_tokens": 29, + "cvar95_kl": 6.418630599975586, + "delta_nll": 2.11112642288208, + "max_kl": 6.877559661865234, + "mean_kl": 2.7114384174346924, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.27586206793785095 + }, + { + "candidate_nll": 4.149670600891113, + "code_tokens": 20, + "cvar95_kl": 5.855801582336426, + "delta_nll": 2.8162527084350586, + "max_kl": 5.855801582336426, + "mean_kl": 2.5561745166778564, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 0.30000001192092896 + }, + { + "candidate_nll": 3.650949239730835, + "code_tokens": 19, + "cvar95_kl": 5.917978286743164, + "delta_nll": 2.534306287765503, + "max_kl": 5.917978286743164, + "mean_kl": 2.588390588760376, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.31578946113586426 + }, + { + "candidate_nll": 4.577055931091309, + "code_tokens": 71, + "cvar95_kl": 9.598867416381836, + "delta_nll": 3.630772352218628, + "max_kl": 10.034027099609375, + "mean_kl": 3.6550474166870117, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.23943662643432617 + }, + { + "candidate_nll": 4.936843395233154, + "code_tokens": 47, + "cvar95_kl": 10.058987617492676, + "delta_nll": 3.2019505500793457, + "max_kl": 11.142730712890625, + "mean_kl": 3.3717684745788574, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.38297873735427856 + }, + { + "candidate_nll": 4.136938095092773, + "code_tokens": 50, + "cvar95_kl": 8.149764060974121, + "delta_nll": 3.3349156379699707, + "max_kl": 8.399584770202637, + "mean_kl": 3.2757153511047363, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 0.3400000035762787 + }, + { + "candidate_nll": 4.197843551635742, + "code_tokens": 113, + "cvar95_kl": 7.655534267425537, + "delta_nll": 3.1458826065063477, + "max_kl": 8.803738594055176, + "mean_kl": 2.9634881019592285, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.3451327383518219 + }, + { + "candidate_nll": 4.422205448150635, + "code_tokens": 26, + "cvar95_kl": 7.058747291564941, + "delta_nll": 2.9820475578308105, + "max_kl": 7.664772033691406, + "mean_kl": 2.955230712890625, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.3076923191547394 + }, + { + "candidate_nll": 3.8408517837524414, + "code_tokens": 62, + "cvar95_kl": 9.530633926391602, + "delta_nll": 3.2725930213928223, + "max_kl": 12.223296165466309, + "mean_kl": 3.2474300861358643, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 0.3870967626571655 + }, + { + "candidate_nll": 4.008029937744141, + "code_tokens": 79, + "cvar95_kl": 7.118758201599121, + "delta_nll": 3.2553114891052246, + "max_kl": 7.379554748535156, + "mean_kl": 3.144963502883911, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.2531645596027374 + }, + { + "candidate_nll": 3.2122790813446045, + "code_tokens": 20, + "cvar95_kl": 5.092899799346924, + "delta_nll": 1.5836303234100342, + "max_kl": 5.092899799346924, + "mean_kl": 1.8003337383270264, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 0.5 + }, + { + "candidate_nll": 3.0860092639923096, + "code_tokens": 186, + "cvar95_kl": 7.954154014587402, + "delta_nll": 2.6828649044036865, + "max_kl": 10.353998184204102, + "mean_kl": 2.7023766040802, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.3655914068222046 + }, + { + "candidate_nll": 4.110910892486572, + "code_tokens": 99, + "cvar95_kl": 8.375022888183594, + "delta_nll": 3.267763137817383, + "max_kl": 8.726351737976074, + "mean_kl": 3.3696184158325195, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.3232323229312897 + }, + { + "candidate_nll": 3.983187198638916, + "code_tokens": 43, + "cvar95_kl": 7.07157564163208, + "delta_nll": 3.1713790893554688, + "max_kl": 7.666134834289551, + "mean_kl": 3.298661231994629, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.25581395626068115 + }, + { + "candidate_nll": 2.8901844024658203, + "code_tokens": 35, + "cvar95_kl": 3.992973804473877, + "delta_nll": 1.4163906574249268, + "max_kl": 4.426108360290527, + "mean_kl": 1.3790677785873413, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.48571428656578064 + } + ], + "policy": { + "default_bits": 4, + "name": "uniform_int4_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 2433024, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 2433024, + "resident_compression_ratio": 7.757575757575758 + }, + "task_delta_nll_distribution": { + "maximum": 5.265247344970703, + "median": 2.9601582288742065, + "minimum": 1.4163906574249268, + "q25": 2.449707865715027, + "q75": 3.37858647108078 + }, + "task_macro": { + "candidate_nll": 4.021193273862203, + "delta_nll": 2.9644690805011327, + "mean_kl": 3.0412321474817063, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.32959764897823335 + }, + "token_weighted": { + "candidate_nll": 3.9759979248046875, + "cvar95_kl": 9.128149032592773, + "delta_nll": 3.048339366912842, + "max_kl": 17.34539794921875, + "mean_kl": 3.12032413482666, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.3269369900226593 + } + }, + "uniform_int8_nearest": { + "by_code_length_quartile": { + "Q1": { + "macro_delta_nll": -0.0001875758171081543, + "macro_mean_kl": 0.010772417431049373, + "macro_top1_agreement": 0.9718172368795975, + "maximum_code_tokens": 30, + "minimum_code_tokens": 13, + "task_count": 23 + }, + "Q2": { + "macro_delta_nll": 0.00858371931573619, + "macro_mean_kl": 0.01399391367221656, + "macro_top1_agreement": 0.9697772160820339, + "maximum_code_tokens": 47, + "minimum_code_tokens": 32, + "task_count": 23 + }, + "Q3": { + "macro_delta_nll": 0.014173171736977318, + "macro_mean_kl": 0.016658776820721952, + "macro_top1_agreement": 0.967708863995292, + "maximum_code_tokens": 76, + "minimum_code_tokens": 48, + "task_count": 22 + }, + "Q4": { + "macro_delta_nll": 0.04083062708377838, + "macro_mean_kl": 0.03569604985585267, + "macro_top1_agreement": 0.9514070397073572, + "maximum_code_tokens": 205, + "minimum_code_tokens": 79, + "task_count": 22 + } + }, + "per_task": [ + { + "candidate_nll": 0.660081148147583, + "code_tokens": 63, + "cvar95_kl": 0.310040682554245, + "delta_nll": 0.0690343976020813, + "max_kl": 0.7474806308746338, + "mean_kl": 0.03134522587060928, + "reference_nll": 0.5910467505455017, + "task_id": 511, + "token_count": 63, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 1.0237468481063843, + "code_tokens": 92, + "cvar95_kl": 0.1572273224592209, + "delta_nll": 0.018023133277893066, + "max_kl": 0.19508598744869232, + "mean_kl": 0.021973012015223503, + "reference_nll": 1.0057237148284912, + "task_id": 512, + "token_count": 92, + "top1_agreement": 0.945652186870575 + }, + { + "candidate_nll": 1.9346351623535156, + "code_tokens": 36, + "cvar95_kl": 0.09448091685771942, + "delta_nll": 0.021145939826965332, + "max_kl": 0.11314337700605392, + "mean_kl": 0.016842085868120193, + "reference_nll": 1.9134892225265503, + "task_id": 513, + "token_count": 36, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.8179537057876587, + "code_tokens": 24, + "cvar95_kl": 0.08254823088645935, + "delta_nll": 0.0005446672439575195, + "max_kl": 0.1080058291554451, + "mean_kl": 0.014093535952270031, + "reference_nll": 1.8174090385437012, + "task_id": 514, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9782454371452332, + "code_tokens": 153, + "cvar95_kl": 0.5254913568496704, + "delta_nll": 0.06129366159439087, + "max_kl": 1.2992838621139526, + "mean_kl": 0.06731460243463516, + "reference_nll": 0.9169517755508423, + "task_id": 515, + "token_count": 153, + "top1_agreement": 0.9346405267715454 + }, + { + "candidate_nll": 1.0834169387817383, + "code_tokens": 133, + "cvar95_kl": 0.3434354364871979, + "delta_nll": 0.016883373260498047, + "max_kl": 0.796250581741333, + "mean_kl": 0.043524254113435745, + "reference_nll": 1.0665335655212402, + "task_id": 516, + "token_count": 133, + "top1_agreement": 0.9248120188713074 + }, + { + "candidate_nll": 0.8689431548118591, + "code_tokens": 39, + "cvar95_kl": 0.04129485785961151, + "delta_nll": 0.0009102821350097656, + "max_kl": 0.057022299617528915, + "mean_kl": 0.006134225055575371, + "reference_nll": 0.8680328726768494, + "task_id": 517, + "token_count": 39, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.2456201314926147, + "code_tokens": 26, + "cvar95_kl": 0.0987609326839447, + "delta_nll": -0.015335559844970703, + "max_kl": 0.10459931194782257, + "mean_kl": 0.012855694629251957, + "reference_nll": 1.2609556913375854, + "task_id": 518, + "token_count": 26, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3204947710037231, + "code_tokens": 38, + "cvar95_kl": 0.052462972700595856, + "delta_nll": 0.004821896553039551, + "max_kl": 0.0766393169760704, + "mean_kl": 0.009308372624218464, + "reference_nll": 1.3156728744506836, + "task_id": 519, + "token_count": 38, + "top1_agreement": 0.9736841917037964 + }, + { + "candidate_nll": 0.9200796484947205, + "code_tokens": 173, + "cvar95_kl": 0.46936464309692383, + "delta_nll": 0.0662873387336731, + "max_kl": 1.0484676361083984, + "mean_kl": 0.0491132028400898, + "reference_nll": 0.8537923097610474, + "task_id": 520, + "token_count": 173, + "top1_agreement": 0.9421965479850769 + }, + { + "candidate_nll": 0.9245966076850891, + "code_tokens": 37, + "cvar95_kl": 0.09565657377243042, + "delta_nll": 0.007773935794830322, + "max_kl": 0.14162375032901764, + "mean_kl": 0.012193853966891766, + "reference_nll": 0.9168226718902588, + "task_id": 521, + "token_count": 37, + "top1_agreement": 0.9459459185600281 + }, + { + "candidate_nll": 0.8641772270202637, + "code_tokens": 205, + "cvar95_kl": 0.2403945028781891, + "delta_nll": 0.0519871711730957, + "max_kl": 0.39936769008636475, + "mean_kl": 0.03126063570380211, + "reference_nll": 0.812190055847168, + "task_id": 522, + "token_count": 205, + "top1_agreement": 0.9512194991111755 + }, + { + "candidate_nll": 0.9444414377212524, + "code_tokens": 162, + "cvar95_kl": 0.4423255920410156, + "delta_nll": 0.04544550180435181, + "max_kl": 1.1258400678634644, + "mean_kl": 0.050369780510663986, + "reference_nll": 0.8989959359169006, + "task_id": 523, + "token_count": 162, + "top1_agreement": 0.9382715821266174 + }, + { + "candidate_nll": 0.687138557434082, + "code_tokens": 132, + "cvar95_kl": 0.3074168264865875, + "delta_nll": 0.024780988693237305, + "max_kl": 0.9763906598091125, + "mean_kl": 0.03030756115913391, + "reference_nll": 0.6623575687408447, + "task_id": 524, + "token_count": 132, + "top1_agreement": 0.9621211886405945 + }, + { + "candidate_nll": 0.6363716125488281, + "code_tokens": 33, + "cvar95_kl": 0.07050937414169312, + "delta_nll": -0.03082209825515747, + "max_kl": 0.10928164422512054, + "mean_kl": 0.00777086429297924, + "reference_nll": 0.6671937108039856, + "task_id": 525, + "token_count": 33, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.3778512477874756, + "code_tokens": 60, + "cvar95_kl": 0.09364253282546997, + "delta_nll": 0.036116600036621094, + "max_kl": 0.1073627769947052, + "mean_kl": 0.013720711693167686, + "reference_nll": 1.3417346477508545, + "task_id": 526, + "token_count": 60, + "top1_agreement": 0.9833333492279053 + }, + { + "candidate_nll": 0.40927883982658386, + "code_tokens": 63, + "cvar95_kl": 0.0686848908662796, + "delta_nll": 0.0024732351303100586, + "max_kl": 0.07620687037706375, + "mean_kl": 0.010775050148367882, + "reference_nll": 0.4068056046962738, + "task_id": 527, + "token_count": 63, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 1.597420334815979, + "code_tokens": 48, + "cvar95_kl": 0.14145655930042267, + "delta_nll": -0.01871049404144287, + "max_kl": 0.22967463731765747, + "mean_kl": 0.017784221097826958, + "reference_nll": 1.6161308288574219, + "task_id": 528, + "token_count": 48, + "top1_agreement": 0.9583333134651184 + }, + { + "candidate_nll": 0.8071182370185852, + "code_tokens": 76, + "cvar95_kl": 0.09406808018684387, + "delta_nll": 0.024434149265289307, + "max_kl": 0.11692332476377487, + "mean_kl": 0.01535926666110754, + "reference_nll": 0.7826840877532959, + "task_id": 529, + "token_count": 76, + "top1_agreement": 0.9868420958518982 + }, + { + "candidate_nll": 0.9585981369018555, + "code_tokens": 63, + "cvar95_kl": 0.2650747299194336, + "delta_nll": 0.007503390312194824, + "max_kl": 0.47725367546081543, + "mean_kl": 0.02765016444027424, + "reference_nll": 0.9510947465896606, + "task_id": 530, + "token_count": 63, + "top1_agreement": 0.9682539701461792 + }, + { + "candidate_nll": 0.9402478933334351, + "code_tokens": 107, + "cvar95_kl": 0.299570769071579, + "delta_nll": 0.07551383972167969, + "max_kl": 0.897932767868042, + "mean_kl": 0.03851960971951485, + "reference_nll": 0.8647340536117554, + "task_id": 531, + "token_count": 107, + "top1_agreement": 0.9906542301177979 + }, + { + "candidate_nll": 0.64936363697052, + "code_tokens": 108, + "cvar95_kl": 0.6140926480293274, + "delta_nll": 0.034191012382507324, + "max_kl": 3.0969557762145996, + "mean_kl": 0.04269355908036232, + "reference_nll": 0.6151726245880127, + "task_id": 532, + "token_count": 108, + "top1_agreement": 0.9814814925193787 + }, + { + "candidate_nll": 0.9688888192176819, + "code_tokens": 46, + "cvar95_kl": 0.11024332046508789, + "delta_nll": -0.004442572593688965, + "max_kl": 0.24693235754966736, + "mean_kl": 0.011418535374104977, + "reference_nll": 0.9733313918113708, + "task_id": 533, + "token_count": 46, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9873120188713074, + "code_tokens": 37, + "cvar95_kl": 0.14705777168273926, + "delta_nll": 0.05805402994155884, + "max_kl": 0.15983930230140686, + "mean_kl": 0.01883709989488125, + "reference_nll": 0.9292579889297485, + "task_id": 534, + "token_count": 37, + "top1_agreement": 0.9459459185600281 + }, + { + "candidate_nll": 1.1458698511123657, + "code_tokens": 29, + "cvar95_kl": 0.2488912045955658, + "delta_nll": -0.006286263465881348, + "max_kl": 0.291776180267334, + "mean_kl": 0.027113934978842735, + "reference_nll": 1.152156114578247, + "task_id": 535, + "token_count": 29, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 1.6688627004623413, + "code_tokens": 13, + "cvar95_kl": 0.029866810888051987, + "delta_nll": 0.0547332763671875, + "max_kl": 0.029866810888051987, + "mean_kl": 0.005748954601585865, + "reference_nll": 1.6141294240951538, + "task_id": 536, + "token_count": 13, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6423396468162537, + "code_tokens": 51, + "cvar95_kl": 0.0723278820514679, + "delta_nll": 0.01924765110015869, + "max_kl": 0.1130494549870491, + "mean_kl": 0.009966827929019928, + "reference_nll": 0.623091995716095, + "task_id": 537, + "token_count": 51, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1285706758499146, + "code_tokens": 29, + "cvar95_kl": 0.11291194707155228, + "delta_nll": -0.0018543004989624023, + "max_kl": 0.16854113340377808, + "mean_kl": 0.01534377783536911, + "reference_nll": 1.130424976348877, + "task_id": 538, + "token_count": 29, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 1.0413150787353516, + "code_tokens": 30, + "cvar95_kl": 0.040638718754053116, + "delta_nll": 0.011719942092895508, + "max_kl": 0.05025689676403999, + "mean_kl": 0.011392575688660145, + "reference_nll": 1.029595136642456, + "task_id": 539, + "token_count": 30, + "top1_agreement": 0.9666666388511658 + }, + { + "candidate_nll": 0.8108336329460144, + "code_tokens": 103, + "cvar95_kl": 0.1606554388999939, + "delta_nll": 0.020095646381378174, + "max_kl": 0.2113724946975708, + "mean_kl": 0.0251189935952425, + "reference_nll": 0.7907379865646362, + "task_id": 540, + "token_count": 103, + "top1_agreement": 0.9320388436317444 + }, + { + "candidate_nll": 0.9276239275932312, + "code_tokens": 119, + "cvar95_kl": 0.37099918723106384, + "delta_nll": 0.027079999446868896, + "max_kl": 0.478093683719635, + "mean_kl": 0.03869787976145744, + "reference_nll": 0.9005439281463623, + "task_id": 541, + "token_count": 119, + "top1_agreement": 0.9159663915634155 + }, + { + "candidate_nll": 1.3180546760559082, + "code_tokens": 22, + "cvar95_kl": 0.04472746700048447, + "delta_nll": 0.02118396759033203, + "max_kl": 0.06079676374793053, + "mean_kl": 0.009496038779616356, + "reference_nll": 1.2968707084655762, + "task_id": 542, + "token_count": 22, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.6791808009147644, + "code_tokens": 51, + "cvar95_kl": 0.09950679540634155, + "delta_nll": 0.04130274057388306, + "max_kl": 0.14307743310928345, + "mean_kl": 0.015783706679940224, + "reference_nll": 0.6378780603408813, + "task_id": 543, + "token_count": 51, + "top1_agreement": 0.9803921580314636 + }, + { + "candidate_nll": 1.7685922384262085, + "code_tokens": 32, + "cvar95_kl": 0.044135697185993195, + "delta_nll": 0.009043574333190918, + "max_kl": 0.0464116632938385, + "mean_kl": 0.011312359943985939, + "reference_nll": 1.7595486640930176, + "task_id": 544, + "token_count": 32, + "top1_agreement": 0.96875 + }, + { + "candidate_nll": 1.087570071220398, + "code_tokens": 116, + "cvar95_kl": 0.08382638543844223, + "delta_nll": 0.02426934242248535, + "max_kl": 0.1313333809375763, + "mean_kl": 0.014004125259816647, + "reference_nll": 1.0633007287979126, + "task_id": 545, + "token_count": 116, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 0.815030038356781, + "code_tokens": 56, + "cvar95_kl": 0.07534674555063248, + "delta_nll": 0.029845058917999268, + "max_kl": 0.07966292649507523, + "mean_kl": 0.013977586291730404, + "reference_nll": 0.7851849794387817, + "task_id": 546, + "token_count": 56, + "top1_agreement": 0.9821428656578064 + }, + { + "candidate_nll": 0.9815215468406677, + "code_tokens": 53, + "cvar95_kl": 0.21612679958343506, + "delta_nll": 0.03364741802215576, + "max_kl": 0.36824968457221985, + "mean_kl": 0.025100966915488243, + "reference_nll": 0.947874128818512, + "task_id": 547, + "token_count": 53, + "top1_agreement": 0.9811320900917053 + }, + { + "candidate_nll": 0.647261381149292, + "code_tokens": 127, + "cvar95_kl": 0.25919440388679504, + "delta_nll": 0.036040544509887695, + "max_kl": 0.4121004343032837, + "mean_kl": 0.028066517785191536, + "reference_nll": 0.6112208366394043, + "task_id": 548, + "token_count": 127, + "top1_agreement": 0.960629940032959 + }, + { + "candidate_nll": 1.1872106790542603, + "code_tokens": 61, + "cvar95_kl": 0.2052706778049469, + "delta_nll": -0.02532362937927246, + "max_kl": 0.270851194858551, + "mean_kl": 0.023426171392202377, + "reference_nll": 1.2125343084335327, + "task_id": 549, + "token_count": 61, + "top1_agreement": 0.9344262480735779 + }, + { + "candidate_nll": 0.5891287922859192, + "code_tokens": 147, + "cvar95_kl": 0.4042048454284668, + "delta_nll": 0.04313302040100098, + "max_kl": 0.6060087084770203, + "mean_kl": 0.03869875892996788, + "reference_nll": 0.5459957718849182, + "task_id": 550, + "token_count": 147, + "top1_agreement": 0.9319728016853333 + }, + { + "candidate_nll": 1.4612605571746826, + "code_tokens": 28, + "cvar95_kl": 0.05563338100910187, + "delta_nll": -0.016725897789001465, + "max_kl": 0.06348421424627304, + "mean_kl": 0.011856402270495892, + "reference_nll": 1.477986454963684, + "task_id": 551, + "token_count": 28, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 0.8371211886405945, + "code_tokens": 66, + "cvar95_kl": 0.11414308100938797, + "delta_nll": 0.000751197338104248, + "max_kl": 0.17100417613983154, + "mean_kl": 0.015381592325866222, + "reference_nll": 0.8363699913024902, + "task_id": 552, + "token_count": 66, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.7548809051513672, + "code_tokens": 33, + "cvar95_kl": 0.34486228227615356, + "delta_nll": -0.07192385196685791, + "max_kl": 0.600721001625061, + "mean_kl": 0.03006862848997116, + "reference_nll": 1.826804757118225, + "task_id": 553, + "token_count": 33, + "top1_agreement": 0.939393937587738 + }, + { + "candidate_nll": 1.1425893306732178, + "code_tokens": 41, + "cvar95_kl": 0.029452577233314514, + "delta_nll": 0.004890084266662598, + "max_kl": 0.039373792707920074, + "mean_kl": 0.006967495661228895, + "reference_nll": 1.1376992464065552, + "task_id": 554, + "token_count": 41, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.7070019245147705, + "code_tokens": 36, + "cvar95_kl": 0.17129828035831451, + "delta_nll": 0.03082418441772461, + "max_kl": 0.2161077857017517, + "mean_kl": 0.022919869050383568, + "reference_nll": 1.676177740097046, + "task_id": 555, + "token_count": 36, + "top1_agreement": 0.9166666865348816 + }, + { + "candidate_nll": 0.9304501414299011, + "code_tokens": 70, + "cvar95_kl": 0.07348554581403732, + "delta_nll": 0.013315379619598389, + "max_kl": 0.10585274547338486, + "mean_kl": 0.012828727252781391, + "reference_nll": 0.9171347618103027, + "task_id": 556, + "token_count": 70, + "top1_agreement": 0.9285714030265808 + }, + { + "candidate_nll": 1.2551378011703491, + "code_tokens": 17, + "cvar95_kl": 0.041086941957473755, + "delta_nll": -0.04432666301727295, + "max_kl": 0.041086941957473755, + "mean_kl": 0.008310675621032715, + "reference_nll": 1.299464464187622, + "task_id": 557, + "token_count": 17, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.158900499343872, + "code_tokens": 23, + "cvar95_kl": 0.10950013995170593, + "delta_nll": -0.026306748390197754, + "max_kl": 0.16496868431568146, + "mean_kl": 0.015521693043410778, + "reference_nll": 1.1852072477340698, + "task_id": 558, + "token_count": 23, + "top1_agreement": 0.95652174949646 + }, + { + "candidate_nll": 0.4103719890117645, + "code_tokens": 101, + "cvar95_kl": 0.20860743522644043, + "delta_nll": 0.03249841928482056, + "max_kl": 0.3610837459564209, + "mean_kl": 0.026239631697535515, + "reference_nll": 0.37787356972694397, + "task_id": 559, + "token_count": 101, + "top1_agreement": 0.9702970385551453 + }, + { + "candidate_nll": 1.4322352409362793, + "code_tokens": 36, + "cvar95_kl": 0.1319730281829834, + "delta_nll": 0.01735854148864746, + "max_kl": 0.18636009097099304, + "mean_kl": 0.01186325866729021, + "reference_nll": 1.4148766994476318, + "task_id": 560, + "token_count": 36, + "top1_agreement": 0.9722222089767456 + }, + { + "candidate_nll": 1.218471884727478, + "code_tokens": 46, + "cvar95_kl": 0.19982950389385223, + "delta_nll": 0.01271820068359375, + "max_kl": 0.20204168558120728, + "mean_kl": 0.03325272724032402, + "reference_nll": 1.2057536840438843, + "task_id": 561, + "token_count": 46, + "top1_agreement": 0.9347826242446899 + }, + { + "candidate_nll": 1.5119072198867798, + "code_tokens": 24, + "cvar95_kl": 0.0273531973361969, + "delta_nll": -0.01893937587738037, + "max_kl": 0.032491713762283325, + "mean_kl": 0.00651096785441041, + "reference_nll": 1.5308465957641602, + "task_id": 562, + "token_count": 24, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.369685411453247, + "code_tokens": 22, + "cvar95_kl": 0.041619762778282166, + "delta_nll": -0.0056438446044921875, + "max_kl": 0.05858394503593445, + "mean_kl": 0.008983158506453037, + "reference_nll": 1.3753292560577393, + "task_id": 563, + "token_count": 22, + "top1_agreement": 0.8636363744735718 + }, + { + "candidate_nll": 0.681867778301239, + "code_tokens": 58, + "cvar95_kl": 0.05319838598370552, + "delta_nll": -0.007004737854003906, + "max_kl": 0.06289892643690109, + "mean_kl": 0.010830877348780632, + "reference_nll": 0.6888725161552429, + "task_id": 564, + "token_count": 58, + "top1_agreement": 0.9655172228813171 + }, + { + "candidate_nll": 1.2680355310440063, + "code_tokens": 15, + "cvar95_kl": 0.03196674585342407, + "delta_nll": -0.04483497142791748, + "max_kl": 0.03196674585342407, + "mean_kl": 0.007303756196051836, + "reference_nll": 1.3128705024719238, + "task_id": 565, + "token_count": 15, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.4492163062095642, + "code_tokens": 40, + "cvar95_kl": 0.07097062468528748, + "delta_nll": 0.009628713130950928, + "max_kl": 0.08086491376161575, + "mean_kl": 0.008994369767606258, + "reference_nll": 0.4395875930786133, + "task_id": 566, + "token_count": 40, + "top1_agreement": 0.949999988079071 + }, + { + "candidate_nll": 0.6044443249702454, + "code_tokens": 37, + "cvar95_kl": 0.026683036237955093, + "delta_nll": -0.00010526180267333984, + "max_kl": 0.027514927089214325, + "mean_kl": 0.005014335736632347, + "reference_nll": 0.6045495867729187, + "task_id": 567, + "token_count": 37, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9824373126029968, + "code_tokens": 21, + "cvar95_kl": 0.015797551721334457, + "delta_nll": -0.03780859708786011, + "max_kl": 0.01850840449333191, + "mean_kl": 0.003907002042979002, + "reference_nll": 1.020245909690857, + "task_id": 568, + "token_count": 21, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.349914789199829, + "code_tokens": 21, + "cvar95_kl": 0.038860201835632324, + "delta_nll": 0.03739893436431885, + "max_kl": 0.06383292376995087, + "mean_kl": 0.007492546457797289, + "reference_nll": 1.3125158548355103, + "task_id": 569, + "token_count": 21, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 0.99249267578125, + "code_tokens": 66, + "cvar95_kl": 0.08850803971290588, + "delta_nll": -0.01587200164794922, + "max_kl": 0.11617076396942139, + "mean_kl": 0.013016321696341038, + "reference_nll": 1.0083646774291992, + "task_id": 570, + "token_count": 66, + "top1_agreement": 0.9696969985961914 + }, + { + "candidate_nll": 0.6934188604354858, + "code_tokens": 138, + "cvar95_kl": 0.2441990077495575, + "delta_nll": 0.049758076667785645, + "max_kl": 0.4365391135215759, + "mean_kl": 0.03093114122748375, + "reference_nll": 0.6436607837677002, + "task_id": 571, + "token_count": 138, + "top1_agreement": 0.9130434989929199 + }, + { + "candidate_nll": 0.739786684513092, + "code_tokens": 22, + "cvar95_kl": 0.026122286915779114, + "delta_nll": 0.005544066429138184, + "max_kl": 0.028681382536888123, + "mean_kl": 0.005450439173728228, + "reference_nll": 0.7342426180839539, + "task_id": 572, + "token_count": 22, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7674877643585205, + "code_tokens": 37, + "cvar95_kl": 0.040544264018535614, + "delta_nll": 0.02190345525741577, + "max_kl": 0.061314377933740616, + "mean_kl": 0.005652249790728092, + "reference_nll": 0.7455843091011047, + "task_id": 573, + "token_count": 37, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9666153788566589, + "code_tokens": 41, + "cvar95_kl": 0.07650309056043625, + "delta_nll": 0.015857696533203125, + "max_kl": 0.14486275613307953, + "mean_kl": 0.010647878982126713, + "reference_nll": 0.9507576823234558, + "task_id": 574, + "token_count": 41, + "top1_agreement": 0.9756097793579102 + }, + { + "candidate_nll": 1.1536811590194702, + "code_tokens": 63, + "cvar95_kl": 0.11447852849960327, + "delta_nll": -0.00480961799621582, + "max_kl": 0.18589890003204346, + "mean_kl": 0.01784656010568142, + "reference_nll": 1.158490777015686, + "task_id": 575, + "token_count": 63, + "top1_agreement": 0.9523809552192688 + }, + { + "candidate_nll": 0.8475289940834045, + "code_tokens": 99, + "cvar95_kl": 0.184243842959404, + "delta_nll": 0.0007771849632263184, + "max_kl": 0.4424732029438019, + "mean_kl": 0.023722533136606216, + "reference_nll": 0.8467518091201782, + "task_id": 576, + "token_count": 99, + "top1_agreement": 0.9494949579238892 + }, + { + "candidate_nll": 0.8943237066268921, + "code_tokens": 65, + "cvar95_kl": 0.14719203114509583, + "delta_nll": -0.0357699990272522, + "max_kl": 0.34054166078567505, + "mean_kl": 0.016750378534197807, + "reference_nll": 0.9300937056541443, + "task_id": 577, + "token_count": 65, + "top1_agreement": 0.9076923131942749 + }, + { + "candidate_nll": 0.9316058158874512, + "code_tokens": 39, + "cvar95_kl": 0.09272844344377518, + "delta_nll": 0.014076709747314453, + "max_kl": 0.14058896899223328, + "mean_kl": 0.010741003789007664, + "reference_nll": 0.9175291061401367, + "task_id": 578, + "token_count": 39, + "top1_agreement": 0.9743589758872986 + }, + { + "candidate_nll": 1.1899582147598267, + "code_tokens": 40, + "cvar95_kl": 0.2120864987373352, + "delta_nll": 0.031197428703308105, + "max_kl": 0.3122096061706543, + "mean_kl": 0.018761320039629936, + "reference_nll": 1.1587607860565186, + "task_id": 579, + "token_count": 40, + "top1_agreement": 0.9750000238418579 + }, + { + "candidate_nll": 1.4174281358718872, + "code_tokens": 97, + "cvar95_kl": 0.5087294578552246, + "delta_nll": 0.12247598171234131, + "max_kl": 1.8912776708602905, + "mean_kl": 0.04524639621376991, + "reference_nll": 1.294952154159546, + "task_id": 580, + "token_count": 97, + "top1_agreement": 0.9587628841400146 + }, + { + "candidate_nll": 1.8928806781768799, + "code_tokens": 23, + "cvar95_kl": 0.04963965713977814, + "delta_nll": -0.02214813232421875, + "max_kl": 0.05526602268218994, + "mean_kl": 0.013627070002257824, + "reference_nll": 1.9150288105010986, + "task_id": 581, + "token_count": 23, + "top1_agreement": 0.9130434989929199 + }, + { + "candidate_nll": 1.042033076286316, + "code_tokens": 25, + "cvar95_kl": 0.0241045244038105, + "delta_nll": 0.0081634521484375, + "max_kl": 0.02425907365977764, + "mean_kl": 0.006869237404316664, + "reference_nll": 1.0338696241378784, + "task_id": 582, + "token_count": 25, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.9004217982292175, + "code_tokens": 57, + "cvar95_kl": 0.11127126216888428, + "delta_nll": 0.04489314556121826, + "max_kl": 0.14365985989570618, + "mean_kl": 0.014802717603743076, + "reference_nll": 0.8555286526679993, + "task_id": 583, + "token_count": 57, + "top1_agreement": 0.9824561476707458 + }, + { + "candidate_nll": 1.1279243230819702, + "code_tokens": 50, + "cvar95_kl": 0.07442963868379593, + "delta_nll": 0.08562469482421875, + "max_kl": 0.09288983792066574, + "mean_kl": 0.013507149182260036, + "reference_nll": 1.0422996282577515, + "task_id": 584, + "token_count": 50, + "top1_agreement": 0.9599999785423279 + }, + { + "candidate_nll": 0.8225131034851074, + "code_tokens": 35, + "cvar95_kl": 0.04097820818424225, + "delta_nll": -0.006070613861083984, + "max_kl": 0.0435415580868721, + "mean_kl": 0.010026775300502777, + "reference_nll": 0.8285837173461914, + "task_id": 585, + "token_count": 35, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 2.360024929046631, + "code_tokens": 29, + "cvar95_kl": 0.06289687007665634, + "delta_nll": 0.015612125396728516, + "max_kl": 0.07199275493621826, + "mean_kl": 0.01701418124139309, + "reference_nll": 2.3444128036499023, + "task_id": 586, + "token_count": 29, + "top1_agreement": 0.931034505367279 + }, + { + "candidate_nll": 1.395137071609497, + "code_tokens": 20, + "cvar95_kl": 0.028027383610606194, + "delta_nll": 0.06171905994415283, + "max_kl": 0.028027383610606194, + "mean_kl": 0.007811838295310736, + "reference_nll": 1.3334180116653442, + "task_id": 587, + "token_count": 20, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1223641633987427, + "code_tokens": 19, + "cvar95_kl": 0.019005559384822845, + "delta_nll": 0.0057212114334106445, + "max_kl": 0.019005559384822845, + "mean_kl": 0.0036423613782972097, + "reference_nll": 1.116642951965332, + "task_id": 588, + "token_count": 19, + "top1_agreement": 0.9473684430122375 + }, + { + "candidate_nll": 0.9433404803276062, + "code_tokens": 71, + "cvar95_kl": 0.1857498586177826, + "delta_nll": -0.002943098545074463, + "max_kl": 0.2259632796049118, + "mean_kl": 0.030054807662963867, + "reference_nll": 0.9462835788726807, + "task_id": 589, + "token_count": 71, + "top1_agreement": 0.9436619877815247 + }, + { + "candidate_nll": 1.7910467386245728, + "code_tokens": 47, + "cvar95_kl": 0.20864252746105194, + "delta_nll": 0.05615389347076416, + "max_kl": 0.2512853145599365, + "mean_kl": 0.03971352428197861, + "reference_nll": 1.7348928451538086, + "task_id": 590, + "token_count": 47, + "top1_agreement": 0.936170220375061 + }, + { + "candidate_nll": 0.8018981218338013, + "code_tokens": 50, + "cvar95_kl": 0.03860790655016899, + "delta_nll": -0.00012433528900146484, + "max_kl": 0.04857344180345535, + "mean_kl": 0.005222035571932793, + "reference_nll": 0.8020224571228027, + "task_id": 591, + "token_count": 50, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 1.1075141429901123, + "code_tokens": 113, + "cvar95_kl": 0.39631226658821106, + "delta_nll": 0.05555319786071777, + "max_kl": 0.6558016538619995, + "mean_kl": 0.04300699010491371, + "reference_nll": 1.0519609451293945, + "task_id": 592, + "token_count": 113, + "top1_agreement": 0.9380530714988708 + }, + { + "candidate_nll": 1.4733006954193115, + "code_tokens": 26, + "cvar95_kl": 0.057321272790431976, + "delta_nll": 0.033142924308776855, + "max_kl": 0.06559640169143677, + "mean_kl": 0.012979870662093163, + "reference_nll": 1.4401577711105347, + "task_id": 593, + "token_count": 26, + "top1_agreement": 0.9615384340286255 + }, + { + "candidate_nll": 0.58243727684021, + "code_tokens": 62, + "cvar95_kl": 0.09759952872991562, + "delta_nll": 0.014178633689880371, + "max_kl": 0.21242763102054596, + "mean_kl": 0.011362023651599884, + "reference_nll": 0.5682586431503296, + "task_id": 594, + "token_count": 62, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.7537509799003601, + "code_tokens": 79, + "cvar95_kl": 0.16904088854789734, + "delta_nll": 0.0010324716567993164, + "max_kl": 0.260947585105896, + "mean_kl": 0.0160736832767725, + "reference_nll": 0.7527185082435608, + "task_id": 595, + "token_count": 79, + "top1_agreement": 0.9873417615890503 + }, + { + "candidate_nll": 1.6090612411499023, + "code_tokens": 20, + "cvar95_kl": 0.04581956937909126, + "delta_nll": -0.01958751678466797, + "max_kl": 0.04581956937909126, + "mean_kl": 0.014439888298511505, + "reference_nll": 1.6286487579345703, + "task_id": 596, + "token_count": 20, + "top1_agreement": 1.0 + }, + { + "candidate_nll": 0.4341113269329071, + "code_tokens": 186, + "cvar95_kl": 0.41045594215393066, + "delta_nll": 0.030966907739639282, + "max_kl": 0.6801561713218689, + "mean_kl": 0.04283649101853371, + "reference_nll": 0.4031444191932678, + "task_id": 597, + "token_count": 186, + "top1_agreement": 0.9569892287254333 + }, + { + "candidate_nll": 0.9033347964286804, + "code_tokens": 99, + "cvar95_kl": 0.3176195025444031, + "delta_nll": 0.06018698215484619, + "max_kl": 0.6391966342926025, + "mean_kl": 0.03759373724460602, + "reference_nll": 0.8431478142738342, + "task_id": 598, + "token_count": 99, + "top1_agreement": 0.9797979593276978 + }, + { + "candidate_nll": 0.8171030282974243, + "code_tokens": 43, + "cvar95_kl": 0.06297186017036438, + "delta_nll": 0.0052947998046875, + "max_kl": 0.10375432670116425, + "mean_kl": 0.007723804097622633, + "reference_nll": 0.8118082284927368, + "task_id": 599, + "token_count": 43, + "top1_agreement": 0.9534883499145508 + }, + { + "candidate_nll": 1.4629303216934204, + "code_tokens": 35, + "cvar95_kl": 0.021748017519712448, + "delta_nll": -0.010863423347473145, + "max_kl": 0.026382938027381897, + "mean_kl": 0.005695376545190811, + "reference_nll": 1.4737937450408936, + "task_id": 600, + "token_count": 35, + "top1_agreement": 0.9428571462631226 + } + ], + "policy": { + "default_bits": 8, + "name": "uniform_int8_nearest", + "rounding": "nearest", + "seed": 2339, + "upgrade_layer": null + }, + "storage": { + "exact_byte_gate": true, + "expected_resident_bytes": 4792320, + "full_precision_equivalent_bytes": 18874368, + "largest_transient_state_bytes": 1048576, + "physical_reduction_realized": true, + "resident_bytes": 4792320, + "resident_compression_ratio": 3.9384615384615387 + }, + "task_delta_nll_distribution": { + "maximum": 0.12247598171234131, + "median": 0.013696044683456421, + "minimum": -0.07192385196685791, + "q25": -0.0026708990335464478, + "q75": 0.033521294593811035 + }, + "task_macro": { + "candidate_nll": 1.0723152567942937, + "delta_nll": 0.015591054161389669, + "mean_kl": 0.019127020025108424, + "reference_nll": 1.056724202632904, + "task_count": 90, + "top1_agreement": 0.9653024699952867 + }, + "token_weighted": { + "candidate_nll": 0.9535408616065979, + "cvar95_kl": 0.26553651690483093, + "delta_nll": 0.025882244110107422, + "max_kl": 3.0969557762145996, + "mean_kl": 0.026149943470954895, + "reference_nll": 0.9276586174964905, + "token_count": 5524, + "top1_agreement": 0.9587255716323853 + } + } + }, + "claim_scope": { + "confirmation_touched": false, + "generated_code_executed": false, + "phase": "development", + "protocol_eligible": true, + "speed_claim_allowed": false, + "teacher_forced_fidelity_only": true, + "whole_model_memory_claim_allowed": false + }, + "continuation_decision": { + "all_gates_pass": true, + "confirmation_permitted": true, + "gates": { + "all_values_finite": true, + "equal_byte_bootstrap_interval_above_zero": true, + "exact_resident_bytes": true, + "primary_cvar95_token_kl_lower_than_uniform_int4": true, + "primary_macro_delta_nll_reduction_at_least_15_percent": true, + "primary_mean_token_kl_lower_than_uniform_int4": true, + "primary_top1_not_lower_than_uniform_int4": true + } + }, + "contrasts": { + "primary_vs_mean_random_equal_byte": { + "paired_bootstrap": { + "bootstrap_samples": 10000, + "confidence": 0.95, + "confidence_interval": [ + 1.965983425400875, + 2.194512400218734 + ], + "mean_improvement": 2.078384230534236, + "paired_examples": 90, + "seed": 2339 + } + }, + "primary_vs_uniform_int4": { + "paired_bootstrap": { + "bootstrap_samples": 10000, + "confidence": 0.95, + "confidence_interval": [ + 2.0807794316444133, + 2.3214647861652904 + ], + "mean_improvement": 2.197980460855696, + "paired_examples": 90, + "seed": 2339 + }, + "primary_macro_delta_nll": 0.7664886196454366, + "relative_reduction": 0.7414415199379093, + "uniform_macro_delta_nll": 2.9644690805011327 + } + }, + "environment": { + "command": [ + "python.exe", + "scripts\\evaluate_mbpp.py", + "--phase", + "development", + "--calibration-artifact", + "evidence\\mbpp-v02-calibration.json", + "--prepared-manifest", + "evidence\\mbpp-v02-development-manifest.json", + "--local-files-only", + "--output", + "artifacts\\mbpp-v02-development.json" + ], + "cuda_runtime": "12.8", + "datasets": "4.8.5", + "device": "cuda", + "device_name": "NVIDIA GeForce RTX 5070 Laptop GPU", + "model_dtype": "torch.bfloat16", + "nvidia_driver_versions": [ + "592.15" + ], + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "torch": "2.11.0+cu128", + "tracked_worktree_clean": true, + "transformers": "5.14.1" + }, + "schedule": { + "candidate_generated_tokens_fed_back": false, + "candidate_order": [ + "uniform_int4_nearest", + "uniform_int8_nearest", + "read_risk_l0_nearest", + "mse_selected_nearest", + "random_l18_nearest", + "random_l4_nearest", + "random_l13_nearest", + "read_risk_l0_stochastic_seed_2339", + "read_risk_l0_stochastic_seed_2340", + "read_risk_l0_stochastic_seed_2341" + ], + "checkpoint_path": "artifacts\\mbpp-v02-development.checkpoint.json", + "elapsed_wall_seconds_not_a_latency_benchmark": 4198.574854500002, + "final_checkpoint_sha256": "982eda24faf5799353367ab4fea6a266ee6e078f2a6bc58e1606909f79ef8e5e", + "group_size": 128, + "phase": "development", + "resumed_task_count": 0, + "row_count": 90, + "run_signature_sha256": "5721890a6f76a5fdcfeb9cac2edf6dae1218102fff3950f88b5d1d91b183fa06", + "scored_first_code_token_from_prefill": true, + "seed": 2339 + }, + "source": { + "calibration_artifact_path": "evidence\\mbpp-v02-calibration.json", + "calibration_evidence_sha256": "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee", + "dataset_manifest": { + "config": "full", + "dataset_id": "google-research-datasets/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "development", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "row_count": 90, + "rows": [ + { + "sha256": "16539782772e967a03051a2424573c762df4d1a0b59fbf720916caa9f988c859", + "task_id": 511 + }, + { + "sha256": "f76d352612a44fc427a30d6a58e5248a52d9e3be67e22e5879db0f565824d11f", + "task_id": 512 + }, + { + "sha256": "33557131013ba79f2690961b71d2144c4c40a64d1384aaeece38c4b4daef0f46", + "task_id": 513 + }, + { + "sha256": "aa78e8ee8b097226d95cb1e3725711aafc807b76ac501cd4741b983763af352a", + "task_id": 514 + }, + { + "sha256": "7b6f0b9f04efaa01338dd58b5e752158ed79dda7efddbbdd72346b72a9a61385", + "task_id": 515 + }, + { + "sha256": "dfd91fce57caf634b348738899534dcaa25f01f399c9eb3d226f8cd7abd7f349", + "task_id": 516 + }, + { + "sha256": "99174fe6e04ee92839869dc8c3e41e5d08b7777ba5659d674e36b30e8a336b40", + "task_id": 517 + }, + { + "sha256": "b9a1893c5fe1eefde29928bc5a4e92dae8de3a14f6db55cc9a970c063dbe69f4", + "task_id": 518 + }, + { + "sha256": "b2cbba2b099669b6d34afb7aa1faba00425831e57544d6f10832e0fd222c5e72", + "task_id": 519 + }, + { + "sha256": "3d333ac553e93e284761c6ccefa1c12a8dc79029a6164c885c153637a6cf6151", + "task_id": 520 + }, + { + "sha256": "09e4fa8324f54eb313947ffee0c3cc905fb95ea82d74d85eeb39e73555f48911", + "task_id": 521 + }, + { + "sha256": "048a2a45595f7ffb348c4751a64579a58d27aaeb13c6892318bbcf79d06bc5e5", + "task_id": 522 + }, + { + "sha256": "cc57ed07e290bcb08265c17225fd97f414f8b8cc7b71b377625c9de373f451a2", + "task_id": 523 + }, + { + "sha256": "7137b48b5888b18f28401d92529ec16111b5fd3193ffaca44b515a6063968405", + "task_id": 524 + }, + { + "sha256": "350a6b69a72702e6118599a3c89489a8992035e9f34c6a5ab2d7222f523ac76a", + "task_id": 525 + }, + { + "sha256": "462863306f0b5a3b547ffb08bdf59913700b3e591a7111f34b8c4155630c867c", + "task_id": 526 + }, + { + "sha256": "1c9ced29d24570a5cf27a6ae168799bf794e229f623306f885ad4f9f5e5febe2", + "task_id": 527 + }, + { + "sha256": "88660b5a56b37453ac32739a92cd8c7fa870906057b09d87e679f80925c02639", + "task_id": 528 + }, + { + "sha256": "f2c36ff713e36744ca27b046dda2eff681a29bf2b70c38d375344a3814c64e90", + "task_id": 529 + }, + { + "sha256": "fc4f6f81691062ff7a8a10627345b1a4189e4f069f1d15295bec02b1eba34d65", + "task_id": 530 + }, + { + "sha256": "63c5923940787de5251261d2ab48c51c43ed505b004c7abe67e2a304b80c6bb9", + "task_id": 531 + }, + { + "sha256": "5cffdbcc7332c69e5fadba155c1a56d20214275a2290b1faea43b9d7454ae0ac", + "task_id": 532 + }, + { + "sha256": "34507c24ec617e760d596c33536375821d165844c053d909aec94e69a5457c5e", + "task_id": 533 + }, + { + "sha256": "981115e5f0d1d2eabe511128f24790c317b8603d4703a9514eddbecfb852c4f0", + "task_id": 534 + }, + { + "sha256": "d44acaada238005e937cb67ffd3294028f3e9ab9225f36d395dea28b9ad1a8f2", + "task_id": 535 + }, + { + "sha256": "df8e21d38a6aa5fce8e18c704c3b9f2046ac39f18283d1c8b68ab67a37564240", + "task_id": 536 + }, + { + "sha256": "6fcfcf8527b287979934b381c55da12c4aadf3dfef6c50c97a0c3d702bf8564e", + "task_id": 537 + }, + { + "sha256": "5bba58d2fe0ae36a471ed13d20fe8f47dbc14673ec10bdb8edcf7398b3b3eae0", + "task_id": 538 + }, + { + "sha256": "2264ff589d94af8120dcd7409c7258d0fdd11f0dc51140949bfd3ec9d2fb6cf9", + "task_id": 539 + }, + { + "sha256": "50d5ad5b1b4b44d60a3e781ecfd4413823d35f32bd40dac603911273c195cca7", + "task_id": 540 + }, + { + "sha256": "d08d7abec63ce529ac868d965e93bd7569f36a4c306cc17cbc37396b29a3554f", + "task_id": 541 + }, + { + "sha256": "561a90eab2d613d8ae2a2ddca21b834e74e6222c9d2f558e1c0c73bb27169080", + "task_id": 542 + }, + { + "sha256": "9b8d91e99303fa2f55a734d90423b5f8efb03b9b0ebb301a69f37dfd8e18b159", + "task_id": 543 + }, + { + "sha256": "1356820047fc2bdfe32e7da566bb4a8d8dbed14f969f3f92ab9c336a40427f1c", + "task_id": 544 + }, + { + "sha256": "4e67548c2742da86a6330bb8d19d71d83f8968942f700369888d545d4cad7764", + "task_id": 545 + }, + { + "sha256": "9a73544e4d97993c354e5eac432a167ba0cef62fd6a278f7548cf87b9bddbc68", + "task_id": 546 + }, + { + "sha256": "57cbf36bde98ad9274dae823802dfe24b9691c166d64be25c3f816efd61d670a", + "task_id": 547 + }, + { + "sha256": "4b0aa73f6d6578d0eae9afd0c325ecd9330aa703e04fb86fa80bc2f8d7860365", + "task_id": 548 + }, + { + "sha256": "9b9468ec54691d614a877f4c12c76f485fa9a4b9f6e9da039ffbdd9a98a6fec5", + "task_id": 549 + }, + { + "sha256": "ccb08366a997c78e4ef6ad25895814c0710963d48954854ba892df32ac0ce178", + "task_id": 550 + }, + { + "sha256": "532dccbc3b4271ae51e5e9fb2d699111c42ccee8e92444e7146524c2e1f10df4", + "task_id": 551 + }, + { + "sha256": "72e288f580dd793c46766ca2e47956ed77cbcffa3b1b6fa04fe9d52b0456fce7", + "task_id": 552 + }, + { + "sha256": "1b908764f7f82f56e0fff3be241f18d9bde7dbfd40368cb99a42ca3edb587ea6", + "task_id": 553 + }, + { + "sha256": "f427548f544ac578a6a256cb50c50a4daee3a95cb6d9149895bad518d7dda229", + "task_id": 554 + }, + { + "sha256": "1451ea9e106beb9cb1279cefdbb1e8062d841f8b4c92b36d81d7e5a6f042659b", + "task_id": 555 + }, + { + "sha256": "34a10f500bcd085581e015c639910ed51630b50ea00760299fa0d6292d8d6819", + "task_id": 556 + }, + { + "sha256": "2c6163d2bb9ce3e8196681c440a9065ce290ab0e2a0f2763cf13d4c7a422a3b5", + "task_id": 557 + }, + { + "sha256": "efb5bff598fa1d04b6351fd14d94fbee3cff1c87f5ef46747408389a0350a396", + "task_id": 558 + }, + { + "sha256": "1eede60a0944c7972fc59bc112eaf31006052fb7cee778e73b64a35e34e8fd88", + "task_id": 559 + }, + { + "sha256": "55bdf6ded6325a008afed7382e77f41cbe1dd8401d7d04d2385fe41e621f6c09", + "task_id": 560 + }, + { + "sha256": "a2148f88d51a1d86d8e11ac3c2eff694c79c1b1dd5ba0edfaf16442f7c5f93e7", + "task_id": 561 + }, + { + "sha256": "666d4b0edc42c6167660973da91c6ac5e60121c4bc3bc9f15e13e7c65b1a208a", + "task_id": 562 + }, + { + "sha256": "682d4ba4668cffba0d4b249070175c6a8fa0e8a01df9466dc2b8378432dffc8b", + "task_id": 563 + }, + { + "sha256": "7787b4664632abd06d95b797075cab545108767ae1866a451a52895b291c4e21", + "task_id": 564 + }, + { + "sha256": "2b46b57870a6449c6618e2f71e5572b2c7efef268d4349cac7a9e41fe555b2a0", + "task_id": 565 + }, + { + "sha256": "9406fc598cede4b4a947644f7d348c168d5f8c108a1dc6564273f79973c94cad", + "task_id": 566 + }, + { + "sha256": "8c693a1b6d0296ffc204899a9d11c972986a1e3c2124110d18d9a87a83d8e725", + "task_id": 567 + }, + { + "sha256": "3e9462a2f5b09c148b1100396544b17c7ce10139b140be0c043aed4cc133cee2", + "task_id": 568 + }, + { + "sha256": "9f62b87560eb770a9f91a6945c8ecff1d7986355780f8e4cadb18b8dc50423f6", + "task_id": 569 + }, + { + "sha256": "8c2d543628a5813257413b916291a434a63b984ea8bed827340ca44a4dd17bb3", + "task_id": 570 + }, + { + "sha256": "61afb38ed4f433618e8800c09242f76faaab53a56a3ab09b690d313b7bab7755", + "task_id": 571 + }, + { + "sha256": "2c8a7379729fc7ec649d043decb0fbfe095623a81c1e863ed817b3d233b7b833", + "task_id": 572 + }, + { + "sha256": "0d282a2ed583a62b8a59069b53937214ba7d9afefd445308255ed340fd6421d1", + "task_id": 573 + }, + { + "sha256": "752df590804b76d952d8bc9878a9488b9e22813ba4c677eda742f63c7b8a0591", + "task_id": 574 + }, + { + "sha256": "734ae9ec818699b804770e128c8c26a97d6bdc595fdac4e9b96baec68342e3bd", + "task_id": 575 + }, + { + "sha256": "9d1d17fb32a8e4ddf053297f52eaf42d352a4f63c4965b69b4557f6556d3b708", + "task_id": 576 + }, + { + "sha256": "300e057c2818103dba1d7aa9ec1ac7375b8e6849545cd1cbe93d4cb6eb17e157", + "task_id": 577 + }, + { + "sha256": "30521482439370a44035109c6905e6e264509da5f50a25850e42f373cb616cad", + "task_id": 578 + }, + { + "sha256": "475595ff73dd1d2640b641f5457fd420ea1edfea26e5140eceab81550c3788e7", + "task_id": 579 + }, + { + "sha256": "a052fee8ed395eb8b7b33186776580eea3085af74ddda97c39993ced93b5c93c", + "task_id": 580 + }, + { + "sha256": "8c1674250fba855d56cadcb2fab7cf900b18a302bcab68fcdb01fa72b0c95ec0", + "task_id": 581 + }, + { + "sha256": "285dbf2c4747ad5230c093ca5013ebfbf7a4f003b95dd00f25deb4e0e3d46b89", + "task_id": 582 + }, + { + "sha256": "e01d6880195ba3de7dc9a2fca236b770a12a8b894b79b20c52c783053c04b74a", + "task_id": 583 + }, + { + "sha256": "cc2b5c6c9c995e0fc0620114e27e63c49bbb75aecc65095558b6f68639eb9793", + "task_id": 584 + }, + { + "sha256": "cf0921125e9bc2fd91ea0e210fb64214adb1a17e91bb7e065de3ff81c1f15fba", + "task_id": 585 + }, + { + "sha256": "cd84ae05c7934dfa84c07ce6db49b746451e500a5d5085ed370bbe9a8da52a34", + "task_id": 586 + }, + { + "sha256": "05e81cc4ddf7cc06f5331be8a8843cfff301058c5021493e3cda6754a1b5bbd7", + "task_id": 587 + }, + { + "sha256": "ff3d99e259cf33a48760347e7e72307ac9cdeddf41593e9106cb32da6403bcb5", + "task_id": 588 + }, + { + "sha256": "207117d0ce0432269271a0884d44d9386c843777421358da52fd9f8f6155b7fb", + "task_id": 589 + }, + { + "sha256": "858c5ff0068fb00087cea9a9517827e516ef3cb65245a2ba9f151c349aa960ce", + "task_id": 590 + }, + { + "sha256": "ca5e685cbc5a7c79af583dcf87b4b755248219a40cf06f9fa85ad2557fd19075", + "task_id": 591 + }, + { + "sha256": "41667aa322e7a77985945c02c82463e4dd0d8b404b9f3ebaea8bbf1adb971d00", + "task_id": 592 + }, + { + "sha256": "c8314d475632313295105669890a5afb95439738ac913806b440b8852ab4b71b", + "task_id": 593 + }, + { + "sha256": "15a67602e1d8469b29e2f8e6d91f362d75f52ab837f35cad404e4d0766fdcc91", + "task_id": 594 + }, + { + "sha256": "b1dd5af59b621b44a8c02d82957f9492a40010290f96d774c58ab6db33b39fe9", + "task_id": 595 + }, + { + "sha256": "0c7ad60813363a6d56a1477ce9f97fecfb10672c5c294bec621e03a86f602d63", + "task_id": 596 + }, + { + "sha256": "a7395a4e215b49910fdf20542f6f01bc57956d8c83c439c93ab7762f68cc649b", + "task_id": 597 + }, + { + "sha256": "2d65c727ca52be489f53491583c17306291d15997fde25bb061997a1bda4b745", + "task_id": 598 + }, + { + "sha256": "f1fce5b002964d65de0ff3ea547ff863953d76c66eccdb4cf9d351f262052c49", + "task_id": 599 + }, + { + "sha256": "5bc8c7767d4fde18f1b51f6fda718af7754e8102de557a96948d3846649e2952", + "task_id": 600 + } + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": null, + "source_split": "validation" + }, + "dataset_manifest_sha256": "8fed3da0aae864f4e30c70ad70b0269f759d3592dccb9ab87f24444fa24d65dc", + "model_id": "Qwen/Qwen3.5-0.8B-Base", + "model_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + "prepared_manifest_evidence_sha256": "2b13dfb1799472b2fd0006cb87cea60c67a7af37825a87f1686a6fe3d8e38d7c", + "repository_commit": "20a5ea95a8ed692600ee1645d2913f3a4b8a6795", + "token_manifest": [ + { + "code_token_ids_sha256": "51374b6a1aaaa4f94ea298d6c0ffd6cbff3feb07292b21ad8bb817749a0efb42", + "code_tokens": 63, + "prompt_token_ids_sha256": "75259b4b9ad144f8051ae373b42264664a1269f4a7b931145eb6fe023168cd77", + "prompt_tokens": 81, + "task_id": 511, + "total_tokens": 144 + }, + { + "code_token_ids_sha256": "77c6cbc907a8142e0967355cbe47e1383ff152405e182630e09dba83e5c9ddf9", + "code_tokens": 92, + "prompt_token_ids_sha256": "dd8d15e4f657d6628474dacc457182c8d1dd0c54ab2a11778c77a8e7b763bfbf", + "prompt_tokens": 231, + "task_id": 512, + "total_tokens": 323 + }, + { + "code_token_ids_sha256": "22100a0192c9be5cda5095a25f4f3bdfd5ec658d286b988cbdd4594f31b8e981", + "code_tokens": 36, + "prompt_token_ids_sha256": "069c694dc4cd41040430380369a4821bf38002a6cc85ed4aefff2810505aab53", + "prompt_tokens": 216, + "task_id": 513, + "total_tokens": 252 + }, + { + "code_token_ids_sha256": "0f48dd5315bc1cdb3f5535c4a8671bc275b8345f05d0b441899c7feadbe1ad1e", + "code_tokens": 24, + "prompt_token_ids_sha256": "d52b7be5af0294918bb9505625012c7fe12e114f908da8666f48204ab9d4fd6d", + "prompt_tokens": 122, + "task_id": 514, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "3159d14cd3c75e4d185b30935c27126c1af21ff91ee6bba3d7491da2fe940d0b", + "code_tokens": 153, + "prompt_token_ids_sha256": "cfcfc5b2d2cd0276903a66f889439012b326cce0cb7dc409595f98b53d301cfa", + "prompt_tokens": 102, + "task_id": 515, + "total_tokens": 255 + }, + { + "code_token_ids_sha256": "000c9196d18f4cdb0ffe066e528c9ffb20aba524f43c7defb02fb55236f98792", + "code_tokens": 133, + "prompt_token_ids_sha256": "d4b8a13b6e52b6e720ee8ca4d1e49f0ff192b2d6c5d3daab08878b7e42cb54e4", + "prompt_tokens": 172, + "task_id": 516, + "total_tokens": 305 + }, + { + "code_token_ids_sha256": "074f33d4d8b723867b2cecb32db5523ef4fab52ea7685b9a6ebb4c1bc5cb24c9", + "code_tokens": 39, + "prompt_token_ids_sha256": "00f9858ee09f99ee47de95b53b74e8c8289ae718e1b9fdbdaffd483d474779e5", + "prompt_tokens": 96, + "task_id": 517, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "769a9f5dec545efe90416aff1c3092d35e4ad33d7e0366ecde6d3fcf461b6c06", + "code_tokens": 26, + "prompt_token_ids_sha256": "3b5a4c340457d6d619ecf1b0873619439dea3ee4db377c6fd07c8b802aa7a83f", + "prompt_tokens": 67, + "task_id": 518, + "total_tokens": 93 + }, + { + "code_token_ids_sha256": "4601be81bcfb579e9985d389a9bfd95d01de6d7b34089f49f52d5b070fec303f", + "code_tokens": 38, + "prompt_token_ids_sha256": "10701accb1e298180004e4faca7dca1941a92f4cfd48c6a0c7c8baf45154b02f", + "prompt_tokens": 93, + "task_id": 519, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "32f7307f163387d6ac00fa8ed642db762ed28eabc2ad53b9259b55b5abcc7958", + "code_tokens": 173, + "prompt_token_ids_sha256": "9f4f84a3737a6307b999d18d3c33f4b0167e897ca21ec55319705298b8494912", + "prompt_tokens": 111, + "task_id": 520, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "82b30bc50679559260db86fbe7ec7c9c813ca57a40d8569aa18030ec4afc0ee9", + "code_tokens": 37, + "prompt_token_ids_sha256": "446652e33075f3163e9f51b858ee2f47f9ed74277b41361215777184214faac5", + "prompt_tokens": 90, + "task_id": 521, + "total_tokens": 127 + }, + { + "code_token_ids_sha256": "bdb5c51e5be3003a1977927d8cfc86c1a173411fad980ad763e5f102f41fb3fd", + "code_tokens": 205, + "prompt_token_ids_sha256": "254263abe8129d6ba10402421cfa441b86486a6a0adf383bdaef1bdda87050a2", + "prompt_tokens": 164, + "task_id": 522, + "total_tokens": 369 + }, + { + "code_token_ids_sha256": "92604d63a2c91ea949c6422282f570352135688738bd36a48a60afa9c8ab97eb", + "code_tokens": 162, + "prompt_token_ids_sha256": "2676c6487486db2d9d24f9402c2db16d2d0398b4edb9d610383b9e28cc17d0f2", + "prompt_tokens": 122, + "task_id": 523, + "total_tokens": 284 + }, + { + "code_token_ids_sha256": "18f82cf0a7adda89da894b33582306c3d65b03e82444a657f1f381fa666d1c3b", + "code_tokens": 132, + "prompt_token_ids_sha256": "2d7972ca89b84d7df49723b706a76057f6512d98f094afd3f14ea566a0ffbf38", + "prompt_tokens": 140, + "task_id": 524, + "total_tokens": 272 + }, + { + "code_token_ids_sha256": "adbd4743c7317532925a3538361651dfb7648a76d95f444557b8bf7e9d0cf2dd", + "code_tokens": 33, + "prompt_token_ids_sha256": "3b4c9309ec726ff6903516c27fd4ad78c81adc968d05be79b40d06dfe49bec2d", + "prompt_tokens": 96, + "task_id": 525, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "c933d3b7225620deb30b98c5dcdb75530fe71324523e45d42b41ee37fc981a8e", + "code_tokens": 60, + "prompt_token_ids_sha256": "27a68f8991973a0d22b0d3a2aac80206f9054298ee64dd5f0a7de6646c5c86e7", + "prompt_tokens": 92, + "task_id": 526, + "total_tokens": 152 + }, + { + "code_token_ids_sha256": "cc3c1f5fea2e1aff7b35f0d792546a3586ad8f6451f29ff934adac7fabb27cad", + "code_tokens": 63, + "prompt_token_ids_sha256": "b9ed5d0bb5681bcf79b906ca643db22f66e72f00e00601749d95cf1f728939da", + "prompt_tokens": 127, + "task_id": 527, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "9c751aa9dc10a946442d4588bf03dcf7c9e261c685441aea99d1ae3e719bd9b4", + "code_tokens": 48, + "prompt_token_ids_sha256": "e36501d3283d8b41af319cb3fa079080eab9b2f5e99c310b374d7845d4d84e63", + "prompt_tokens": 140, + "task_id": 528, + "total_tokens": 188 + }, + { + "code_token_ids_sha256": "143ec378ef4a3491fcefd1cd895cdd4481b0d3e84ca81a44c8e0f4b401f4a0a1", + "code_tokens": 76, + "prompt_token_ids_sha256": "4cb8a5b00a5a292cb2ed0c860ec6de91dceaaeb8d6927af1285ad2df6e67f205", + "prompt_tokens": 85, + "task_id": 529, + "total_tokens": 161 + }, + { + "code_token_ids_sha256": "8138852a6a1ab85b2ce53557ac38dd11fe3d67b0763225027c6108f42d72479d", + "code_tokens": 63, + "prompt_token_ids_sha256": "6d2b7047e1098ae60806bfa835123a5da10f882ed0b9ae8072ac3ec48e353b05", + "prompt_tokens": 175, + "task_id": 530, + "total_tokens": 238 + }, + { + "code_token_ids_sha256": "3d00f2afaae35918bc269ee7c39bcd0d546375083cdbb6260e3c5f3245292f9c", + "code_tokens": 107, + "prompt_token_ids_sha256": "eb0d6242db32c79d92aad0ae39255447d6fa5072a6673127d19dfda8b3838f2a", + "prompt_tokens": 107, + "task_id": 531, + "total_tokens": 214 + }, + { + "code_token_ids_sha256": "20aac8f787234bcaa336e6ae465d6a10a395e328e85700937f1f24edb10dc1de", + "code_tokens": 108, + "prompt_token_ids_sha256": "06294dbec467aa598b4371d9d9d813b7a437b967041d26094a7364a88245a57c", + "prompt_tokens": 84, + "task_id": 532, + "total_tokens": 192 + }, + { + "code_token_ids_sha256": "46d59a814add2a2778a5e40848bfef52ce2bcf53dc4b432443756e7048830764", + "code_tokens": 46, + "prompt_token_ids_sha256": "c09c801c7c6ac0b649151d1849e098ce881c25a6fcb9e12e60c70ab7c792f107", + "prompt_tokens": 136, + "task_id": 533, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "e35bd424ab07b25c36f31bd857e6b218edf65b42db4435e3714633d07e6cabd3", + "code_tokens": 37, + "prompt_token_ids_sha256": "575d23bccdbee14ecb2a336e4590c89ac9ecd69802a31a08db34866d12f06111", + "prompt_tokens": 102, + "task_id": 534, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "4f65a9707281c698d80cec5c5422f8b142d54115f6d9d948d759ed0de696a02a", + "code_tokens": 29, + "prompt_token_ids_sha256": "3ee1ca9226cb8adadc0932ea4785e1a9141ea7b119b3086940ee52df83fd1b67", + "prompt_tokens": 110, + "task_id": 535, + "total_tokens": 139 + }, + { + "code_token_ids_sha256": "e26dfa7ced89a887560aceff73afc50e9c2bd530b4237113873f3cbd6953b418", + "code_tokens": 13, + "prompt_token_ids_sha256": "3289151a3436f9f0d6b4e0e86e19e49706da28817f7c278fb8205a6a965f028c", + "prompt_tokens": 147, + "task_id": 536, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "d38931f56e911f2b37ca025f685c144e0b91198aba10e4934e306ec0c223c18a", + "code_tokens": 51, + "prompt_token_ids_sha256": "33c14d0229cade7ac382322706f276d02dfddef6588774dfa4c548900ee25611", + "prompt_tokens": 90, + "task_id": 537, + "total_tokens": 141 + }, + { + "code_token_ids_sha256": "4cf4521bbd35856831644bcb747c1925ffaddb8a00bec610a71a87db3f4972a0", + "code_tokens": 29, + "prompt_token_ids_sha256": "f7363fdc3b2b06194c20e182a14171b4118c78bc7d2fa5b08af71ee6d19f3aff", + "prompt_tokens": 144, + "task_id": 538, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "e6f786238b93dddcc4ce4fe1211e622c2f39c49bbbfca7cc06387cc357c1d75d", + "code_tokens": 30, + "prompt_token_ids_sha256": "7c64a69f1a3710020356623911ae68407e8c77f117ed60148666e91ad7be2b6c", + "prompt_tokens": 701, + "task_id": 539, + "total_tokens": 731 + }, + { + "code_token_ids_sha256": "ebfd1768c1c99bb8e2b5861818c73fdfe71ce2ef9725d65dc074e1ae056f194f", + "code_tokens": 103, + "prompt_token_ids_sha256": "e0c55cd39206f43105dbf5350e8d8ada362ddaa569801e04613be2e285a319cb", + "prompt_tokens": 124, + "task_id": 540, + "total_tokens": 227 + }, + { + "code_token_ids_sha256": "d9354f12861dec6dce841b82c6622d90d6b30271b8d09885282ee2852cc2f1a0", + "code_tokens": 119, + "prompt_token_ids_sha256": "05a59f82d30668cb33b13052dcfc64216005b820ec7331267951a00459b3d650", + "prompt_tokens": 76, + "task_id": 541, + "total_tokens": 195 + }, + { + "code_token_ids_sha256": "cea3a4282e98a85f19f1205dddd0e6dde09d24ca6e89d2694576f95d4b0ebffe", + "code_tokens": 22, + "prompt_token_ids_sha256": "1163a0581db48fcd93d6a868d446004a50afa275a664a4a42b92ac6299cb48b1", + "prompt_tokens": 120, + "task_id": 542, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "97c3db09b220d423604683a36b402917de5ef017595b5fe127b03033680326a8", + "code_tokens": 51, + "prompt_token_ids_sha256": "8a901d20e8392578080c95c58fdd664005a24577338ce80ade129c03ffc26b70", + "prompt_tokens": 102, + "task_id": 543, + "total_tokens": 153 + }, + { + "code_token_ids_sha256": "a4e481b7ad37f0fdc86a49666d9b76d0904317dda1e496bb9872122e7cdaf515", + "code_tokens": 32, + "prompt_token_ids_sha256": "0634b3db81a69635cbf3b87ee15272ff7e1f8636b3e435c27de19bcb37729164", + "prompt_tokens": 214, + "task_id": 544, + "total_tokens": 246 + }, + { + "code_token_ids_sha256": "024f5f88354da3fe4564aef78da670ce8fd0dfa2c4841da17e21ef48661f9af7", + "code_tokens": 116, + "prompt_token_ids_sha256": "9023a4481fc5950af9ebb1c1f997fc5c4c1e581960997207b7228eb740a1ebc1", + "prompt_tokens": 84, + "task_id": 545, + "total_tokens": 200 + }, + { + "code_token_ids_sha256": "b515f1f4c958b0461b960569600bca4a378fa19a9ee822f9030348ae49c68900", + "code_tokens": 56, + "prompt_token_ids_sha256": "a4a0328eb44f57e83b2e9333ebcba25eb7dd20b2a6ea3994fec0ff40738a7707", + "prompt_tokens": 79, + "task_id": 546, + "total_tokens": 135 + }, + { + "code_token_ids_sha256": "c4a23f4ee95a4913df228749ed6716734c5fa8514a81f882d4cdc8b50f1a642b", + "code_tokens": 53, + "prompt_token_ids_sha256": "65f76fdbff8c9bec92167d208b27f552857a87c5aa5074b1348de1f02adaac3f", + "prompt_tokens": 89, + "task_id": 547, + "total_tokens": 142 + }, + { + "code_token_ids_sha256": "0198e0a8df3b9ebe2cb0ecb13b3e3af2737b8a049c64ec617f58319d7e6a40d0", + "code_tokens": 127, + "prompt_token_ids_sha256": "acd3f8a15a3349d5bbb4940e11200a437fbac95f8936ce403c20cd5f17e7af67", + "prompt_tokens": 144, + "task_id": 548, + "total_tokens": 271 + }, + { + "code_token_ids_sha256": "c0464aa780660588b93032dd6e3f725c05bf5de9c6ee6dcce9d8e2ff52b63149", + "code_tokens": 61, + "prompt_token_ids_sha256": "21fff0ae75e9ab5439adb71bfe4c61c78b7e1126846aac5a15fe300dcf9a0183", + "prompt_tokens": 85, + "task_id": 549, + "total_tokens": 146 + }, + { + "code_token_ids_sha256": "0393dde6e334233444b072427256a83ce1733d121aed9aefa1012f60290f1fad", + "code_tokens": 147, + "prompt_token_ids_sha256": "c1c4083ea9b0310f65f2ffcc4f9f4670da5f8c0614679e349f487730ebd6d962", + "prompt_tokens": 104, + "task_id": 550, + "total_tokens": 251 + }, + { + "code_token_ids_sha256": "916b13753989ac021c075b35a952673224a9c6472d2169cf6e0ba31db04b945f", + "code_tokens": 28, + "prompt_token_ids_sha256": "12bbc144384b85e069d3902d8dc9a6a9d1c490c373edabd87faac4e38ba471e6", + "prompt_tokens": 192, + "task_id": 551, + "total_tokens": 220 + }, + { + "code_token_ids_sha256": "032bb173044d1b1baad318413f1e9a319de5fb55cc3970d095ba60ea9661bb01", + "code_tokens": 66, + "prompt_token_ids_sha256": "50529dbe74b719b38f1ed65f449a60c15cfd00fa0cb0a37e443c7bae3faddaad", + "prompt_tokens": 100, + "task_id": 552, + "total_tokens": 166 + }, + { + "code_token_ids_sha256": "7c356523732a7257d7795ace17f6949088120e447cabd6d8e79054e2bf2ba861", + "code_tokens": 33, + "prompt_token_ids_sha256": "a0885f78db1f8f9dd822fef4e9ba4563285905ec835f90d5774bfdf9a4fb6b83", + "prompt_tokens": 98, + "task_id": 553, + "total_tokens": 131 + }, + { + "code_token_ids_sha256": "a03f695dd30a01cbbaa31308ddfeb9d51d49684c571fc9bdce218bbb064b21ea", + "code_tokens": 41, + "prompt_token_ids_sha256": "67d69de764f87d5cf6afc07ca06ae5cd4214993c4a0a4b0dd872b60382017064", + "prompt_tokens": 107, + "task_id": 554, + "total_tokens": 148 + }, + { + "code_token_ids_sha256": "4c96f929f3c9cf8910e259e2b54308c7fa4858dc8ab59fac3202b6e08522484e", + "code_tokens": 36, + "prompt_token_ids_sha256": "ff9f070b889222df347d50b3873a368ff8d35eca5c3d5395a8c2bf7e50253418", + "prompt_tokens": 82, + "task_id": 555, + "total_tokens": 118 + }, + { + "code_token_ids_sha256": "7215dc188f10a72e1420e7ff3ab14f7260be052a9f4095d408d5e262ee94b666", + "code_tokens": 70, + "prompt_token_ids_sha256": "1ecf16e96af48a484540536f9b1bbc6d01cea072dca6a2574d0fb568fe4500bd", + "prompt_tokens": 112, + "task_id": 556, + "total_tokens": 182 + }, + { + "code_token_ids_sha256": "06df1dab9774538ddae69cd618fc09e7f63f2c97c5e4818770d3ed38ba908259", + "code_tokens": 17, + "prompt_token_ids_sha256": "07983a4a48a52077b6ff90eaaaf6920db154647cdfe9511a772d294583c163eb", + "prompt_tokens": 78, + "task_id": 557, + "total_tokens": 95 + }, + { + "code_token_ids_sha256": "0c461b835bbc821963a03738489506b9e9de4d394c39626b80d38c462ef96e77", + "code_tokens": 23, + "prompt_token_ids_sha256": "d253628f23c9e1bd5bef177cb0c90fb20a7ebae0d84b2b022e73d405030b03ea", + "prompt_tokens": 84, + "task_id": 558, + "total_tokens": 107 + }, + { + "code_token_ids_sha256": "5129d280fb187905d8f85b6f1d65906dcf3d30e1952b92cb316ff4f275ba964b", + "code_tokens": 101, + "prompt_token_ids_sha256": "87541f80a7e96d417c952a6d0b3221993a533cc896561e28926363b96da64445", + "prompt_tokens": 152, + "task_id": 559, + "total_tokens": 253 + }, + { + "code_token_ids_sha256": "c2c08c5013c0499acdab47c393de668803b0b9104eec67319389a37d4c0a5f25", + "code_tokens": 36, + "prompt_token_ids_sha256": "11258d79258bb892a11ffbef932656b8d6854baf7183e72728f9c25e7285fce8", + "prompt_tokens": 201, + "task_id": 560, + "total_tokens": 237 + }, + { + "code_token_ids_sha256": "5d95e6e698c4a0356b7e5cda82d46476cb44b66fe434bc4aa2edb704d0720bde", + "code_tokens": 46, + "prompt_token_ids_sha256": "5561ed193f416f7b3bc28893683b061159c9efc497a2274b45d64711b59ec0f6", + "prompt_tokens": 266, + "task_id": 561, + "total_tokens": 312 + }, + { + "code_token_ids_sha256": "e27e7bfdb239e4650f0614f134b84d1d988e9b4e48935e706ef19c00b7ed3a3e", + "code_tokens": 24, + "prompt_token_ids_sha256": "b2b117219ac089754a20f737af8241a878354c891f7223b88d891feef1bfcc85", + "prompt_tokens": 126, + "task_id": 562, + "total_tokens": 150 + }, + { + "code_token_ids_sha256": "cab54112e86c322a8e94c3f0333b5484b6c3937a8af2d6f9117f506b8cb2ad9c", + "code_tokens": 22, + "prompt_token_ids_sha256": "c8a705344cba3f1b53d50cf0d20ebc9abf1558025a8ed6390ec5c8e666dc9d7c", + "prompt_tokens": 104, + "task_id": 563, + "total_tokens": 126 + }, + { + "code_token_ids_sha256": "86de1f8f78bfb419746e498637d064512cb8bc9bbe389d5b9116fd548df14ee7", + "code_tokens": 58, + "prompt_token_ids_sha256": "523346c7e4a28f98c0a353c1afd625c7d4ac18c59facdcefb2e68f76e6391597", + "prompt_tokens": 98, + "task_id": 564, + "total_tokens": 156 + }, + { + "code_token_ids_sha256": "1225aeb027e811b628fdc7e265bac65a2cccee232f70f778ff683a47f305acaf", + "code_tokens": 15, + "prompt_token_ids_sha256": "af8a292c6485ff7a67aa7be7b61dd4f32a0adabb6cc637e841f315db89900c4a", + "prompt_tokens": 95, + "task_id": 565, + "total_tokens": 110 + }, + { + "code_token_ids_sha256": "c0cb7caed68465ebca3f4e0c860f4222e5d3911cdac8462ecfe52f3819dfebc9", + "code_tokens": 40, + "prompt_token_ids_sha256": "1eaf3e3744abb794f24c4a9c5868e0d414eb9ef8807d572384f1e8e095478711", + "prompt_tokens": 69, + "task_id": 566, + "total_tokens": 109 + }, + { + "code_token_ids_sha256": "3f6ef7be7f6afc6346e63d6dadc064c64b0d68a7c2247333c40941fc0eafabcf", + "code_tokens": 37, + "prompt_token_ids_sha256": "78a699beff16fe1ec862cd98e30dec9685bc79279179a4110eacc0390c04b627", + "prompt_tokens": 147, + "task_id": 567, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "c513feb62303b9292018bec3d45145c495d4288f5f9f70f58d15bafa0d82e977", + "code_tokens": 21, + "prompt_token_ids_sha256": "8982a4f08613e6078cb5a9bc7eea08f200d718ce53457c0f685d8dd201d0d338", + "prompt_tokens": 79, + "task_id": 568, + "total_tokens": 100 + }, + { + "code_token_ids_sha256": "0bfe140a51a356bc36803306c997a4deede74b240b28c3b2dcb31a9b809c8c1f", + "code_tokens": 21, + "prompt_token_ids_sha256": "6abbb94c85a6bf620395238e7e19749180c779e4293bd5f5fd88700e3f97015a", + "prompt_tokens": 184, + "task_id": 569, + "total_tokens": 205 + }, + { + "code_token_ids_sha256": "05850d31a888779f204340efca917fd9b19fddd87d630bd7cb442352bd7a3e66", + "code_tokens": 66, + "prompt_token_ids_sha256": "63f7b0721b16ddc2fd30e54479a7108b4a93763c21fbe12468f47855932638d7", + "prompt_tokens": 169, + "task_id": 570, + "total_tokens": 235 + }, + { + "code_token_ids_sha256": "04fd2b294cb34945613e5b9de88984a32d58796306b949f79414f253fdf65860", + "code_tokens": 138, + "prompt_token_ids_sha256": "73727137a63fae90dd5520430e344582c703eccd31d7ce84799137d07a28d649", + "prompt_tokens": 171, + "task_id": 571, + "total_tokens": 309 + }, + { + "code_token_ids_sha256": "f3b601ee8f39328aa6168c673aa7a25133e2146709681bc6bb696cf33dad5f54", + "code_tokens": 22, + "prompt_token_ids_sha256": "57d6b5a4961d3b8ab3e375fec11a10b37c759d067f83b0eebc7bad4c0294d79b", + "prompt_tokens": 135, + "task_id": 572, + "total_tokens": 157 + }, + { + "code_token_ids_sha256": "ab34c0081e5c216cb42afadfca0adf3c19b0dbe1e9a6ab497a786980efa46137", + "code_tokens": 37, + "prompt_token_ids_sha256": "beca07f10658235a9bc69d69405832cfbe1d6d84ae1d58afcf1678d68ac5d0f1", + "prompt_tokens": 136, + "task_id": 573, + "total_tokens": 173 + }, + { + "code_token_ids_sha256": "e4c376d85d55085bed7bde727b22416668f2567755d06476c0ef484a139cc6e8", + "code_tokens": 41, + "prompt_token_ids_sha256": "dc30e7d94350b7dc255ec828efbbeb0dee52859d5b98d6c74df26dc275ff1911", + "prompt_tokens": 104, + "task_id": 574, + "total_tokens": 145 + }, + { + "code_token_ids_sha256": "470667f4d5f96fdc5a083bdf428fc4c28bbac0db1192f14ac1a47db679617fff", + "code_tokens": 63, + "prompt_token_ids_sha256": "d023ce639ec2dcbc7e7a0ce0f2234119d1940d1a050b691ce8c583eecc2a6eb7", + "prompt_tokens": 101, + "task_id": 575, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "88d005f8c44e0a678858e25205e9eb911ca580807e4b4a9a0d2ea563a0d14903", + "code_tokens": 99, + "prompt_token_ids_sha256": "3a88fc72092f5076beec4b32b5a630f8df7355d9af2bcb895d42ef78daf74e7b", + "prompt_tokens": 117, + "task_id": 576, + "total_tokens": 216 + }, + { + "code_token_ids_sha256": "1681d2006b64d27c99a4add5ceaa2f0e11de29623677271ac6aac49b609b6ea2", + "code_tokens": 65, + "prompt_token_ids_sha256": "1267efcc04062cbda430893ade55eac4fca2ec7f21426024d180e6f770eae9e5", + "prompt_tokens": 86, + "task_id": 577, + "total_tokens": 151 + }, + { + "code_token_ids_sha256": "b2f7a02a32599611162313b8ec78e99927fa8de8033e5a50bbf731c96f29bfbb", + "code_tokens": 39, + "prompt_token_ids_sha256": "0ef0064fd0f4dbe294dd7db67c7a80d723171b72b954babe4fa5951ac544861f", + "prompt_tokens": 273, + "task_id": 578, + "total_tokens": 312 + }, + { + "code_token_ids_sha256": "36cfcd07cc222f406192992d0004a955ada75bd94bba1bd4127f4b2a996332af", + "code_tokens": 40, + "prompt_token_ids_sha256": "31b0c6f7aeab1caa69330d6ceff3781ba69ac2bb2f46a00c2ef2affb19117f14", + "prompt_tokens": 184, + "task_id": 579, + "total_tokens": 224 + }, + { + "code_token_ids_sha256": "7b4567d0f07f4d5852ce8684b988be0841a65366c2a9096063d04cc462239062", + "code_tokens": 97, + "prompt_token_ids_sha256": "edf415720df08853c0463118174d20e5daa5e58df4ac941d9d6522eabe18b524", + "prompt_tokens": 177, + "task_id": 580, + "total_tokens": 274 + }, + { + "code_token_ids_sha256": "2638278c24f225b62586e1dbd81c2a2747549f39bf002fab89eea18a49e42e05", + "code_tokens": 23, + "prompt_token_ids_sha256": "a679e29787a668b865f07fff79d29288f63ff4fbeb7c869b7ee49df872a1cb4e", + "prompt_tokens": 81, + "task_id": 581, + "total_tokens": 104 + }, + { + "code_token_ids_sha256": "60bdae86096f5b23913acf6eace3638935051c4ee6312860b4d18735ac68907d", + "code_tokens": 25, + "prompt_token_ids_sha256": "045d1683f12b963599f152102d9bb9d3e9b820a7d6c08b4879f4c40788c4cff1", + "prompt_tokens": 67, + "task_id": 582, + "total_tokens": 92 + }, + { + "code_token_ids_sha256": "d616ceeb5baf6144088303469aaad4f54625879238b4854a9aa38125be917cc4", + "code_tokens": 57, + "prompt_token_ids_sha256": "b64dcce5c6936277b647d6e13611de98831e6fbe61857987475ae71ec0be7779", + "prompt_tokens": 72, + "task_id": 583, + "total_tokens": 129 + }, + { + "code_token_ids_sha256": "6cca857ae596513638ba100c5de4ffdeb88df0acbb51642271e1261efc8a410f", + "code_tokens": 50, + "prompt_token_ids_sha256": "b784b1b932fe81aac8959dd6437b6bb02a1f02797087ab84a486598b54d15dcb", + "prompt_tokens": 114, + "task_id": 584, + "total_tokens": 164 + }, + { + "code_token_ids_sha256": "aba30fa670c5f07f2b26e0c5f6314c31268993bd210b6f32463eeffe179f02c3", + "code_tokens": 35, + "prompt_token_ids_sha256": "3f3d1b321ad2d9cedbc03840b0b8e53d1674f2df51496a9ff6821f73aa780d0b", + "prompt_tokens": 310, + "task_id": 585, + "total_tokens": 345 + }, + { + "code_token_ids_sha256": "96c31933f9fdca36e7cefc47888825398ca0a8450205d4bbf3e010b280906704", + "code_tokens": 29, + "prompt_token_ids_sha256": "5c17a36d98e4571cf2da7b0200b11d30a32ba5d52a7ba9a472912609214e64cf", + "prompt_tokens": 159, + "task_id": 586, + "total_tokens": 188 + }, + { + "code_token_ids_sha256": "71625271f8ed82e626b3c864229dad2a05008c15c8243ac229d352293e21f693", + "code_tokens": 20, + "prompt_token_ids_sha256": "c09ab27564d859ff4f20e7381932285ccbf9882f4d92caf229545a28d045d17b", + "prompt_tokens": 163, + "task_id": 587, + "total_tokens": 183 + }, + { + "code_token_ids_sha256": "7625cecdcfaf215b042d36e4bf4e036373f11e7bb00b404dd5047b7e52a92b2c", + "code_tokens": 19, + "prompt_token_ids_sha256": "3d1a5aece2df5d1a46135904ea9e2329bfa1cab7635348956e5b174cefad7410", + "prompt_tokens": 89, + "task_id": 588, + "total_tokens": 108 + }, + { + "code_token_ids_sha256": "8237eb0d6ac12638b2b7d912fbc2e7f463be40a997bbc34ba838eb58978f3a5e", + "code_tokens": 71, + "prompt_token_ids_sha256": "980ca6f6dc5995989f774a2c0ad0ab1868477a1be7fe355d7dc0a97d3bc19bb3", + "prompt_tokens": 131, + "task_id": 589, + "total_tokens": 202 + }, + { + "code_token_ids_sha256": "aa89cf8d51a5e5c6451b748749ffa09c015a08874708c9d5221d1490ab573e61", + "code_tokens": 47, + "prompt_token_ids_sha256": "11b67c36adc8358b170008df9bb3e82a9b709aab9cbcc908eb2084e3ec8bf04f", + "prompt_tokens": 249, + "task_id": 590, + "total_tokens": 296 + }, + { + "code_token_ids_sha256": "7b07b1e6c1f0050b648a8cb7e315156bcbdc3fc41f2072fafc66b3bee84d76cd", + "code_tokens": 50, + "prompt_token_ids_sha256": "9ca2ba10212bb44e5c594536e8c56d109e300d873ba4d8ce6c1bf8929dcf8934", + "prompt_tokens": 130, + "task_id": 591, + "total_tokens": 180 + }, + { + "code_token_ids_sha256": "527507dc33ed195f959ba5d9eb4a9319095b88ea156e4c90a1207ab13e59dfdd", + "code_tokens": 113, + "prompt_token_ids_sha256": "4781f3198d0e1616079cd96eb30cc938ea3c1c0ceb5741e8980c3f3ae51312b4", + "prompt_tokens": 77, + "task_id": 592, + "total_tokens": 190 + }, + { + "code_token_ids_sha256": "f0102775691666d079133bf1824e8a244c8625708e333807f283a722586f8135", + "code_tokens": 26, + "prompt_token_ids_sha256": "5c4de560c133f6d86b586eaf9091855a43c52d1647735dd0dfed53ec11a03ac3", + "prompt_tokens": 134, + "task_id": 593, + "total_tokens": 160 + }, + { + "code_token_ids_sha256": "cea1449991b5740774cb67f9b41b4d21138ce0d715912221375ebe8b6d391985", + "code_tokens": 62, + "prompt_token_ids_sha256": "c860f17a1e8b23e40417cbd3ea961144577844cd93c8daf78e3fc6b8d7e04eaa", + "prompt_tokens": 113, + "task_id": 594, + "total_tokens": 175 + }, + { + "code_token_ids_sha256": "a0bc157b791156284cb59efa7c8b5323e5ceff2cd0570c4c613b2a60b4c74dce", + "code_tokens": 79, + "prompt_token_ids_sha256": "68dc877d1339ccd102b221aac28e7289332c535debf453ece9b207008ddc469a", + "prompt_tokens": 105, + "task_id": 595, + "total_tokens": 184 + }, + { + "code_token_ids_sha256": "1ed12c70ac0e1f42b45254e355df9a2f2fb344bc83d23709ab50d39daf1e9a72", + "code_tokens": 20, + "prompt_token_ids_sha256": "50bc50fa175cd0ab564bac33ffd115aacf5e5b59b541ef0c70467fe3f3d13ddf", + "prompt_tokens": 202, + "task_id": 596, + "total_tokens": 222 + }, + { + "code_token_ids_sha256": "7e9b720e1a6de57fda397dba6a641c141745797fbbdf065c7d2682521e6a2710", + "code_tokens": 186, + "prompt_token_ids_sha256": "1f175f5a46d86e9fb077540932e5420027773ceb901c18665caf88ba2ed6126a", + "prompt_tokens": 209, + "task_id": 597, + "total_tokens": 395 + }, + { + "code_token_ids_sha256": "5c1585934b732425f8a4a794d598ee0d72bcb0c36e2a91efb1c5bf4e558a854a", + "code_tokens": 99, + "prompt_token_ids_sha256": "4580aa7ec549d56af454e8619ab5e37750b99259c83564f622cc2b3274cb6213", + "prompt_tokens": 75, + "task_id": 598, + "total_tokens": 174 + }, + { + "code_token_ids_sha256": "101147ad3965a807926a2f2dca4a0b64cb4170b35f412e3e7bb32030fa44587a", + "code_tokens": 43, + "prompt_token_ids_sha256": "b5ec314f6027e558c46e6bb121bbbbb03cfd3447dd6878de9c6f58e348f67720", + "prompt_tokens": 94, + "task_id": 599, + "total_tokens": 137 + }, + { + "code_token_ids_sha256": "ad71990faceebd8ab5e0df092325c417c0cec66bdf5dc1d8d9aacc06ba3d2ec5", + "code_tokens": 35, + "prompt_token_ids_sha256": "f5292225c83fe88c95f576f67804ffbef8674929550913a759f1adc1f1394977", + "prompt_tokens": 74, + "task_id": 600, + "total_tokens": 109 + } + ], + "token_manifest_sha256": "3c19f37f3c35cb22f17e66a5438bb9968b214e0aff64067930bc3a7124f63f5c", + "tokenizer_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + }, + "validity": { + "configured_gdn_layer_indices": [ + 0, + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 16, + 17, + 18, + 20, + 21, + 22 + ], + "packed_qdq_preflight": { + "absolute_tolerance": 1e-06, + "maximum_absolute_difference_by_candidate": { + "mse_selected_nearest": 0.0, + "random_l13_nearest": 0.0, + "random_l18_nearest": 0.0, + "random_l4_nearest": 0.0, + "read_risk_l0_nearest": 0.0, + "read_risk_l0_stochastic_seed_2339": 0.0, + "read_risk_l0_stochastic_seed_2340": 0.0, + "read_risk_l0_stochastic_seed_2341": 0.0, + "uniform_int4_nearest": 0.0, + "uniform_int8_nearest": 0.0 + }, + "passed": true, + "task_id": 945 + }, + "reference_recurrent_state_bytes": 18874368 + } + }, + "schema_version": 1 +} diff --git a/examples/qwen35_quickstart.py b/examples/qwen35_quickstart.py new file mode 100644 index 0000000..50dee77 --- /dev/null +++ b/examples/qwen35_quickstart.py @@ -0,0 +1,6 @@ +"""Run the installed RecurQuant Qwen3.5 quickstart workflow.""" + +from recurquant.qwen35_quickstart import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/notebooks/recurquant_qwen35_colab.ipynb b/notebooks/recurquant_qwen35_colab.ipynb new file mode 100644 index 0000000..e8977c1 --- /dev/null +++ b/notebooks/recurquant_qwen35_colab.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "intro", + "metadata": {}, + "source": [ + "# RecurQuant: packed Qwen3.5 recurrent-state demo\n", + "\n", + "Run these cells from top to bottom in a fresh **Google Colab GPU runtime**. The demo installs RecurQuant from an immutable GitHub ref, downloads the public `Qwen/Qwen3.5-0.8B-Base` checkpoint at the revision pinned by RecurQuant, and runs the installed `recurquant qwen35` command. No Hugging Face token is required.\n", + "\n", + "The command uses RecurQuant's `mixed-v02` default: model layer 0 is stored at INT8 and the other supported recurrent layers at INT4, with group size 128 and FP16 scales. This is an alpha research demo, not a speed or whole-model memory benchmark." + ] + }, + { + "cell_type": "markdown", + "id": "install-note", + "metadata": {}, + "source": [ + "## 1. Install the pinned package\n", + "\n", + "The immutable `v0.2.0a1` tag keeps the notebook tied to the tested alpha release." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "install", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "RECURQUANT_REPOSITORY = \"https://github.com/Labeeb2339/recurquant.git\"\n", + "RECURQUANT_REF = \"v0.2.0a1\" # Immutable alpha release tag.\n", + "install_target = f\"recurquant @ git+{RECURQUANT_REPOSITORY}@{RECURQUANT_REF}\"\n", + "subprocess.run(\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"--disable-pip-version-check\",\n", + " install_target,\n", + " ],\n", + " check=True,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "id": "runtime-note", + "metadata": {}, + "source": [ + "## 2. Check the Colab runtime\n", + "\n", + "If CUDA is missing, choose **Runtime > Change runtime type > GPU** and run the notebook again. BF16 is the configuration used for RecurQuant's public full-model fidelity evidence. On an older GPU without BF16, the installed command can fall back to FP16, but that weight dtype is outside the current full-model evidence boundary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "runtime-check", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "from importlib.metadata import version\n", + "\n", + "import torch\n", + "\n", + "if not torch.cuda.is_available():\n", + " raise RuntimeError(\n", + " \"CUDA is unavailable. In Colab, select Runtime > Change runtime type > GPU, \"\n", + " \"then run all cells again.\"\n", + " )\n", + "\n", + "gpu_name = torch.cuda.get_device_name(0)\n", + "bf16_supported = torch.cuda.is_bf16_supported()\n", + "print(f\"GPU: {gpu_name}\")\n", + "print(f\"CUDA available: {torch.cuda.is_available()}\")\n", + "print(f\"BF16 supported: {bf16_supported}\")\n", + "print(f\"RecurQuant: {version('recurquant')}\")\n", + "print(f\"PyTorch: {torch.__version__}\")\n", + "print(f\"Transformers: {version('transformers')}\")\n", + "\n", + "if not bf16_supported:\n", + " warnings.warn(\n", + " \"This GPU does not support BF16. The CLI will fall back to FP16, but \"\n", + " \"RecurQuant's public full-model fidelity evidence currently covers BF16 only.\",\n", + " RuntimeWarning,\n", + " stacklevel=1,\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "run-note", + "metadata": {}, + "source": [ + "## 3. Run the installed mixed-v02 default\n", + "\n", + "The first run downloads the pinned 0.8B model and can take several minutes. The command deliberately omits `--policy`, so it exercises the installed `mixed-v02` default rather than the uniform-INT4 stress baseline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "run", + "metadata": {}, + "outputs": [], + "source": [ + "import shlex\n", + "\n", + "PROMPT = \"Explain recurrent-state quantization in two short sentences.\"\n", + "MAX_NEW_TOKENS = 24\n", + "command = [\n", + " \"recurquant\",\n", + " \"qwen35\",\n", + " \"--device\",\n", + " \"cuda\",\n", + " \"--max-new-tokens\",\n", + " str(MAX_NEW_TOKENS),\n", + " \"--prompt\",\n", + " PROMPT,\n", + "]\n", + "print(\"Running:\", shlex.join(command))\n", + "subprocess.run(command, check=True)\n" + ] + }, + { + "cell_type": "markdown", + "id": "counters", + "metadata": {}, + "source": [ + "## Read the byte counters honestly\n", + "\n", + "- `resident_recurrent_state_bytes` is the exact live tensor storage used by the packed persistent recurrent states.\n", + "- `full_precision_equivalent_recurrent_state_bytes` is how many bytes those same states would occupy in their original model dtype.\n", + "- `largest_materialized_recurrent_state_bytes` is the size of the largest single state materialized while a layer runs. It is **not** the layer's full workspace or a CUDA allocator peak.\n", + "- `resident_compression_ratio` compares only the first two counters.\n", + "\n", + "These numbers do not measure total GPU memory, model weights, ordinary attention KV caches, activations, temporary quantization workspaces, or inference speed. The current Python implementation materializes one recurrent state during layer execution, so this notebook makes no claim of lower peak VRAM or lower latency. The generated text is a functional demo, not a generated-output quality evaluation." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "recurquant_qwen35_colab.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index c7f6894..d704dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,20 +4,40 @@ build-backend = "hatchling.build" [project] name = "recurquant" -version = "0.1.0.dev0" -description = "Research harness for persistent-state quantization in Gated DeltaNet language models." +version = "0.2.0a1" +description = "Packed recurrent-state quantization for Gated DeltaNet language models." readme = "README.md" requires-python = ">=3.11" -license = { file = "LICENSE" } +license = "Apache-2.0" +license-files = ["LICENSE"] authors = [ { name = "Muhammad Labeeb Aryan" }, ] +keywords = [ + "gated-deltanet", + "language-model-inference", + "quantization", + "recurrent-state", + "transformers", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] dependencies = [ "huggingface-hub>=0.36", "numpy>=2.0", + "packaging>=24", "safetensors>=0.5", "torch>=2.7", - "transformers>=5.14.1,<6", + # PackedRecurrentStateCache subclasses an internal cache layer. Keep the + # alpha on the exact tested release until another release is covered in CI. + "transformers==5.14.1", ] [project.optional-dependencies] @@ -26,6 +46,15 @@ dev = [ "pytest-cov>=6.2", "ruff>=0.12", ] +eval = [ + "datasets>=4.3,<5", +] + +[project.urls] +Homepage = "https://github.com/Labeeb2339/recurquant" +Repository = "https://github.com/Labeeb2339/recurquant" +Issues = "https://github.com/Labeeb2339/recurquant/issues" +"Compatibility and support" = "https://github.com/Labeeb2339/recurquant/blob/main/docs/compatibility.md" [project.scripts] recurquant = "recurquant.cli:main" diff --git a/research/CALIBRATION_002.md b/research/CALIBRATION_002.md new file mode 100644 index 0000000..85756c2 --- /dev/null +++ b/research/CALIBRATION_002.md @@ -0,0 +1,101 @@ +# Calibration 002: frozen MBPP layer plan + +Status: passed and frozen before development + +Date: 2026-07-22 + +## Frozen inputs + +- Repository commit: `cc35f4396ef4dd475908d8f96e05fe9c559f13be` +- Model: `Qwen/Qwen3.5-0.8B-Base` +- Model revision: `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68` +- Dataset: `google-research-datasets/mbpp`, configuration `full` +- Dataset revision: `4bb6404fdc6cacfda99d4ac4205087b89d32030c` +- Partition: the 128 training tasks fixed by + `SHA256("rq-v0.2|{task_id}")` in the public protocol +- Dataset manifest hash: + `129698d01bdf7f08989878ff7c980230456095c5b2cae42a010f567ecc49dc1a` +- Eligible single-token state-read steps: 7,134 + +The calibration artifact records Python, PyTorch, Transformers, Datasets, CUDA, +GPU, driver, command, clean-worktree status, task IDs, row hashes, token counts, +and both full 18-layer selector vectors. + +## Frozen outcome + +The normalized state-MSE baseline selected model layer `0`. Layer `0` also +ranked first under the diagnostic query-weighted read-risk measure. The primary +policy was already frozen to layer `0` before public calibration and was not +changed by this result. + +Highest normalized-state-MSE layers: + +| Rank | Model layer | Task-macro normalized state MSE | +| ---: | ---: | ---: | +| 1 | 0 | 0.175292 | +| 2 | 17 | 0.105471 | +| 3 | 18 | 0.091148 | +| 4 | 16 | 0.090392 | +| 5 | 21 | 0.081859 | + +Highest diagnostic read-risk layers: + +| Rank | Model layer | Task-macro read-risk | +| ---: | ---: | ---: | +| 1 | 0 | 0.397649 | +| 2 | 6 | 0.241932 | +| 3 | 5 | 0.228262 | +| 4 | 17 | 0.220239 | +| 5 | 16 | 0.213679 | + +The exact-byte development plan is now fixed: + +- `read_risk_l0_nearest`: layer 0 INT8, the other 17 GDN layers INT4. +- `mse_selected_nearest`: layer 0 INT8, the other 17 GDN layers INT4. +- Random controls: layers 18, 4, and 13 respectively at INT8. +- Uniform INT4 and INT8 quality anchors. +- Layer-0 mixed stochastic controls at base seeds 2339, 2340, and 2341. + +Because the independently specified MSE baseline selected the same layout as +the primary policy, their outputs should be identical. Both names remain in the +artifact so that the prespecified method matrix is not rewritten after seeing +calibration. + +## Reproduction check + +An initial full calibration completed before repository metadata was added to +the artifact schema. It was treated as incomplete evidence rather than edited +afterward. The full run was repeated from clean commit `cc35f43`. + +The two runs matched exactly on: + +- all candidate layers; +- every value in both 18-layer score vectors; +- all per-task token counts; +- the 7,134 eligible steps; and +- the complete dataset manifest. + +Accepted calibration evidence hash: +`7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee` + +Accepted file SHA256: +`d3d2f9acf6113ad455cce78d1b957a265c6675236845564d855c7cb537267125` + +## Split-isolation note + +During an earlier one-task diagnostic smoke, the non-streaming Hugging Face +Datasets builder prepared local cache files for all four MBPP splits even though +the loader was called with `split="train"`. No validation or test rows were +returned to the calibration code, inspected for method outcomes, or scored. The +loader was then changed to request streaming explicitly, and both accepted full +calibration runs received only the pinned training split. The confirmation guard still +rejects partial access and requires an explicit acknowledgement token before +the test split can be returned. + +## Decision + +Stage 0 passes. The frozen evaluator may now run all 90 validation tasks once. +This calibration agreement is not a public quality result, a novelty result, or +permission to open the 500-task confirmation split. Confirmation remains gated +on every preregistered development condition in +`PUBLIC_EVAL_PROTOCOL_V02.md`. diff --git a/research/CLAIM_BOUNDARY.md b/research/CLAIM_BOUNDARY.md index a2b30cc..0a2613d 100644 --- a/research/CLAIM_BOUNDARY.md +++ b/research/CLAIM_BOUNDARY.md @@ -5,24 +5,32 @@ Last reviewed: 2026-07-22 ## What is already established - Weight-only and activation quantization for language models are mature fields. -- SSM persistent/cached states have already been quantized, including 8-bit - cached-state work in [Quamba2](https://arxiv.org/abs/2503.22879). +- SSM persistent/cached states have already been quantized, including 4-bit + Mamba2 state caches in + [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) and 8-bit + cached-state kernels in [Quamba2](https://arxiv.org/abs/2503.22879). - Stochastic rounding, periodic state checkpoints, activation/update replay, and fewer persistent-state writes exist in current recurrent-model systems work. - Update residuals have been used as importance signals for bounded auxiliary memory in Gated DeltaNet. - Mixed precision and memory-budget allocation are broad existing ideas. -## Hypothesis that remains testable +## Scoped v0.2 confirmation finding -At an equal modeled persistent-state bit budget, Gated DeltaNet-specific -precision allocation may reduce tail output divergence relative to uniform -state quantization. The current candidate uses query-weighted recurrent-read -error. Mean decay, write gate, state-update magnitude, and residual magnitude -were tested and rejected as selectors in the diagnostic pilot. +On the pinned `Qwen/Qwen3.5-0.8B-Base` revision, the frozen layer-0 INT8 plus +17-layer INT4 policy passed every preregistered quality gate on all 500 untouched +MBPP test tasks and 30,244 teacher-forced reference-code tokens. Relative to +uniform INT4, task-macro excess NLL fell from 2.949743 to 0.803713 nats/token, a +72.75% reduction. The paired improvement was 2.146030 nats/token with a 95% +bootstrap interval of `[2.092249, 2.199866]`. The policy physically stored its +persistent recurrent-state payloads and FP16 scales in exactly 2,564,096 bytes. -This is a hypothesis, not a novelty or performance claim. A broader literature -audit and confirmatory experiments are required before stronger wording. +This confirms the frozen quality hypothesis only for the model, dataset +construction, and teacher-forced metric in +[`CONFIRMATION_002.md`](CONFIRMATION_002.md). It is not evidence of novelty, +generated-code correctness, lower latency, lower whole-model or peak memory, or +cross-model generality. Mean decay, write gate, state-update magnitude, and +residual magnitude remain rejected diagnostic selectors from the earlier pilot. ## Claims prohibited without new evidence @@ -30,14 +38,19 @@ audit and confirmatory experiments are required before stronger wording. - "First sub-8-bit recurrent model." - "First update-aware recurrent cache." - "Lossless" unless exact output equivalence is demonstrated in the stated mode. -- "Reduces memory" while only storing dequantized PyTorch tensors. -- "Speeds up inference" without a packed kernel and wall-clock comparison. -- "Breakthrough," "state of the art," or "novel" based only on the pilot. +- "Reduces total model memory by 7.36x." The measured ratio covers resident + recurrent-state payloads and scales only. +- "Speeds up inference" without passing the separate latency gate under the + stated benchmark. +- "Breakthrough," "state of the art," or "novel" based on the v0.2 evidence. - Authorship of Qwen3.5 or any other base model. ## What this project contributes -RecurQuant's contribution is its allocation method, implementation, evaluation -protocol, and clearly labelled derivative artifacts. It does not claim -authorship of Qwen3.5 or its architecture, which remain credited to the Qwen -team. +RecurQuant packages a physical low-bit recurrent-state cache, a frozen per-layer +allocation policy, exact-byte controls, an independently checkable confirmation +verifier, a preregistered evaluation protocol, and labelled evidence artifacts. +Its scoped held-out result is reproducible without expanding the claim. The +current Python path is not a fused low-bit recurrence kernel. RecurQuant does +not claim authorship of Qwen3.5 or its architecture, which remain credited to +the Qwen team. diff --git a/research/CONFIRMATION_002.md b/research/CONFIRMATION_002.md new file mode 100644 index 0000000..c36b203 --- /dev/null +++ b/research/CONFIRMATION_002.md @@ -0,0 +1,130 @@ +# MBPP confirmation report (v0.2) + +Date completed: 2026-07-22 + +Protocol: [`PUBLIC_EVAL_PROTOCOL_V02.md`](PUBLIC_EVAL_PROTOCOL_V02.md) + +Result artifact: [`../evidence/mbpp-v02-confirmation.json`](../evidence/mbpp-v02-confirmation.json) + +## Verdict + +The frozen v0.2 quality hypothesis passed on the untouched MBPP test split. +All preregistered validity, fidelity, equal-byte, and exact-storage gates passed, +including the confirmation-only requirement that the primary-versus-uniform +paired interval remain above zero. + +This is a scoped held-out result for one pinned model, dataset construction, +and teacher-forced metric. It is not a breakthrough, state-of-the-art, +first-method, generated-code-correctness, speed, whole-model-memory, or +cross-model claim. + +## Frozen scope + +- Model: `Qwen/Qwen3.5-0.8B-Base` at + `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`. +- Evaluated source commit: + `6bd5bed2b61e192526ba8fdbec8232801cbea843`, with a clean tracked worktree. +- Data: MBPP test task IDs 11 through 510, exactly 500 tasks. +- Scored tokens: 30,244 teacher-forced reference-code tokens. +- Frozen primary: recurrent model layer 0 at INT8, the other 17 Gated + DeltaNet layers at INT4, group size 128, FP16 scales, nearest rounding. +- Baselines: uniform INT4, uniform INT8, three equal-byte random layer + placements, the MSE-selected placement, and three stochastic-rounding seeds. + +Candidate-generated tokens were not fed back, executed, or graded. + +## Primary result + +| Measure | Uniform INT4 | Frozen mixed layout | Direction | +|---|---:|---:|---| +| Task-macro excess NLL, nats/token | 2.949743 | 0.803713 | 72.75% lower | +| Token-weighted mean KL | 3.149969 | 0.914580 | lower | +| Token-weighted worst-5% KL | 9.002207 | 4.839139 | lower | +| Token-weighted top-1 agreement | 0.321155 | 0.665190 | higher | + +The paired primary-versus-uniform task-macro improvement was 2.146030 +nats/token. Its 10,000-sample paired bootstrap 95% interval was +`[2.092249, 2.199866]`, entirely above zero. + +Against the mean of the three same-byte random high-precision layer +placements, the paired improvement was 2.033167 nats/token with a 95% interval +of `[1.980220, 2.086136]`, also entirely above zero. + +## Physical storage + +| Layout | Resident recurrent-state bytes | FP32-state equivalent ratio | +|---|---:|---:| +| FP32 reference | 18,874,368 | 1.000x | +| Uniform INT4 plus FP16 scales | 2,433,024 | 7.758x smaller | +| Layer 0 INT8, rest INT4, plus FP16 scales | 2,564,096 | 7.361x smaller | +| Uniform INT8 plus FP16 scales | 4,792,320 | 3.938x smaller | + +Every candidate matched its prespecified resident-byte count. Packed-cache and +QDQ preflight metrics matched exactly for every candidate; the measured maximum +absolute difference was 0.0 against a frozen tolerance of `1e-6`. + +These values cover persistent recurrent-state payloads and scales only. The +Python path still materializes one 1,048,576-byte recurrent state while its +layer executes. The result does not measure model weights, ordinary KV caches, +allocator overhead, peak CUDA memory, latency, or throughput. + +## Interpretation and negative controls + +- The confirmation reduction, 72.75%, is close to the 74.14% development + reduction without changing the frozen layer placement. +- The MSE selector also chose layer 0, so it is the exact same candidate and + not an independent win for the read-risk selector. +- All three equal-byte random placements were substantially worse than the + frozen layer-0 layout. +- Stochastic rounding was substantially worse across all three frozen seeds on + this protocol. That is a negative result for these settings, not a general + statement that stochastic rounding is ineffective. +- Uniform INT8 remained much closer to the FP32-state reference, but used + 4,792,320 resident bytes rather than the mixed layout's 2,564,096 bytes. + +The evidence therefore supports a narrow result: on this pinned protocol, a +prespecified one-layer INT8 allocation preserved teacher-forced fidelity much +better than uniform INT4 and same-byte random placements while physically +storing the persistent recurrent state in 2,564,096 bytes. + +## Integrity anchors + +- Final artifact file SHA256: + `70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9` +- Final canonical evidence SHA256: + `2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0` +- Final checkpoint file SHA256: + `df0040cc9cebdbc442992e75d19f9090456f9b249da062c095840e731b6c4609` +- Final checkpoint canonical-state SHA256: + `1293d93cb620d2193e9251f49c05d0bdaeebde16d3515c1f0e021c96b5d4fe1c` +- Frozen run-signature SHA256: + `5d15268224357bb078315ef2c2b6e710a7eb8a2734df2527f9637b382951c78a` +- Prepared-manifest file SHA256: + `c6a7d0db6ef7577a66ac19fbbc0be166279488f6a6be432b364bd9eb6833f7b0` +- Prepared-manifest canonical evidence SHA256: + `21a6d18c6a0887b1499d156a3d610d4bfafdd59d3557713485b62038e263b96a` + +Using the retained local checkpoint, +`recurquant verify-confirmation` independently reconstructed the per-task and +token summaries from the final arrays, re-ran every gate, verified the source +and manifest anchors, and returned `result: pass`, `checkpoint_verified: true`, +and `outcome_verified: true` with no warnings or errors. The raw checkpoint is +not committed to Git; a compressed copy is distributed with the `v0.2.0a1` +release so this verification mode remains available without enlarging the +repository. + +## Infrastructure record + +The evaluator wrote an atomic checkpoint after every task. The long run was +resumed only after infrastructure interruptions, with the same command, +frozen source commit, prepared manifest, calibration artifact, and candidate +plan. One attempted resume stalled before evaluation because the streamed +dataset loader lacked network access; it was stopped before the checkpoint +changed. A later process exited during an atomic checkpoint replacement after +370 accepted tasks, leaving the previous checkpoint intact and a separate +temporary file. The final run resumed from the accepted 370-task checkpoint +and recomputed the interrupted next task once. + +No partial candidate metric was inspected, no policy or gate was changed, and +no unfavorable result was rerun. Outcomes were opened only after all 500 tasks +completed and the final artifact existed. diff --git a/research/DEVELOPMENT_002.md b/research/DEVELOPMENT_002.md new file mode 100644 index 0000000..9d29adb --- /dev/null +++ b/research/DEVELOPMENT_002.md @@ -0,0 +1,98 @@ +# MBPP v0.2 development decision + +Status: all frozen Stage 1 gates passed; the v0.2 policy remains unchanged and +untouched confirmation is authorized + +Decision recorded: 2026-07-22 + +## Frozen run + +- Phase: MBPP `validation`, task IDs 511-600 (all 90 tasks). +- Scored target: 5,524 reference-code tokens under teacher forcing. +- Model: `Qwen/Qwen3.5-0.8B-Base` at + `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`. +- Source commit: `20a5ea95a8ed692600ee1645d2913f3a4b8a6795`. +- Repository state recorded by the evaluator: tracked worktree clean. +- Device: NVIDIA GeForce RTX 5070 Laptop GPU; BF16 model weights; driver + 592.15. +- Command: + + ```powershell + .\.venv\Scripts\python.exe scripts\evaluate_mbpp.py ` + --phase development ` + --calibration-artifact evidence\mbpp-v02-calibration.json ` + --prepared-manifest evidence\mbpp-v02-development-manifest.json ` + --local-files-only ` + --output artifacts\mbpp-v02-development.json + ``` + +The run completed without resuming. Candidate-generated tokens were never fed +back, no generated code was executed, and the first code token was scored from +the prefill logits. + +## Primary result + +The frozen primary policy stores model layer 0 at INT8 and the other 17 Gated +DeltaNet recurrent states at INT4. Its exact resident state is 2,564,096 bytes, +or 7.361x smaller than the 18,874,368-byte FP32 recurrent-state reference. + +| Method | Resident bytes | Macro excess NLL | Mean token KL | Worst-5% token KL | Top-1 agreement | +|---|---:|---:|---:|---:|---:| +| Uniform INT4 nearest | 2,433,024 | 2.96447 | 3.12032 | 9.12815 | 0.32694 | +| Layer 0 INT8, rest INT4 nearest | 2,564,096 | 0.76649 | 0.90949 | 4.91772 | 0.68338 | +| Uniform INT8 nearest | 4,792,320 | 0.01559 | 0.02615 | 0.26554 | 0.95873 | + +Relative to uniform INT4, the primary policy reduced task-macro excess NLL by +74.14%. The paired task-bootstrap mean improvement was 2.19798 excess-NLL +points with a 95% percentile interval of `[2.08078, 2.32146]`. + +At the identical 2,564,096-byte budget, the three prespecified random controls +promoted layers 18, 4, and 13. The primary policy's gain over their within-task +mean was 2.07838 excess-NLL points with a paired 95% interval of +`[1.96598, 2.19451]`. + +Calibration MSE independently selected layer 0, so the MSE baseline and frozen +primary policy are identical in this version. This agreement is reported; it is +not a separate successful comparison. + +The stochastic-rounding controls were substantially worse on this run: their +macro excess NLL values were 5.80807, 5.81640, and 5.82662 for seeds 2339, +2340, and 2341. This negative result is retained rather than omitted. + +## Validity checks + +- All seven preregistered continuation gates passed. +- All values were finite. +- Every candidate used the same 90 tasks and 5,524-token manifest. +- Every resident packed-byte total exactly matched its registered layout. +- Packed-cache and QDQ logits matched with maximum absolute difference `0.0` + across all candidates in the preflight. +- The prepared-manifest, calibration-evidence, dataset-manifest, token-manifest, + repository-commit, artifact, and canonical-evidence hashes were independently + rechecked after the run. +- Confirmation data had not been evaluated when this decision was recorded. + +Artifact integrity: + +- Final artifact SHA256: + `5980fd58aa0933ad97deb896d4901fcd37350c4a57d8a80022ab218aaf77e727` +- Canonical evidence SHA256: + `301c52e194bbd23059a0040a8e94aeac97dc33de1100f13edbf17dc877755488` +- Development dataset manifest SHA256: + `8fed3da0aae864f4e30c70ad70b0269f759d3592dccb9ab87f24444fa24d65dc` + +The per-token manifest digest remains embedded in the canonical artifact and +was included in the independent provenance check. + +## Decision and claim boundary + +Proceed once to the already frozen 500-task MBPP confirmation split. Do not +retune the promoted layer, bit allocation, group size, rounding rule, prompts, +baselines, seeds, or gates after seeing this development result. + +This is positive development evidence, not confirmation and not proof of a +breakthrough. It measures teacher-forced reference-code fidelity on one pinned +model. It does not establish generated-code correctness, generality across +models or datasets, total-model memory reduction, latency improvement, or +state-of-the-art performance. This quality run did not collect a controlled +latency result, so no speed claim is permitted. diff --git a/research/EXPERIMENT_002_SCALE_CORRECTION.md b/research/EXPERIMENT_002_SCALE_CORRECTION.md new file mode 100644 index 0000000..5d1daed --- /dev/null +++ b/research/EXPERIMENT_002_SCALE_CORRECTION.md @@ -0,0 +1,78 @@ +# Experiment 002: stored-scale correction and packed-cache parity + +Status: correction replay complete; public-dataset validation pending + +Date: 2026-07-22 + +## What was wrong + +RecurQuant v0.1 counted each group scale as 16 bits in its modeled storage but +performed the numerical quantize/dequantize calculation with an FP32 scale. +That made the byte estimate internally inconsistent with the simulated number +format. The discrepancy was found while implementing real integer residency. + +The implementation now casts each stored scale to the declared FP16 or FP32 +format before it is used, includes padded groups in the payload byte count, and +clamps FP16 scales to a finite representable range. INT4 and INT8 payloads can +also be stored physically rather than kept as dequantized floating-point state. + +The original v0.1 artifacts remain in history as an audit trail. Their headline +numbers are superseded by the correction runs below; they are not silently +rewritten. + +## Corrected diagnostic results + +All runs use Qwen3.5-0.8B-Base revision +`dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`, group size 128, deterministic +nearest rounding, and stored FP16 scales. These are still short diagnostic +traces rather than a public benchmark. + +| Trace | Uniform INT4 CVaR95 KL | Layer 0 INT8, rest INT4 | Relative reduction | +| --- | ---: | ---: | ---: | +| Retrieval | 7.2744 | 1.2270 | 83.1% | +| Code | 5.4844 | 2.0482 | 62.7% | +| Multilingual correction replay | 5.7644 | 1.4277 | 75.2% | + +The retrieval and code sensitivity sweeps still rank layer 0 as the best +single-layer INT8 promotion after the correction. The simple diagnostic +signals remain unsuitable as a general selector: their rank correlations vary +substantially by trace, including sign changes for forgetting activity. + +## Realized resident storage + +The packed cache stores INT4 codes two per byte, INT8 codes as signed bytes, and +the declared FP16 scale tensor. At batch one, the 18 Gated DeltaNet matrix +states occupy: + +| Policy | Resident bytes | Ratio versus FP32 states | +| --- | ---: | ---: | +| FP32 recurrent states | 18,874,368 | 1.000x | +| Uniform INT4 plus FP16 scales | 2,433,024 | 7.758x smaller | +| Layer 0 INT8, rest INT4, plus FP16 scales | 2,564,096 | 7.361x smaller | + +The packed multilingual replay produced exactly the same token-level metrics as +the corrected QDQ replay. This establishes implementation parity for that run, +not losslessness relative to the FP32-state model. The current Python path +materializes one floating-point recurrent state while each layer executes, so +it proves persistent resident-byte reduction but makes no speed or peak-memory +claim. + +## Reproducibility records + +- Corrected static retrieval evidence hash: + `61abb54b16989d697f995c04807401bd5920b4c03c4911d1340a4e4082e7dadf` +- Corrected retrieval sweep evidence hash: + `a077f18867c69e5581b4dd003e4e7ed6d03f9843720fbbe2e1522f837518f1de` +- Corrected code sweep evidence hash: + `cbac481cd90c99e30bf550a8c6026e680471480a99ff60600bbae57daa4b9776` +- Corrected multilingual QDQ replay evidence hash: + `38522abbd9b8fc593ee2b02d886f9a48a22d3c589d049d256fb8235800198931` +- Packed multilingual replay evidence hash: + `62ac233fd51dcbb722850593dfbbb3c1bec54e189c3f8e3812d1aeaa251b3523` + +## What this permits + +This correction permits a public-dataset development evaluation and a useful +experimental packed-cache release. It does not permit “breakthrough,” novelty, +quality-preserving, end-to-end memory, or speed claims. Those remain gated on +the frozen public protocol and, for systems claims, a fused runtime kernel. diff --git a/research/PRIOR_ART.md b/research/PRIOR_ART.md index a382f18..cd8da01 100644 --- a/research/PRIOR_ART.md +++ b/research/PRIOR_ART.md @@ -7,15 +7,25 @@ the search before any paper or novelty statement. ## Persistent recurrent-state quantization -- [Quamba2](https://arxiv.org/abs/2503.22879) stores calibrated grouped Mamba - cached states at INT8. RecurQuant cannot claim the first recurrent-state or - first INT8 SSM cache quantizer. +- [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) evaluates + `W8A8H4`, including 4-bit persistent Mamba2 state caches with decoupled + state/channel scaling and selectivity reconstruction. RecurQuant cannot + claim the first 4-bit or sub-8-bit persistent SSM-state quantizer. +- [Quamba2](https://arxiv.org/abs/2503.22879) and its + [official implementation](https://github.com/enyac-group/Quamba) load and + store calibrated Mamba cached states at INT8 in sequence kernels. Quamba2 + also searches mixed W4A8/A16 blocks, although that search is not the same as + RecurQuant's per-recurrent-layer cache layout. RecurQuant cannot claim the + first INT8 SSM cache, broad sensitivity-guided SSM quantization, or a first + packed recurrent kernel. - [Nemotron 3 Super](https://arxiv.org/abs/2604.12374) studies recurrent rounding - accumulation and stochastic rounding. + accumulation and uses FP16 stochastic rounding in its released deployment + path. - [Nemotron 3 Ultra](https://arxiv.org/abs/2606.15007) evaluates FP16, INT8, and FP8 recurrent states with block scaling, stochastic rounding, checkpoints, - and activation replay. These are mandatory stability baselines, not new ideas - available for RecurQuant to claim. + and cached-input replay. Its reported INT8/FP8 checkpoint experiments were + emulated while optimized 8-bit kernels were still under development. These + methods remain prior art and mandatory stability baselines for RecurQuant. ## Existing Gated DeltaNet systems work @@ -34,6 +44,22 @@ the search before any paper or novelty statement. - [AVMP](https://arxiv.org/abs/2605.22416) dynamically rebalances KV and SSM memory pools. +## Mixed-precision cache allocation + +- [KVTuner](https://arxiv.org/abs/2502.04420), + [KVmix](https://arxiv.org/abs/2506.08018), and + [Quantize What Counts](https://arxiv.org/abs/2502.15075) already use + layer-wise sensitivity or loss signals to allocate cache precision under + hardware or memory constraints. +- [RateQuant](https://arxiv.org/abs/2605.06675), + [PM-KVQ](https://arxiv.org/abs/2505.18610), and + [SpectrumKV](https://arxiv.org/abs/2606.08635) cover fixed-average-bit, + progressive, or per-token mixed-precision KV policies. + +These works target transformer KV caches rather than Gated DeltaNet recurrent +matrices, but they preclude broad claims that sensitivity-guided, per-layer, +dynamic, or equal-budget cache precision allocation is new. + ## Architecture evidence - [Gated DeltaNet-2](https://arxiv.org/abs/2605.22791) separates channel-wise @@ -45,7 +71,7 @@ the search before any paper or novelty statement. to use sparse reads and writes over a larger state. RecurQuant instead studies post-training storage of an existing model's state. -## Narrow lane under test +## Audited gap, not a novelty claim The scan did not locate a published implementation combining all three: @@ -53,7 +79,16 @@ The scan did not locate a published implementation combining all three: 2. Gated DeltaNet-specific read/sensitivity-based precision allocation; and 3. a packed end-to-end runtime kernel with measured quality and latency. -That negative search is not a novelty claim. A defensible result would still -need to beat uniform per-head/per-block quantization, Quamba2-style calibration, -Nemotron stochastic/checkpoint baselines, ReplaySSM/KVBuffer, magnitude and -update-residual scoring, and ordinary KV-cache quantization. +That negative search is not proof of firstness, and the current RecurQuant +alpha does not satisfy item 3: it physically stores the cache in packed form +but materializes one recurrent state for each layer call and has no fused +recurrence kernel or latency result. + +The frozen v0.2 protocol compares uniform INT4, one uniform INT8 reference, +three same-byte random layer placements, MSE-selected placement, nearest and +stochastic rounding, and the prespecified layer-0 policy. It does **not** +compare against Q-Mamba's DSQ method, Nemotron checkpoint/replay, ReplaySSM, +KVBuffer, per-head/per-block allocation, or fused-kernel baselines. A later +claim of novelty or general superiority would require those comparisons, +multiple models and tasks, a repeated prior-art search, and measured end-to-end +latency. diff --git a/research/PUBLIC_EVAL_PROTOCOL_V02.md b/research/PUBLIC_EVAL_PROTOCOL_V02.md new file mode 100644 index 0000000..6532569 --- /dev/null +++ b/research/PUBLIC_EVAL_PROTOCOL_V02.md @@ -0,0 +1,387 @@ +# Public evaluation protocol v0.2 + +Status: frozen protocol; evaluation completed without changing the protocol or +gates below + +Date frozen: 2026-07-22 + +> **Post-freeze outcome (2026-07-22):** The frozen quality hypothesis passed on +> all 500 untouched confirmation tasks. See +> [`CONFIRMATION_002.md`](CONFIRMATION_002.md) for the exact effects, integrity +> anchors, and claim boundary. This annotation records the outcome; the protocol +> and gates below remain unchanged. + +## Purpose + +This experiment tests one narrow question: on a pinned Qwen3.5 checkpoint and +public MBPP reference programs, does the already frozen RecurQuant policy +preserve teacher-forced code-token predictions better than prespecified +baselines while physically storing the persistent Gated DeltaNet state in the +declared number of bytes? + +This is not a code-generation benchmark. It does not sample programs, execute +untrusted code, or measure `pass@k`. It measures the effect of recurrent-state +quantization while every method receives the same gold prefix. + +## Why v0.2 is a correction, not a victory lap + +The v0.1 result used three short synthetic traces. It demonstrated that the +adapter worked and that a frozen layer-0 mixed-precision policy passed its small +diagnostic confirmation. It did not establish public-task generalization, +novelty, total inference-memory savings, or speed. + +The prior-art boundary is also stricter than the initial repository wording: +[Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) already reports +4-bit Mamba-2 state-cache quantization, and +[Quamba2](https://arxiv.org/abs/2503.22879) and +[Nemotron 3 Super](https://docs.nvidia.com/nemotron/0.1.0/nemotron/super3/quantization.html) +provide additional recurrent-state quantization and rounding precedents. +RecurQuant must not claim to be the first recurrent-state or first 4-bit +recurrent-cache method. Stochastic rounding is therefore a required baseline, +not an optional favorable comparison. + +The packed cache in v0.2 can establish resident tensor bytes. Because it still +dequantizes a state while a layer executes and has no fused quantized recurrent +kernel, resident-state bytes must be reported separately from transient memory, +whole-model peak memory, and latency. + +## Frozen assets + +### Model + +- Model and tokenizer: `Qwen/Qwen3.5-0.8B-Base`. +- Revision: `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`. +- Revision permalink: + . +- Expected architecture: 24 language layers, including exactly 18 Gated + DeltaNet layers at model-layer indices + `0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22`. +- Expected recurrent-state shape at batch one: `[1, 16, 128, 128]` per Gated + DeltaNet layer. +- Model weights use BF16 on CUDA. The unquantized persistent-state reference is + the model's FP32 recurrent state. +- `trust_remote_code=False`, eager attention, evaluation mode, batch size one, + and no sampling. + +Any model revision, layer-count, state-shape, or state-dtype mismatch invalidates +the run. It must not be silently adapted after seeing evaluation results. + +The reference software environment is Python 3.11.15, PyTorch 2.11.0+cu128, +Transformers 5.14.1, and Datasets 4.8.5. Every artifact records exact package, +CUDA, driver, device, repository-commit, and command information. A result from +another environment is a replication, not a byte-for-byte repeat. + +### Dataset + +- Dataset: `google-research-datasets/mbpp`. +- Configuration: `full`. +- Revision: `4bb6404fdc6cacfda99d4ac4205087b89d32030c`. +- Revision permalink: + . +- Original split specification: + . + +The official ID ranges are kept intact: + +| Role | Source split | Task IDs | Count | +|---|---|---:|---:| +| Excluded few-shot examples | `prompt` | 1-10 | 10 | +| Untouched confirmation | `test` | 11-510 | 500 | +| Development | `validation` | 511-600 | 90 | +| Calibration pool | `train` | 601-974 | 374 | + +Calibration uses exactly 128 tasks. For each training ID, compute lowercase +hexadecimal SHA256 over the UTF-8 bytes of `rq-v0.2|{task_id}` with no newline, +sort by `(digest, task_id)`, and take the first 128. In ascending task-ID order, +the frozen set is: + +```text +602, 605, 606, 607, 614, 615, 616, 622, 627, 629, 630, 635, 636, 640, 641, +644, 646, 647, 648, 651, 653, 656, 657, 662, 666, 669, 671, 672, 674, 677, +679, 686, 687, 695, 698, 702, 704, 705, 707, 708, 709, 710, 712, 713, 720, +722, 725, 727, 728, 732, 741, 744, 751, 755, 756, 763, 764, 768, 771, 772, +783, 785, 789, 792, 793, 794, 795, 797, 800, 803, 804, 809, 819, 820, 821, +822, 823, 824, 827, 835, 839, 840, 846, 847, 848, 849, 851, 853, 854, 857, +858, 860, 862, 867, 868, 869, 870, 872, 874, 877, 878, 882, 884, 885, 886, +894, 895, 902, 903, 907, 908, 911, 918, 919, 920, 924, 929, 930, 936, 937, +944, 945, 946, 950, 955, 960, 962, 973 +``` + +The remaining 246 training tasks are unused. The prompt split is not used for +few-shot examples. Development uses all 90 validation tasks, and confirmation +uses all 500 test tasks. There is no random task subsampling. + +Before any development run, the evaluator must write and verify a manifest +containing each task ID, normalized input-field digest, prompt-token digest, +target-token digest, and token counts. No task may be removed because its result +is inconvenient. If any example exceeds the pinned model context after frozen +serialization, the run stops and the protocol is revised before inspecting +method comparisons. + +## Frozen input construction + +Normalize `\r\n` and bare `\r` to `\n` in `text`, each `test_list` entry, and +`code`; make no other content edits. Let `tests` be the three normalized test +strings joined by one newline. The zero-shot prompt is exactly: + +```text +You are an expert Python programmer, and here is your task: {text} +Your code should pass these tests: + +{tests} +[BEGIN] +``` + +Encode the prompt with `add_special_tokens=True`. Encode the normalized +reference `code` separately with `add_special_tokens=False`, then concatenate +the token-ID lists. Separate encoding intentionally makes the target boundary +unambiguous. Do not append `[DONE]` or EOS to the scored target. + +Prefill the complete prompt. For code tokens `y_1, ..., y_T`, score `y_1` from +the final prefill logits, then feed each gold token to score the next token. +Candidate-generated tokens are never fed back. Quantize and pack the persistent +state once after prefill and after every single-token teacher-forced update. +Weights, convolution state, ordinary attention KV caches, token IDs, and model +arithmetic are otherwise shared with the reference. + +## Frozen quantizer and byte accounting + +All quantized methods use symmetric signed per-group absmax scaling, group size +128 over the flattened final two state dimensions, FP16 scales, no zero point, +and either the stated nearest or stochastic rounding. For `b` bits, +`qmax = 2^(b-1) - 1`; the absmax scale is stored in FP16 and clamped to the FP16 +range with minimum positive value `2^-24`. Nearest rounding is PyTorch's +round-to-nearest-even behavior. INT4 stores two signed nibbles per byte; INT8 +stores signed bytes. Padding, scales, and any required metadata count. Static +layer formats require no per-token precision tag. + +For one `[1, 16, 128, 128]` state there are 262,144 elements and 2,048 groups: + +| Format | Payload/layer | Scales/layer | Total/layer | +|---|---:|---:|---:| +| FP32 reference | 1,048,576 B | 0 B | 1,048,576 B | +| INT4 | 131,072 B | 4,096 B | 135,168 B | +| INT8 | 262,144 B | 4,096 B | 266,240 B | + +The frozen whole-cache budgets are therefore: + +| Policy layout | Resident recurrent-state bytes | Role | +|---|---:|---| +| 18 FP32 layers | 18,874,368 | Reference storage anchor | +| 18 INT4 layers | 2,433,024 | Lower-byte quality anchor | +| One INT8 layer plus 17 INT4 layers | 2,564,096 | Exact-byte comparison budget | +| 18 INT8 layers | 4,792,320 | Higher-byte quality anchor | + +The primary mixed plan is a modeled 7.36x reduction in persistent-state bytes +relative to its FP32-state reference. The public artifact may call this a real +resident recurrent-state reduction only if the sum of the live packed payload +and scale tensor storages is exactly 2,564,096 bytes. It must also report the +largest transient materialized state and CUDA allocator peaks separately. + +## Frozen methods + +All exact-byte mixed methods store exactly one Gated DeltaNet layer at INT8 and +the other 17 at INT4, for 2,564,096 resident bytes. + +1. `fp32_state`: unquantized persistent-state reference. +2. `uniform_int4_nearest`: lower-byte anchor. +3. `uniform_int8_nearest`: higher-byte anchor. +4. `read_risk_l0_nearest`: primary frozen candidate; model layer 0 is INT8. +5. `mse_selected_nearest`: baseline selected once on the 128 calibration tasks. + At every post-prefill and teacher-forced state write from the FP32-state run, + compute `||Q4(S)-S||_F^2 / max(||S||_F^2, 1e-24)` for each layer, average + equally over writes and then tasks, and select the maximum; exact ties choose + the lower layer ID. +6. `random_l18_nearest`, `random_l4_nearest`, and `random_l13_nearest`: + prespecified random-layer controls. For seeds 1101, 2202, and 3303, compute + `int(SHA256("rq-v0.2-random|{seed}"), 16) mod 18` over the UTF-8 string with + no newline and use that position in the ordered Gated DeltaNet layer list + above. The frozen results are model layers 18, 4, and 13 respectively. +7. `read_risk_l0_stochastic`: the same bit layout as the primary candidate, + evaluated at stochastic-rounding base seeds 2339, 2340, and 2341. The seed + schedule adds `layer_id * 1,000,000 + per_layer_cache_update_index`. + +The primary layer remains 0 regardless of what MBPP calibration shows. Public +calibration may measure its read-risk rank as a diagnostic, but cannot replace +it. The MSE selector is the only baseline whose layer is chosen from the public +calibration partition. No confirmation-set layer sweep or oracle selection is +permitted. + +QDQ versions of the same policies are implementation controls, not memory +baselines: they retain dequantized FP32 tensors. Packed and QDQ logits and +metrics must agree within the frozen numerical tolerance before a packed result +is accepted. + +## Metrics + +For task `i`, target code token `t`, method `m`, gold token `y_it`, and logits +`z_mit`, define + +```text +nll_mit = -log_softmax(z_mit)[y_it] +excess_nll_mit = nll_mit - nll_fp32,it +``` + +First average over code tokens within each task, then average equally over +tasks. This macro task mean is primary; a long reference program must not count +as many independent tasks. The primary fidelity quantity is + +```text +DeltaNLL_m = mean_i(mean_t(excess_nll_mit)). +``` + +Smaller is better and zero means no mean NLL change from the FP32-state run. +The prespecified 15% effect is evaluated against uniform INT4 when its +`DeltaNLL` is positive: + +```text +relative_reduction = + (DeltaNLL_uniform_int4 - DeltaNLL_read_risk_l0) + / DeltaNLL_uniform_int4. +``` + +If uniform INT4 has non-positive `DeltaNLL`, this ratio is undefined and the +15% continuation gate fails rather than being redefined after inspection. + +The equal-byte primary contrast averages the three random controls within each +task, then computes + +```text +equal_byte_gain_i = + mean_random_DeltaNLL_i - read_risk_l0_DeltaNLL_i. +``` + +Required secondary metrics are: + +- token-weighted and task-macro candidate NLL and excess NLL; +- mean and worst-5% token `KL(fp32_state || candidate)`; +- top-1 next-token agreement with `fp32_state`; +- task-level median and quartiles of excess NLL; +- the same metrics by frozen reference-code-token-count quartile; +- resident payload, scale, metadata, padding, and transient bytes; +- prefill latency, teacher-forced decode latency per token, and peak CUDA + allocated and reserved bytes. + +Latency uses CUDA events with explicit synchronization, fixed GPU and power +settings recorded, one untimed warm-up pass, and five timed repeats over the +calibration set. Report the median repeat and dispersion. No sample may be +dropped as an outlier. + +## Statistical procedure + +The task is the resampling unit because tokens inside one program are not +independent. Use paired task-level percentile bootstrap intervals with 10,000 +resamples and RNG seed 2339. All candidates use identical tasks and target +tokens. Report point estimates and two-sided 95% intervals, including failures +and negative effects. + +There is one confirmatory statistical contrast: `read_risk_l0_nearest` versus +the within-task mean of the three exact-byte random controls on macro excess +NLL. It needs no multiplicity correction. MSE, stochastic-rounding, KL, top-1, +length-quartile, latency, and individual-random-layer comparisons are +prespecified secondary analyses. They receive effect sizes and intervals but no +standalone "significant" wording. If later reporting applies null-hypothesis +tests to that secondary family, Holm correction across the complete family is +required; favorable tests may not be selected after inspection. + +## Development and confirmation sequence + +### Stage 0: validity and calibration + +1. Verify revisions, split IDs, architecture, state shapes, and token manifest. +2. Confirm that packed INT4 and INT8 round trips reproduce their QDQ values and + that packed and QDQ model metrics agree to absolute tolerance `1e-6`. +3. Confirm the exact resident bytes in the table above. +4. Select and record the MSE baseline layer using only the 128 calibration + tasks. Record the diagnostic public-data rank of layer 0 without changing + the primary policy. +5. Freeze the evaluator commit, environment, command matrix, selected MSE + layer, and artifact schema before development. + +Any failed integration check stops the study. Fixing implementation defects is +allowed, but all affected calibration artifacts must be regenerated and the +fix disclosed before development. + +### Stage 1: development + +Run every frozen method on all 90 development tasks. Proceed to confirmation +only if all of the following hold: + +- all values are finite and every method scores the identical token manifest; +- packed resident bytes exactly match the registered layouts; +- `read_risk_l0_nearest` reduces macro excess NLL by at least 15% relative to + uniform INT4; +- the paired-bootstrap 95% interval for its equal-byte gain over the mean + random control is entirely above zero; +- its mean token KL and worst-5% token KL are both lower than uniform INT4; and +- its top-1 agreement is not lower than uniform INT4. + +These are continuation gates, not publishable confirmation. If a gate fails, +report the development failure and do not inspect the confirmation outcomes. +Changing a policy after development creates a new version and requires a new +untouched dataset. + +Before confirmation, commit the frozen method configuration and a development +decision record. Merely changing formatting or correcting an evaluator bug does +not authorize retuning. + +### Stage 2: untouched confirmation + +If every development gate passes, run the already frozen method matrix once on +all 500 confirmation tasks. Do not run exploratory layer interventions, prompt +variants, threshold sweeps, or partial previews on these tasks first. + +The v0.2 quality hypothesis passes only if: + +- the point estimate gives at least 15% lower macro excess NLL than uniform + INT4 and the paired-bootstrap interval favors `read_risk_l0_nearest`; +- the 95% interval for equal-byte gain over the mean random control is entirely + above zero; +- mean and worst-5% token KL are lower than uniform INT4; +- top-1 agreement is not lower than uniform INT4; and +- every validity and byte-accounting condition still passes. + +The performance claim gate is separate: packed median teacher-forced decode +latency must be no more than 1.10 times the FP32-state reference under the same +loop. Failing this gate does not erase a valid fidelity or resident-byte result, +but it prohibits a speed, efficient-runtime, or deployment-ready claim. Until a +fused kernel exists, latency regression is an expected possible result and must +be reported plainly. + +A run may be replaced only for a documented infrastructure or evaluator failure +defined without reference to which method looked better. The invalid artifact, +reason, code change, and complete rerun must remain disclosed. A statistically +unfavorable result is not a failed run and cannot be rerun away. + +## Claim boundary after confirmation + +If the gates pass, defensible wording is limited to the evaluated scope, for +example: + +> On `Qwen3.5-0.8B-Base` at the pinned revision, the frozen layer-0 INT8 plus +> 17-layer INT4 policy used 2,564,096 bytes of resident recurrent-state storage +> and preserved MBPP reference-code token likelihood better than the +> prespecified baselines under this teacher-forced protocol. + +The exact measured effect and interval must replace any vague adjective. + +This protocol cannot support claims that RecurQuant: + +- is a new base model or that its authors created Qwen3.5; +- is the first recurrent-state, Mamba-state, or 4-bit cache quantizer; +- is a breakthrough, state of the art, lossless, or generally superior; +- improves generated-code correctness, `pass@k`, or executable behavior; +- reduces total model memory by 7.36x; +- speeds up inference unless the separate latency gate passes; +- generalizes beyond this checkpoint, architecture, language, or dataset; or +- is free of benchmark contamination. + +MBPP is old and widely used. Whether its tasks or solutions appeared in the +Qwen pretraining corpus is unknown. Teacher-forced paired comparisons isolate +the effect of state quantization better than an absolute coding score, but they +do not remove possible pretraining contamination. + +Failure is a valid result. If the frozen candidate misses a gate, publish the +negative evidence and treat any revised selector, including a future +state-Jacobian or empirical-Fisher method, as a separately preregistered +hypothesis rather than rewriting v0.2 after the fact. diff --git a/research/STATUS.md b/research/STATUS.md index d9acd92..957a2a6 100644 --- a/research/STATUS.md +++ b/research/STATUS.md @@ -1,5 +1,18 @@ # Research status +The frozen v0.2 public-data study completed on 2026-07-22. Every preregistered +quality gate passed on all 500 untouched MBPP test tasks and 30,244 scored +tokens. The exact result, integrity anchors, infrastructure-resume record, and +claim boundary are in +[`CONFIRMATION_002.md`](CONFIRMATION_002.md). The frozen design remains in +[`PUBLIC_EVAL_PROTOCOL_V02.md`](PUBLIC_EVAL_PROTOCOL_V02.md), and the earlier +development decision is in [`DEVELOPMENT_002.md`](DEVELOPMENT_002.md). + +## v0.1 pilot archive + +The remainder of this file preserves the diagnostic v0.1 snapshot. Do not read +its historical "next action" as the current project state. + Last updated: 2026-07-22 ## Confirmed implementation @@ -10,7 +23,7 @@ Last updated: 2026-07-22 `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`. - Captured 18 recurrent states at `[1, 16, 128, 128]`, FP32, totalling 18 MiB. - QDQ occurs once after prefill and after every teacher-forced decode token. -- All 26 unit/integration tests and lint are green. +- At this snapshot, all 26 unit/integration tests and lint were green. ## Calibration and development evidence @@ -70,7 +83,7 @@ The deterministic repeat matched canonical evidence hash `1961ecc395d138cf505a20e55c3465260dcfded8b1ac3a9c95c40af775d2d722`. Gate C passed. See [Confirmation 001](CONFIRMATION_001.md). -## Next action +## Historical next action Move from synthetic diagnostics to a preregistered public-data evaluation with longer horizons, multiple seeds, sequence NLL, equal-byte baselines, and no diff --git a/scripts/calibrate_mbpp_layers.py b/scripts/calibrate_mbpp_layers.py new file mode 100644 index 0000000..a5c93e3 --- /dev/null +++ b/scripts/calibrate_mbpp_layers.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Freeze public-MBPP layer selectors for the RecurQuant v0.2 evaluation.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import subprocess +import sys +from collections import defaultdict +from datetime import UTC, datetime +from pathlib import Path +from statistics import fmean +from typing import Any + +import torch +from transformers import AutoTokenizer, DynamicCache, Qwen3_5ForCausalLM + +from recurquant.cache import iter_recurrent_states +from recurquant.public_data import ( + MBPP_CALIBRATION_SIZE, + format_mbpp_example, + load_mbpp_rows, + mbpp_manifest, + mbpp_manifest_sha256, +) +from recurquant.quantization import QuantizationSpec, quantize_dequantize +from recurquant.signals import GatedDeltaSignal, GatedDeltaSignalRecorder + +MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" +MODEL_REVISION = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +SEED = 2339 +RANDOM_BASELINE_SEEDS = (1101, 2202, 3303) + + +def canonical_json_bytes(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def git_commit() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def tracked_worktree_is_clean() -> bool: + result = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + check=True, + capture_output=True, + text=True, + ) + return not result.stdout.strip() + + +def nvidia_driver_versions(device: torch.device) -> list[str]: + if device.type != "cuda": + return [] + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Calibrate frozen one-layer INT8 selectors on pinned MBPP train rows." + ) + parser.add_argument( + "--output", + type=Path, + default=Path("artifacts/mbpp-v02-calibration.json"), + ) + parser.add_argument( + "--limit", + type=int, + help="Diagnostic-only prefix of the frozen 128-row calibration population.", + ) + parser.add_argument("--group-size", type=int, default=128) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--model-id", default=MODEL_ID) + parser.add_argument("--revision", default=MODEL_REVISION) + parser.add_argument( + "--random-seeds", + type=int, + nargs="+", + default=list(RANDOM_BASELINE_SEEDS), + ) + return parser.parse_args() + + +def select_device(requested: str) -> torch.device: + if requested == "auto": + requested = "cuda" if torch.cuda.is_available() else "cpu" + if requested == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested but torch.cuda.is_available() is false") + return torch.device(requested) + + +def _mean_by_layer( + values: dict[int, list[float]], + layer_indices: list[int], +) -> dict[str, float]: + missing = [layer_index for layer_index in layer_indices if not values[layer_index]] + if missing: + raise RuntimeError(f"No calibration observations for GDN layers: {missing}") + return { + str(layer_index): fmean(values[layer_index]) + for layer_index in layer_indices + } + + +def _argmax_layer(means: dict[str, float]) -> int: + return min( + (int(layer_index) for layer_index in means), + key=lambda layer_index: (-means[str(layer_index)], layer_index), + ) + + +def _random_baseline_layer(layer_indices: list[int], seed: int) -> int: + digest = hashlib.sha256(f"rq-v0.2-random|{seed}".encode()).digest() + return layer_indices[int.from_bytes(digest, "big") % len(layer_indices)] + + +def _consume_signals( + records: list[GatedDeltaSignal], + *, + read_risk: dict[int, list[float]], + state_error: dict[int, list[float]], +) -> None: + if not records: + raise RuntimeError("Signal recorder captured no Gated DeltaNet calls") + for record in records: + if record.probe_read_relative_l2 is None or record.probe_state_relative_l2 is None: + raise RuntimeError( + f"Layer {record.layer_index} did not expose a decode-time quantization probe" + ) + read_risk[record.layer_index].append(record.probe_read_relative_l2) + state_error[record.layer_index].append(record.probe_state_relative_l2**2) + + +def main() -> int: + args = parse_args() + if args.limit is not None and args.limit < 1: + raise ValueError("--limit must be at least 1") + if len(set(args.random_seeds)) != len(args.random_seeds): + raise ValueError("--random-seeds must be unique") + + torch.manual_seed(SEED) + device = select_device(args.device) + dtype = torch.bfloat16 if device.type == "cuda" else torch.float32 + rows = load_mbpp_rows("calibration", limit=args.limit) + dataset_manifest = mbpp_manifest(rows, phase="calibration") + + tokenizer = AutoTokenizer.from_pretrained( + args.model_id, + revision=args.revision, + local_files_only=args.local_files_only, + trust_remote_code=False, + ) + model = Qwen3_5ForCausalLM.from_pretrained( + args.model_id, + revision=args.revision, + dtype=dtype, + attn_implementation="eager", + low_cpu_mem_usage=True, + use_safetensors=True, + local_files_only=args.local_files_only, + ).to(device) + model.eval() + + layer_indices = [ + index + for index, layer_type in enumerate(model.config.layer_types) + if layer_type == "linear_attention" + ] + if len(layer_indices) != 18: + raise RuntimeError(f"Expected 18 GDN layers, found {len(layer_indices)}") + + probe_spec = QuantizationSpec( + bits=4, + group_size=args.group_size, + scale_bits=16, + rounding="nearest", + seed=SEED, + ) + task_mean_read_risk: dict[int, list[float]] = defaultdict(list) + task_mean_state_error: dict[int, list[float]] = defaultdict(list) + task_token_counts: list[dict[str, int]] = [] + scored_tokens = 0 + + recorder = GatedDeltaSignalRecorder(model, probe_spec=probe_spec) + with torch.inference_mode(), recorder: + for row_index, row in enumerate(rows, start=1): + task_read_risk: dict[int, list[float]] = defaultdict(list) + task_state_error: dict[int, list[float]] = defaultdict(list) + formatted = format_mbpp_example(row) + prompt_ids = tokenizer( + formatted.prompt, + add_special_tokens=True, + return_tensors="pt", + )["input_ids"].to(device) + code_ids = tokenizer( + formatted.code, + add_special_tokens=False, + return_tensors="pt", + )["input_ids"].to(device) + if code_ids.shape[1] < 2: + raise RuntimeError(f"MBPP task {row['task_id']} has fewer than two code tokens") + + cache = DynamicCache(config=model.config) + recorder.enabled = False + model( + prompt_ids, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ) + for state in iter_recurrent_states(cache): + qdq = quantize_dequantize(state.tensor, probe_spec) + task_state_error[state.layer_index].append(qdq.relative_l2_error**2) + for token_index in range(code_ids.shape[1] - 1): + recorder.records.clear() + recorder.enabled = True + model( + code_ids[:, token_index : token_index + 1], + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ) + recorder.enabled = False + _consume_signals( + recorder.records, + read_risk=task_read_risk, + state_error=task_state_error, + ) + + for layer_index in layer_indices: + if not task_read_risk[layer_index] or not task_state_error[layer_index]: + raise RuntimeError( + f"Task {row['task_id']} has no observations for GDN layer {layer_index}" + ) + task_mean_read_risk[layer_index].append(fmean(task_read_risk[layer_index])) + task_mean_state_error[layer_index].append(fmean(task_state_error[layer_index])) + + eligible = int(code_ids.shape[1] - 1) + scored_tokens += eligible + task_token_counts.append( + { + "task_id": row["task_id"], + "prompt_tokens": int(prompt_ids.shape[1]), + "code_tokens": int(code_ids.shape[1]), + "scored_state_reads": eligible, + } + ) + print( + f"[{row_index}/{len(rows)}] task={row['task_id']} " + f"code_tokens={code_ids.shape[1]}", + flush=True, + ) + + read_risk_means = _mean_by_layer(task_mean_read_risk, layer_indices) + state_error_means = _mean_by_layer(task_mean_state_error, layer_indices) + candidate_layers: dict[str, int] = { + "read_risk_l0_nearest": 0, + "mse_selected_nearest": _argmax_layer(state_error_means), + } + for random_seed in args.random_seeds: + random_layer = _random_baseline_layer(layer_indices, random_seed) + candidate_layers[f"random_l{random_layer}_nearest"] = random_layer + + evidence: dict[str, Any] = { + "claim_scope": { + "protocol_eligible": len(rows) == MBPP_CALIBRATION_SIZE and args.limit is None, + "selector_only": True, + "development_touched": False, + "confirmation_touched": False, + "research_result": False, + }, + "source": { + "model_id": args.model_id, + "model_revision": args.revision, + "tokenizer_revision": args.revision, + "dataset_manifest": dataset_manifest, + "dataset_manifest_sha256": mbpp_manifest_sha256(rows, phase="calibration"), + "repository_commit": git_commit(), + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + "torch": torch.__version__, + "transformers": importlib.metadata.version("transformers"), + "datasets": importlib.metadata.version("datasets"), + "cuda_runtime": torch.version.cuda, + "nvidia_driver_versions": nvidia_driver_versions(device), + "device": str(device), + "device_name": ( + torch.cuda.get_device_name(device) if device.type == "cuda" else "CPU" + ), + "model_dtype": str(next(model.parameters()).dtype), + "command": [Path(sys.executable).name, *sys.argv], + "tracked_worktree_clean": tracked_worktree_is_clean(), + }, + "calibration": { + "seed": SEED, + "group_size": args.group_size, + "quantization": { + "low_bits": 4, + "high_bits": 8, + "scale_bits": 16, + "rounding": "nearest", + }, + "row_count": len(rows), + "scored_state_reads": scored_tokens, + "task_token_counts": task_token_counts, + "diagnostic_read_risk_mean_by_layer": read_risk_means, + "diagnostic_read_risk_selected_layer": _argmax_layer(read_risk_means), + "normalized_state_mse_mean_by_layer": state_error_means, + "averaging": "equal writes within task, then equal tasks", + "random_baseline_seeds": list(args.random_seeds), + "candidate_layers": candidate_layers, + }, + } + evidence_hash = sha256_bytes(canonical_json_bytes(evidence)) + artifact = { + "schema_version": 1, + "artifact_kind": "recurquant_mbpp_layer_calibration", + "created_at_utc": datetime.now(UTC).isoformat(), + "canonical_evidence_sha256": evidence_hash, + "evidence": evidence, + } + payload = canonical_json_bytes(artifact) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(payload) + print( + json.dumps( + { + "output": str(args.output.resolve()), + "artifact_sha256": sha256_bytes(payload), + "canonical_evidence_sha256": evidence_hash, + "protocol_eligible": evidence["claim_scope"]["protocol_eligible"], + "candidate_layers": candidate_layers, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_mbpp.py b/scripts/evaluate_mbpp.py new file mode 100644 index 0000000..8031a68 --- /dev/null +++ b/scripts/evaluate_mbpp.py @@ -0,0 +1,1304 @@ +#!/usr/bin/env python3 +"""Run the frozen RecurQuant v0.2 teacher-forced MBPP evaluation.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import math +import platform +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from statistics import fmean +from typing import Any + +import numpy as np +import torch +from transformers import AutoTokenizer, DynamicCache, Qwen3_5ForCausalLM + +from recurquant.cache import iter_recurrent_states +from recurquant.evaluation import ( + TokenFidelity, + fidelity_summary, + paired_bootstrap_mean_improvement, +) +from recurquant.packed_cache import PackedRecurrentStateCache +from recurquant.public_data import ( + MBPP_CALIBRATION_SIZE, + MBPP_CONFIRMATION_LOCK, + MBPP_REVISION, + MBPPPhase, + format_mbpp_example, + load_mbpp_rows, + mbpp_manifest, + mbpp_manifest_sha256, +) +from recurquant.quantization import QuantizationSpec +from recurquant.transformers_cache import RecurrentStateQDQCache + +MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" +MODEL_REVISION = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +SEED = 2339 +GDN_LAYER_INDICES = (0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22) +EXPECTED_BYTES = { + "fp32_state": 18_874_368, + "uniform_int4": 2_433_024, + "mixed_int4_int8": 2_564_096, + "uniform_int8": 4_792_320, +} + + +@dataclass(frozen=True, slots=True) +class CandidateDefinition: + name: str + default_bits: int + upgrade_layer: int | None + rounding: str = "nearest" + seed: int = SEED + + @property + def layout(self) -> str: + if self.default_bits == 8: + return "uniform_int8" + if self.upgrade_layer is None: + return "uniform_int4" + return "mixed_int4_int8" + + +@dataclass(frozen=True, slots=True) +class EncodedTask: + task_id: int + prompt_ids: tuple[int, ...] + code_ids: tuple[int, ...] + prompt_sha256: str + code_sha256: str + + @property + def total_tokens(self) -> int: + return len(self.prompt_ids) + len(self.code_ids) + + def manifest_dict(self) -> dict[str, Any]: + return { + "task_id": self.task_id, + "prompt_tokens": len(self.prompt_ids), + "code_tokens": len(self.code_ids), + "total_tokens": self.total_tokens, + "prompt_token_ids_sha256": self.prompt_sha256, + "code_token_ids_sha256": self.code_sha256, + } + + +def canonical_json_bytes(value: Any) -> bytes: + return ( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n" + ).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def token_ids_sha256(token_ids: tuple[int, ...]) -> str: + return sha256_bytes( + json.dumps(token_ids, separators=(",", ":"), allow_nan=False).encode("utf-8") + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate packed recurrent-state fidelity on pinned MBPP data." + ) + parser.add_argument( + "--phase", + choices=tuple(phase.value for phase in MBPPPhase), + default=MBPPPhase.CALIBRATION.value, + ) + parser.add_argument("--calibration-artifact", type=Path, required=True) + parser.add_argument( + "--output", + type=Path, + default=Path("artifacts/mbpp-v02-evaluation.json"), + ) + parser.add_argument( + "--limit", + type=int, + help="Diagnostic-only prefix; forbidden for confirmation by the data loader.", + ) + parser.add_argument("--confirmation-lock") + parser.add_argument("--group-size", type=int, default=128) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--model-id", default=MODEL_ID) + parser.add_argument("--revision", default=MODEL_REVISION) + parser.add_argument("--bootstrap-samples", type=int, default=10_000) + parser.add_argument("--skip-qdq-preflight", action="store_true") + parser.add_argument( + "--manifest-only", + action="store_true", + help="Write the frozen row/token manifest without running model comparisons.", + ) + parser.add_argument( + "--prepared-manifest", + type=Path, + help="Previously committed manifest-only artifact required for full public phases.", + ) + parser.add_argument( + "--checkpoint", + type=Path, + help="Atomic per-task checkpoint path; defaults beside --output.", + ) + parser.add_argument( + "--allow-diagnostic-calibration", + action="store_true", + help="Allow a non-protocol calibration artifact for calibration-split smoke runs only.", + ) + return parser.parse_args() + + +def select_device(requested: str) -> torch.device: + if requested == "auto": + requested = "cuda" if torch.cuda.is_available() else "cpu" + if requested == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested but torch.cuda.is_available() is false") + return torch.device(requested) + + +def git_commit() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def nvidia_driver_versions(device: torch.device) -> list[str]: + if device.type != "cuda": + return [] + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def load_checkpoint(path: Path, *, signature: str) -> dict[str, Any] | None: + if not path.exists(): + return None + checkpoint = json.loads(path.read_text(encoding="utf-8")) + if checkpoint.get("schema_version") != 1: + raise ValueError("checkpoint schema version is not supported") + if checkpoint.get("run_signature_sha256") != signature: + raise ValueError("checkpoint belongs to a different frozen evaluation run") + state = checkpoint.get("state") + if not isinstance(state, dict): + raise ValueError("checkpoint does not contain state") + if checkpoint.get("state_sha256") != sha256_bytes(canonical_json_bytes(state)): + raise ValueError("checkpoint state hash does not match its contents") + return state + + +def write_checkpoint(path: Path, *, signature: str, state: dict[str, Any]) -> None: + state_hash = sha256_bytes(canonical_json_bytes(state)) + checkpoint = { + "schema_version": 1, + "run_signature_sha256": signature, + "state_sha256": state_hash, + "state": state, + } + payload = canonical_json_bytes(checkpoint) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_bytes(payload) + temporary.replace(path) + + +def load_calibration_artifact( + path: Path, + *, + phase: str, + allow_diagnostic: bool, + model_id: str, + revision: str, +) -> dict[str, Any]: + artifact = json.loads(path.read_text(encoding="utf-8")) + if artifact.get("artifact_kind") != "recurquant_mbpp_layer_calibration": + raise ValueError("--calibration-artifact has the wrong artifact_kind") + evidence = artifact.get("evidence") + if not isinstance(evidence, dict): + raise ValueError("calibration artifact does not contain evidence") + expected_hash = sha256_bytes(canonical_json_bytes(evidence)) + if artifact.get("canonical_evidence_sha256") != expected_hash: + raise ValueError("calibration artifact evidence hash does not match its contents") + + source = evidence.get("source", {}) + if source.get("model_id") != model_id or source.get("model_revision") != revision: + raise ValueError("calibration artifact model pin does not match the evaluator") + dataset_manifest = source.get("dataset_manifest", {}) + if dataset_manifest.get("revision") != MBPP_REVISION: + raise ValueError("calibration artifact MBPP revision does not match the evaluator") + + protocol_eligible = evidence.get("claim_scope", {}).get("protocol_eligible") is True + diagnostic_allowed = ( + allow_diagnostic and phase == MBPPPhase.CALIBRATION.value + ) + if not protocol_eligible and not diagnostic_allowed: + raise ValueError( + "calibration artifact is diagnostic-only; run the full 128-task calibration " + "before development" + ) + return artifact + + +def load_prepared_manifest( + path: Path, + *, + phase: str, + model_id: str, + revision: str, + dataset_manifest_sha256: str, + token_manifest_sha256: str, + calibration_evidence_sha256: str, +) -> dict[str, Any]: + artifact = json.loads(path.read_text(encoding="utf-8")) + if artifact.get("artifact_kind") != "recurquant_mbpp_prepared_manifest": + raise ValueError("--prepared-manifest has the wrong artifact_kind") + evidence = artifact.get("evidence") + if not isinstance(evidence, dict): + raise ValueError("prepared manifest does not contain evidence") + if artifact.get("canonical_evidence_sha256") != sha256_bytes( + canonical_json_bytes(evidence) + ): + raise ValueError("prepared manifest evidence hash does not match its contents") + source = evidence.get("source", {}) + expected = { + "model_id": model_id, + "model_revision": revision, + "dataset_manifest_sha256": dataset_manifest_sha256, + "token_manifest_sha256": token_manifest_sha256, + "calibration_evidence_sha256": calibration_evidence_sha256, + } + for key, value in expected.items(): + if source.get(key) != value: + raise ValueError(f"prepared manifest {key} does not match the current run") + if evidence.get("claim_scope", {}).get("phase") != phase: + raise ValueError("prepared manifest phase does not match the current run") + if evidence.get("claim_scope", {}).get("protocol_eligible") is not True: + raise ValueError("prepared manifest is diagnostic-only") + return artifact + + +def candidate_definitions(calibration_artifact: dict[str, Any]) -> list[CandidateDefinition]: + layers = calibration_artifact["evidence"]["calibration"]["candidate_layers"] + required = { + "read_risk_l0_nearest": 0, + "random_l18_nearest": 18, + "random_l4_nearest": 4, + "random_l13_nearest": 13, + } + for name, expected_layer in required.items(): + if layers.get(name) != expected_layer: + raise ValueError(f"calibration artifact changed frozen candidate {name}") + mse_layer = layers.get("mse_selected_nearest") + if isinstance(mse_layer, bool) or not isinstance(mse_layer, int): + raise TypeError("mse_selected_nearest must be an integer layer index") + if mse_layer not in GDN_LAYER_INDICES: + raise ValueError("mse_selected_nearest is not a Gated DeltaNet layer") + + candidates = [ + CandidateDefinition("uniform_int4_nearest", 4, None), + CandidateDefinition("uniform_int8_nearest", 8, None), + CandidateDefinition("read_risk_l0_nearest", 4, 0), + CandidateDefinition("mse_selected_nearest", 4, mse_layer), + CandidateDefinition("random_l18_nearest", 4, 18), + CandidateDefinition("random_l4_nearest", 4, 4), + CandidateDefinition("random_l13_nearest", 4, 13), + ] + candidates.extend( + CandidateDefinition( + f"read_risk_l0_stochastic_seed_{seed}", + 4, + 0, + rounding="stochastic", + seed=seed, + ) + for seed in (2339, 2340, 2341) + ) + return candidates + + +def candidate_execution_plan( + candidates: list[CandidateDefinition], +) -> tuple[list[CandidateDefinition], dict[str, str]]: + """Deduplicate policies whose quantization streams are exactly identical.""" + + representative_by_key: dict[tuple[int, int | None, str, int], str] = {} + representatives: list[CandidateDefinition] = [] + aliases: dict[str, str] = {} + for candidate in candidates: + effective_seed = candidate.seed if candidate.rounding == "stochastic" else 0 + key = ( + candidate.default_bits, + candidate.upgrade_layer, + candidate.rounding, + effective_seed, + ) + representative = representative_by_key.get(key) + if representative is None: + representative_by_key[key] = candidate.name + representatives.append(candidate) + representative = candidate.name + aliases[candidate.name] = representative + return representatives, aliases + + +def candidate_specs( + candidate: CandidateDefinition, + *, + group_size: int, +) -> tuple[QuantizationSpec, dict[int, QuantizationSpec]]: + default_spec = QuantizationSpec( + bits=candidate.default_bits, + group_size=group_size, + scale_bits=16, + rounding=candidate.rounding, + seed=candidate.seed, + ) + layer_specs: dict[int, QuantizationSpec] = {} + if candidate.upgrade_layer is not None: + layer_specs[candidate.upgrade_layer] = QuantizationSpec( + bits=8, + group_size=group_size, + scale_bits=16, + rounding=candidate.rounding, + seed=candidate.seed, + ) + return default_spec, layer_specs + + +def make_packed_cache( + config: object, + candidate: CandidateDefinition, + *, + group_size: int, +) -> PackedRecurrentStateCache: + spec, layer_specs = candidate_specs(candidate, group_size=group_size) + return PackedRecurrentStateCache( + config, + spec=spec, + layer_specs=layer_specs, + record_evidence=False, + ) + + +def make_qdq_cache( + config: object, + candidate: CandidateDefinition, + *, + group_size: int, +) -> RecurrentStateQDQCache: + spec, layer_specs = candidate_specs(candidate, group_size=group_size) + return RecurrentStateQDQCache( + config, + spec=spec, + layer_specs=layer_specs, + record_evidence=False, + ) + + +def encode_tasks( + tokenizer: Any, + rows: tuple[dict[str, Any], ...], + *, + max_position_embeddings: int, +) -> list[EncodedTask]: + encoded: list[EncodedTask] = [] + for row in rows: + formatted = format_mbpp_example(row) + prompt_ids = tuple( + tokenizer(formatted.prompt, add_special_tokens=True)["input_ids"] + ) + code_ids = tuple(tokenizer(formatted.code, add_special_tokens=False)["input_ids"]) + if not prompt_ids: + raise RuntimeError(f"MBPP task {row['task_id']} produced an empty prompt") + if not code_ids: + raise RuntimeError(f"MBPP task {row['task_id']} produced an empty code target") + if len(prompt_ids) + len(code_ids) > max_position_embeddings: + raise RuntimeError( + f"MBPP task {row['task_id']} exceeds the pinned model context" + ) + encoded.append( + EncodedTask( + task_id=row["task_id"], + prompt_ids=prompt_ids, + code_ids=code_ids, + prompt_sha256=token_ids_sha256(prompt_ids), + code_sha256=token_ids_sha256(code_ids), + ) + ) + return encoded + + +def _tensor_ids(values: tuple[int, ...], device: torch.device) -> torch.Tensor: + return torch.tensor([values], dtype=torch.long, device=device) + + +def _assert_logits_close( + packed_logits: torch.Tensor, + qdq_logits: torch.Tensor, + *, + candidate: str, + step: int, + tolerance: float, +) -> float: + difference = (packed_logits.to(torch.float32) - qdq_logits.to(torch.float32)).abs().max() + maximum = float(difference.item()) + if maximum > tolerance: + raise RuntimeError( + f"Packed/QDQ parity failed for {candidate} at step {step}: {maximum} > {tolerance}" + ) + return maximum + + +def verify_qdq_parity( + model: Qwen3_5ForCausalLM, + task: EncodedTask, + candidates: list[CandidateDefinition], + *, + group_size: int, + device: torch.device, + tolerance: float = 1e-6, +) -> dict[str, Any]: + prompt = _tensor_ids(task.prompt_ids, device) + code = _tensor_ids(task.code_ids, device) + maximum_by_candidate: dict[str, float] = {} + representatives, aliases = candidate_execution_plan(candidates) + + with torch.inference_mode(): + for candidate in representatives: + packed_cache = make_packed_cache( + model.config, candidate, group_size=group_size + ) + qdq_cache = make_qdq_cache(model.config, candidate, group_size=group_size) + packed_output = model( + prompt, + past_key_values=packed_cache, + use_cache=True, + logits_to_keep=1, + ) + qdq_output = model( + prompt, + past_key_values=qdq_cache, + use_cache=True, + logits_to_keep=1, + ) + maximum = _assert_logits_close( + packed_output.logits, + qdq_output.logits, + candidate=candidate.name, + step=0, + tolerance=tolerance, + ) + for token_index in range(code.shape[1] - 1): + token = code[:, token_index : token_index + 1] + packed_output = model( + token, + past_key_values=packed_cache, + use_cache=True, + logits_to_keep=1, + ) + qdq_output = model( + token, + past_key_values=qdq_cache, + use_cache=True, + logits_to_keep=1, + ) + maximum = max( + maximum, + _assert_logits_close( + packed_output.logits, + qdq_output.logits, + candidate=candidate.name, + step=token_index + 1, + tolerance=tolerance, + ), + ) + + packed_states = { + (state.layer_index, state.state_index): state.tensor + for state in iter_recurrent_states(packed_cache) + } + qdq_states = { + (state.layer_index, state.state_index): state.tensor + for state in iter_recurrent_states(qdq_cache) + } + if packed_states.keys() != qdq_states.keys(): + raise RuntimeError(f"Packed/QDQ state layout differs for {candidate.name}") + for state_key in packed_states: + state_difference = ( + packed_states[state_key].to(torch.float32) + - qdq_states[state_key].to(torch.float32) + ).abs().max() + maximum = max(maximum, float(state_difference.item())) + if maximum > tolerance: + raise RuntimeError( + f"Packed/QDQ state parity failed for {candidate.name}: {maximum}" + ) + maximum_by_candidate[candidate.name] = maximum + + maximum_by_candidate = { + candidate.name: maximum_by_candidate[aliases[candidate.name]] + for candidate in candidates + } + + return { + "task_id": task.task_id, + "absolute_tolerance": tolerance, + "maximum_absolute_difference_by_candidate": maximum_by_candidate, + "passed": True, + } + + +def _append_step_metrics( + reference_logits: torch.Tensor, + candidate_logits: dict[str, torch.Tensor], + target_id: torch.Tensor, + accumulators: dict[str, dict[str, list[torch.Tensor]]], +) -> None: + reference = reference_logits.to(torch.float32) + reference_log_probs = torch.log_softmax(reference, dim=-1) + reference_probs = reference_log_probs.exp() + target = target_id.to(torch.int64).reshape(1, 1, 1) + reference_nll = -reference_log_probs.gather(-1, target).squeeze(-1) + reference_top1 = reference.argmax(dim=-1) + + for name, logits in candidate_logits.items(): + candidate = logits.to(torch.float32) + candidate_log_probs = torch.log_softmax(candidate, dim=-1) + kl = ( + reference_probs * (reference_log_probs - candidate_log_probs) + ).sum(dim=-1) + candidate_nll = -candidate_log_probs.gather(-1, target).squeeze(-1) + accumulators[name]["kl"].append(kl.detach()) + accumulators[name]["reference_nll"].append(reference_nll.detach()) + accumulators[name]["candidate_nll"].append(candidate_nll.detach()) + accumulators[name]["top1"].append( + (reference_top1 == candidate.argmax(dim=-1)).detach() + ) + + +def _task_fidelity( + values: dict[str, list[torch.Tensor]], +) -> TokenFidelity: + return TokenFidelity( + kl=torch.cat(values["kl"], dim=1).cpu(), + reference_nll=torch.cat(values["reference_nll"], dim=1).cpu(), + candidate_nll=torch.cat(values["candidate_nll"], dim=1).cpu(), + top1_agreement=torch.cat(values["top1"], dim=1).cpu(), + ) + + +def evaluate_task( + model: Qwen3_5ForCausalLM, + task: EncodedTask, + candidates: list[CandidateDefinition], + *, + group_size: int, + device: torch.device, +) -> tuple[dict[str, TokenFidelity], dict[str, dict[str, Any]], int]: + prompt = _tensor_ids(task.prompt_ids, device) + code = _tensor_ids(task.code_ids, device) + reference_cache = DynamicCache(config=model.config) + representatives, aliases = candidate_execution_plan(candidates) + caches = { + candidate.name: make_packed_cache( + model.config, candidate, group_size=group_size + ) + for candidate in representatives + } + accumulators: dict[str, dict[str, list[torch.Tensor]]] = { + candidate.name: { + "kl": [], + "reference_nll": [], + "candidate_nll": [], + "top1": [], + } + for candidate in representatives + } + + with torch.inference_mode(): + reference_output = model( + prompt, + past_key_values=reference_cache, + use_cache=True, + logits_to_keep=1, + ) + candidate_outputs = { + name: model( + prompt, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ).logits + for name, cache in caches.items() + } + _append_step_metrics( + reference_output.logits, + candidate_outputs, + code[:, 0], + accumulators, + ) + + for token_index in range(code.shape[1] - 1): + token = code[:, token_index : token_index + 1] + reference_output = model( + token, + past_key_values=reference_cache, + use_cache=True, + logits_to_keep=1, + ) + candidate_outputs = { + name: model( + token, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ).logits + for name, cache in caches.items() + } + _append_step_metrics( + reference_output.logits, + candidate_outputs, + code[:, token_index + 1], + accumulators, + ) + + reference_bytes = sum( + state.tensor.numel() * state.tensor.element_size() + for state in iter_recurrent_states(reference_cache) + ) + storage = {name: cache.storage_summary() for name, cache in caches.items()} + representative_fidelity = { + name: _task_fidelity(values) for name, values in accumulators.items() + } + return ( + { + candidate.name: representative_fidelity[aliases[candidate.name]] + for candidate in candidates + }, + { + candidate.name: storage[aliases[candidate.name]] + for candidate in candidates + }, + reference_bytes, + ) + + +def _length_quartile_membership(tasks: list[EncodedTask]) -> dict[int, str]: + ordered = sorted(tasks, key=lambda task: (len(task.code_ids), task.task_id)) + chunks = np.array_split(np.asarray([task.task_id for task in ordered]), 4) + membership: dict[int, str] = {} + for index, chunk in enumerate(chunks, start=1): + for task_id in chunk.tolist(): + membership[int(task_id)] = f"Q{index}" + return membership + + +def _quantiles(values: list[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "minimum": float(array.min()), + "q25": float(np.quantile(array, 0.25)), + "median": float(np.quantile(array, 0.5)), + "q75": float(np.quantile(array, 0.75)), + "maximum": float(array.max()), + } + + +def _expected_resident_bytes(candidate: CandidateDefinition) -> int: + return EXPECTED_BYTES[candidate.layout] + + +def main() -> int: + args = parse_args() + phase = MBPPPhase(args.phase) + if args.group_size != 128: + raise ValueError("the frozen v0.2 protocol requires --group-size 128") + if args.bootstrap_samples != 10_000: + raise ValueError("the frozen v0.2 protocol requires 10,000 bootstrap samples") + if phase is MBPPPhase.CONFIRMATION and args.confirmation_lock != MBPP_CONFIRMATION_LOCK: + raise ValueError("confirmation requires the explicit frozen confirmation lock") + + calibration_artifact = load_calibration_artifact( + args.calibration_artifact, + phase=phase.value, + allow_diagnostic=args.allow_diagnostic_calibration, + model_id=args.model_id, + revision=args.revision, + ) + candidates = candidate_definitions(calibration_artifact) + rows = load_mbpp_rows( + phase, + limit=args.limit, + confirmation_lock=args.confirmation_lock, + ) + dataset_manifest = mbpp_manifest(rows, phase=phase) + + torch.manual_seed(SEED) + device = select_device(args.device) + dtype = torch.bfloat16 if device.type == "cuda" else torch.float32 + tokenizer = AutoTokenizer.from_pretrained( + args.model_id, + revision=args.revision, + local_files_only=args.local_files_only, + trust_remote_code=False, + ) + model = Qwen3_5ForCausalLM.from_pretrained( + args.model_id, + revision=args.revision, + dtype=dtype, + attn_implementation="eager", + low_cpu_mem_usage=True, + use_safetensors=True, + local_files_only=args.local_files_only, + ).to(device) + model.eval() + + configured_gdn_layers = tuple( + index + for index, layer_type in enumerate(model.config.layer_types) + if layer_type == "linear_attention" + ) + if configured_gdn_layers != GDN_LAYER_INDICES: + raise RuntimeError( + f"Pinned GDN layout mismatch: {configured_gdn_layers} != {GDN_LAYER_INDICES}" + ) + + tasks = encode_tasks( + tokenizer, + rows, + max_position_embeddings=model.config.max_position_embeddings, + ) + token_manifest = [task.manifest_dict() for task in tasks] + token_manifest_sha256 = sha256_bytes(canonical_json_bytes(token_manifest)) + expected_count = { + MBPPPhase.CALIBRATION: MBPP_CALIBRATION_SIZE, + MBPPPhase.DEVELOPMENT: 90, + MBPPPhase.CONFIRMATION: 500, + }[phase] + manifest_protocol_eligible = ( + len(rows) == expected_count + and args.limit is None + and calibration_artifact["evidence"]["claim_scope"]["protocol_eligible"] is True + ) + + if args.manifest_only: + manifest_evidence: dict[str, Any] = { + "claim_scope": { + "phase": phase.value, + "protocol_eligible": manifest_protocol_eligible, + "outcomes_computed": False, + "confirmation_touched": phase is MBPPPhase.CONFIRMATION, + }, + "source": { + "model_id": args.model_id, + "model_revision": args.revision, + "tokenizer_revision": args.revision, + "dataset_manifest": dataset_manifest, + "dataset_manifest_sha256": mbpp_manifest_sha256(rows, phase=phase), + "token_manifest": token_manifest, + "token_manifest_sha256": token_manifest_sha256, + "calibration_evidence_sha256": calibration_artifact[ + "canonical_evidence_sha256" + ], + "repository_commit": git_commit(), + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + "torch": torch.__version__, + "transformers": importlib.metadata.version("transformers"), + "datasets": importlib.metadata.version("datasets"), + "tokenizer_class": tokenizer.__class__.__name__, + "command": [Path(sys.executable).name, *sys.argv], + "tracked_worktree_clean": not subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + check=True, + capture_output=True, + text=True, + ).stdout.strip(), + }, + "schedule": { + "row_count": len(rows), + "candidate_plan": [asdict(candidate) for candidate in candidates], + }, + } + manifest_evidence_hash = sha256_bytes(canonical_json_bytes(manifest_evidence)) + manifest_artifact = { + "schema_version": 1, + "artifact_kind": "recurquant_mbpp_prepared_manifest", + "created_at_utc": datetime.now(UTC).isoformat(), + "canonical_evidence_sha256": manifest_evidence_hash, + "evidence": manifest_evidence, + } + manifest_payload = canonical_json_bytes(manifest_artifact) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(manifest_payload) + print( + json.dumps( + { + "output": str(args.output.resolve()), + "artifact_sha256": sha256_bytes(manifest_payload), + "canonical_evidence_sha256": manifest_evidence_hash, + "phase": phase.value, + "row_count": len(rows), + "token_manifest_sha256": token_manifest_sha256, + "protocol_eligible": manifest_protocol_eligible, + "outcomes_computed": False, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + + prepared_manifest: dict[str, Any] | None = None + if phase in (MBPPPhase.DEVELOPMENT, MBPPPhase.CONFIRMATION): + if args.prepared_manifest is None: + raise ValueError("public development/confirmation requires --prepared-manifest") + prepared_manifest = load_prepared_manifest( + args.prepared_manifest, + phase=phase.value, + model_id=args.model_id, + revision=args.revision, + dataset_manifest_sha256=mbpp_manifest_sha256(rows, phase=phase), + token_manifest_sha256=token_manifest_sha256, + calibration_evidence_sha256=calibration_artifact[ + "canonical_evidence_sha256" + ], + ) + + parity: dict[str, Any] | None = None + if not args.skip_qdq_preflight: + calibration_row = load_mbpp_rows("calibration", limit=1) + parity_task = encode_tasks( + tokenizer, + calibration_row, + max_position_embeddings=model.config.max_position_embeddings, + )[0] + parity = verify_qdq_parity( + model, + parity_task, + candidates, + group_size=args.group_size, + device=device, + ) + elif phase is not MBPPPhase.CALIBRATION or args.limit is None: + raise ValueError("protocol-eligible runs may not skip the packed/QDQ preflight") + + run_signature_evidence = { + "phase": phase.value, + "model_id": args.model_id, + "model_revision": args.revision, + "repository_commit": git_commit(), + "calibration_evidence_sha256": calibration_artifact[ + "canonical_evidence_sha256" + ], + "prepared_manifest_evidence_sha256": ( + prepared_manifest["canonical_evidence_sha256"] + if prepared_manifest is not None + else None + ), + "token_manifest_sha256": token_manifest_sha256, + "group_size": args.group_size, + "candidate_plan": [asdict(candidate) for candidate in candidates], + } + run_signature = sha256_bytes(canonical_json_bytes(run_signature_evidence)) + checkpoint_path = args.checkpoint or args.output.with_suffix(".checkpoint.json") + + global_values: dict[str, dict[str, list[float | bool]]] = { + candidate.name: { + "kl": [], + "reference_nll": [], + "candidate_nll": [], + "top1": [], + } + for candidate in candidates + } + per_task: dict[str, list[dict[str, Any]]] = { + candidate.name: [] for candidate in candidates + } + storage_by_candidate: dict[str, dict[str, Any]] | None = None + reference_state_bytes: int | None = None + completed_task_ids: list[int] = [] + elapsed_before_resume = 0.0 + checkpoint_state = load_checkpoint(checkpoint_path, signature=run_signature) + if checkpoint_state is not None: + completed_task_ids = [int(value) for value in checkpoint_state["completed_task_ids"]] + expected_prefix = [task.task_id for task in tasks[: len(completed_task_ids)]] + if completed_task_ids != expected_prefix: + raise ValueError("checkpoint task IDs are not a prefix of the frozen manifest") + global_values = checkpoint_state["global_values"] + per_task = checkpoint_state["per_task"] + storage_by_candidate = checkpoint_state["storage_by_candidate"] + reference_state_bytes = checkpoint_state["reference_state_bytes"] + elapsed_before_resume = float(checkpoint_state["elapsed_wall_seconds"]) + expected_candidate_names = {candidate.name for candidate in candidates} + if ( + set(global_values) != expected_candidate_names + or set(per_task) != expected_candidate_names + ): + raise ValueError("checkpoint candidate set does not match the frozen plan") + if any(len(rows) != len(completed_task_ids) for rows in per_task.values()): + raise ValueError("checkpoint per-task rows do not match its completed task count") + print( + f"[resume] loaded {len(completed_task_ids)}/{len(tasks)} completed tasks " + f"from {checkpoint_path}", + flush=True, + ) + resumed_task_count = len(completed_task_ids) + started = time.perf_counter() + + for task_index, task in enumerate(tasks, start=1): + if task_index <= len(completed_task_ids): + continue + task_fidelity, storage, task_reference_bytes = evaluate_task( + model, + task, + candidates, + group_size=args.group_size, + device=device, + ) + if reference_state_bytes is None: + reference_state_bytes = task_reference_bytes + elif reference_state_bytes != task_reference_bytes: + raise RuntimeError("reference recurrent-state bytes changed between tasks") + if storage_by_candidate is None: + storage_by_candidate = storage + elif storage_by_candidate != storage: + raise RuntimeError("packed recurrent-state storage changed between tasks") + + for candidate in candidates: + fidelity = task_fidelity[candidate.name] + summary = fidelity_summary(fidelity) + per_task[candidate.name].append( + { + "task_id": task.task_id, + "code_tokens": len(task.code_ids), + **summary, + } + ) + global_values[candidate.name]["kl"].extend( + float(value) for value in fidelity.kl.flatten().tolist() + ) + global_values[candidate.name]["reference_nll"].extend( + float(value) for value in fidelity.reference_nll.flatten().tolist() + ) + global_values[candidate.name]["candidate_nll"].extend( + float(value) for value in fidelity.candidate_nll.flatten().tolist() + ) + global_values[candidate.name]["top1"].extend( + bool(value) for value in fidelity.top1_agreement.flatten().tolist() + ) + completed_task_ids.append(task.task_id) + current_elapsed = elapsed_before_resume + (time.perf_counter() - started) + write_checkpoint( + checkpoint_path, + signature=run_signature, + state={ + "completed_task_ids": completed_task_ids, + "global_values": global_values, + "per_task": per_task, + "storage_by_candidate": storage_by_candidate, + "reference_state_bytes": reference_state_bytes, + "elapsed_wall_seconds": current_elapsed, + }, + ) + print( + f"[{task_index}/{len(tasks)}] task={task.task_id} " + f"prompt={len(task.prompt_ids)} code={len(task.code_ids)}", + flush=True, + ) + + elapsed_seconds = elapsed_before_resume + (time.perf_counter() - started) + assert storage_by_candidate is not None + assert reference_state_bytes is not None + if reference_state_bytes != EXPECTED_BYTES["fp32_state"]: + raise RuntimeError( + f"FP32-state bytes {reference_state_bytes} != {EXPECTED_BYTES['fp32_state']}" + ) + + length_membership = _length_quartile_membership(tasks) + candidate_results: dict[str, Any] = {} + for candidate in candidates: + name = candidate.name + values = global_values[name] + fidelity = TokenFidelity( + kl=torch.tensor(values["kl"], dtype=torch.float32), + reference_nll=torch.tensor(values["reference_nll"], dtype=torch.float32), + candidate_nll=torch.tensor(values["candidate_nll"], dtype=torch.float32), + top1_agreement=torch.tensor(values["top1"], dtype=torch.bool), + ) + token_weighted = fidelity_summary(fidelity) + task_rows = per_task[name] + task_delta = [float(row["delta_nll"]) for row in task_rows] + task_reference_nll = [float(row["reference_nll"]) for row in task_rows] + task_candidate_nll = [float(row["candidate_nll"]) for row in task_rows] + by_quartile: dict[str, Any] = {} + for quartile in ("Q1", "Q2", "Q3", "Q4"): + quartile_rows = [ + row + for row in task_rows + if length_membership[int(row["task_id"])] == quartile + ] + if not quartile_rows: + by_quartile[quartile] = { + "task_count": 0, + "minimum_code_tokens": None, + "maximum_code_tokens": None, + "macro_delta_nll": None, + "macro_mean_kl": None, + "macro_top1_agreement": None, + } + continue + by_quartile[quartile] = { + "task_count": len(quartile_rows), + "minimum_code_tokens": min(int(row["code_tokens"]) for row in quartile_rows), + "maximum_code_tokens": max(int(row["code_tokens"]) for row in quartile_rows), + "macro_delta_nll": fmean(float(row["delta_nll"]) for row in quartile_rows), + "macro_mean_kl": fmean(float(row["mean_kl"]) for row in quartile_rows), + "macro_top1_agreement": fmean( + float(row["top1_agreement"]) for row in quartile_rows + ), + } + + expected_bytes = _expected_resident_bytes(candidate) + actual_storage = storage_by_candidate[name] + if actual_storage["resident_bytes"] != expected_bytes: + raise RuntimeError( + f"{name} resident bytes {actual_storage['resident_bytes']} != {expected_bytes}" + ) + candidate_results[name] = { + "policy": asdict(candidate), + "storage": { + **actual_storage, + "expected_resident_bytes": expected_bytes, + "exact_byte_gate": True, + }, + "token_weighted": token_weighted, + "task_macro": { + "task_count": len(task_rows), + "reference_nll": fmean(task_reference_nll), + "candidate_nll": fmean(task_candidate_nll), + "delta_nll": fmean(task_delta), + "mean_kl": fmean(float(row["mean_kl"]) for row in task_rows), + "top1_agreement": fmean( + float(row["top1_agreement"]) for row in task_rows + ), + }, + "task_delta_nll_distribution": _quantiles(task_delta), + "by_code_length_quartile": by_quartile, + "per_task": task_rows, + } + + uniform_rows = per_task["uniform_int4_nearest"] + primary_rows = per_task["read_risk_l0_nearest"] + uniform_delta = [float(row["delta_nll"]) for row in uniform_rows] + primary_delta = [float(row["delta_nll"]) for row in primary_rows] + random_names = ("random_l18_nearest", "random_l4_nearest", "random_l13_nearest") + random_mean_delta = [ + fmean(float(per_task[name][task_index]["delta_nll"]) for name in random_names) + for task_index in range(len(tasks)) + ] + macro_uniform = fmean(uniform_delta) + macro_primary = fmean(primary_delta) + relative_reduction = ( + (macro_uniform - macro_primary) / macro_uniform if macro_uniform > 0 else None + ) + uniform_bootstrap = paired_bootstrap_mean_improvement( + uniform_delta, + primary_delta, + samples=args.bootstrap_samples, + seed=SEED, + ) + equal_byte_bootstrap = paired_bootstrap_mean_improvement( + random_mean_delta, + primary_delta, + samples=args.bootstrap_samples, + seed=SEED, + ) + primary_token = candidate_results["read_risk_l0_nearest"]["token_weighted"] + uniform_token = candidate_results["uniform_int4_nearest"]["token_weighted"] + finite_values = all( + math.isfinite(float(value)) + for candidate_values in global_values.values() + for metric_values in candidate_values.values() + if metric_values and not isinstance(metric_values[0], bool) + for value in metric_values + ) + continuation_gates = { + "all_values_finite": finite_values, + "exact_resident_bytes": all( + result["storage"]["exact_byte_gate"] for result in candidate_results.values() + ), + "primary_macro_delta_nll_reduction_at_least_15_percent": ( + relative_reduction is not None and relative_reduction >= 0.15 + ), + "equal_byte_bootstrap_interval_above_zero": ( + equal_byte_bootstrap["confidence_interval"][0] > 0 + ), + "primary_mean_token_kl_lower_than_uniform_int4": ( + primary_token["mean_kl"] < uniform_token["mean_kl"] + ), + "primary_cvar95_token_kl_lower_than_uniform_int4": ( + primary_token["cvar95_kl"] < uniform_token["cvar95_kl"] + ), + "primary_top1_not_lower_than_uniform_int4": ( + primary_token["top1_agreement"] >= uniform_token["top1_agreement"] + ), + } + all_continuation_gates_pass = all(continuation_gates.values()) + protocol_eligible = ( + len(rows) == expected_count + and args.limit is None + and parity is not None + and completed_task_ids == [task.task_id for task in tasks] + and calibration_artifact["evidence"]["claim_scope"]["protocol_eligible"] is True + ) + + evidence: dict[str, Any] = { + "claim_scope": { + "phase": phase.value, + "protocol_eligible": protocol_eligible, + "teacher_forced_fidelity_only": True, + "generated_code_executed": False, + "speed_claim_allowed": False, + "whole_model_memory_claim_allowed": False, + "confirmation_touched": phase is MBPPPhase.CONFIRMATION, + }, + "source": { + "model_id": args.model_id, + "model_revision": args.revision, + "tokenizer_revision": args.revision, + "dataset_manifest": dataset_manifest, + "dataset_manifest_sha256": mbpp_manifest_sha256(rows, phase=phase), + "token_manifest": token_manifest, + "token_manifest_sha256": token_manifest_sha256, + "calibration_artifact_path": str(args.calibration_artifact), + "calibration_evidence_sha256": calibration_artifact[ + "canonical_evidence_sha256" + ], + "repository_commit": git_commit(), + "prepared_manifest_evidence_sha256": ( + prepared_manifest["canonical_evidence_sha256"] + if prepared_manifest is not None + else None + ), + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + "torch": torch.__version__, + "transformers": importlib.metadata.version("transformers"), + "datasets": importlib.metadata.version("datasets"), + "cuda_runtime": torch.version.cuda, + "nvidia_driver_versions": nvidia_driver_versions(device), + "device": str(device), + "device_name": ( + torch.cuda.get_device_name(device) if device.type == "cuda" else "CPU" + ), + "model_dtype": str(next(model.parameters()).dtype), + "command": [Path(sys.executable).name, *sys.argv], + "tracked_worktree_clean": not subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + check=True, + capture_output=True, + text=True, + ).stdout.strip(), + }, + "schedule": { + "seed": SEED, + "phase": phase.value, + "row_count": len(rows), + "group_size": args.group_size, + "candidate_order": [candidate.name for candidate in candidates], + "scored_first_code_token_from_prefill": True, + "candidate_generated_tokens_fed_back": False, + "elapsed_wall_seconds_not_a_latency_benchmark": elapsed_seconds, + "run_signature_sha256": run_signature, + "checkpoint_path": str(checkpoint_path), + "resumed_task_count": resumed_task_count, + "final_checkpoint_sha256": sha256_bytes(checkpoint_path.read_bytes()), + }, + "validity": { + "configured_gdn_layer_indices": list(configured_gdn_layers), + "reference_recurrent_state_bytes": reference_state_bytes, + "packed_qdq_preflight": parity, + }, + "candidates": candidate_results, + "contrasts": { + "primary_vs_uniform_int4": { + "uniform_macro_delta_nll": macro_uniform, + "primary_macro_delta_nll": macro_primary, + "relative_reduction": relative_reduction, + "paired_bootstrap": uniform_bootstrap, + }, + "primary_vs_mean_random_equal_byte": { + "paired_bootstrap": equal_byte_bootstrap, + }, + }, + "continuation_decision": { + "gates": continuation_gates, + "all_gates_pass": all_continuation_gates_pass, + "confirmation_permitted": ( + phase is MBPPPhase.DEVELOPMENT + and protocol_eligible + and all_continuation_gates_pass + ), + }, + } + evidence_hash = sha256_bytes(canonical_json_bytes(evidence)) + artifact = { + "schema_version": 1, + "artifact_kind": "recurquant_mbpp_teacher_forced_evaluation", + "created_at_utc": datetime.now(UTC).isoformat(), + "canonical_evidence_sha256": evidence_hash, + "evidence": evidence, + } + payload = canonical_json_bytes(artifact) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(payload) + print( + json.dumps( + { + "output": str(args.output.resolve()), + "artifact_sha256": sha256_bytes(payload), + "canonical_evidence_sha256": evidence_hash, + "protocol_eligible": protocol_eligible, + "phase": phase.value, + "primary_vs_uniform_int4": evidence["contrasts"][ + "primary_vs_uniform_int4" + ], + "primary_vs_mean_random_equal_byte": evidence["contrasts"][ + "primary_vs_mean_random_equal_byte" + ], + "continuation_decision": evidence["continuation_decision"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_readme_assets.py b/scripts/generate_readme_assets.py new file mode 100644 index 0000000..c1042ea --- /dev/null +++ b/scripts/generate_readme_assets.py @@ -0,0 +1,856 @@ +"""Generate deterministic README charts from committed RecurQuant evidence. + +The script deliberately parses the source records instead of duplicating their +numbers. Run it without arguments to update the SVG files, or with ``--check`` +to fail when committed assets are stale. +""" + +# SVG templates remain one element per line so generated diffs stay readable. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import re +import sys +import xml.etree.ElementTree as ET +from decimal import Decimal +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "research" / "EXPERIMENT_002_SCALE_CORRECTION.md" +DEVELOPMENT_SOURCE = ROOT / "evidence" / "mbpp-v02-development.json" +CONFIRMATION_SOURCE = ROOT / "evidence" / "mbpp-v02-confirmation.json" +ASSETS = ROOT / "assets" + + +def _capture(source: str, pattern: str, label: str) -> tuple[str, ...]: + match = re.search(pattern, source, flags=re.MULTILINE) + if match is None: + raise ValueError(f"Could not find {label} in {SOURCE.relative_to(ROOT)}") + return match.groups() + + +def _table_row(source: str, label: str, columns: int) -> tuple[str, ...]: + cells = r"\s*\|\s*".join([r"([^|]+?)"] * columns) + pattern = rf"^\|\s*{re.escape(label)}\s*\|\s*{cells}\s*\|\s*$" + return tuple(cell.strip() for cell in _capture(source, pattern, label)) + + +def _evidence_hash(source: str, label: str) -> str: + (digest,) = _capture( + source, + rf"^-\s*{re.escape(label)}:\s*`([0-9a-f]{{64}})`\s*$", + label, + ) + return digest + + +def _number(value: str) -> Decimal: + return Decimal(value.replace(",", "").replace("%", "").split("x", 1)[0].strip()) + + +def _source_data() -> dict[str, object]: + raw = SOURCE.read_bytes() + text = raw.decode("utf-8") + + diagnostics = [] + for source_label, display_label in ( + ("Retrieval", "Retrieval"), + ("Code", "Code"), + ("Multilingual correction replay", "Multilingual replay"), + ): + uniform, mixed, reduction = _table_row(text, source_label, 3) + diagnostics.append( + { + "label": display_label, + "uniform": uniform, + "mixed": mixed, + "reduction": reduction, + } + ) + + storage = [] + for source_label, display_label, detail in ( + ("FP32 recurrent states", "FP32 states", "reference"), + ("Uniform INT4 plus FP16 scales", "Uniform INT4", "FP16 scales"), + ( + "Layer 0 INT8, rest INT4, plus FP16 scales", + "Mixed: L0 INT8", + "17 layers INT4 · FP16 scales", + ), + ): + resident_bytes, ratio = _table_row(text, source_label, 2) + storage.append( + { + "label": display_label, + "detail": detail, + "bytes": resident_bytes, + "ratio": ratio, + } + ) + + evidence_hashes = { + "retrieval_sweep": _evidence_hash(text, "Corrected retrieval sweep evidence hash"), + "code_sweep": _evidence_hash(text, "Corrected code sweep evidence hash"), + "multilingual_qdq": _evidence_hash(text, "Corrected multilingual QDQ replay evidence hash"), + "multilingual_packed": _evidence_hash(text, "Packed multilingual replay evidence hash"), + } + return { + "document_sha256": hashlib.sha256(raw).hexdigest(), + "diagnostics": diagnostics, + "storage": storage, + "evidence_hashes": evidence_hashes, + } + + +def _development_data() -> dict[str, object]: + raw = DEVELOPMENT_SOURCE.read_bytes() + document = json.loads(raw) + evidence = document["evidence"] + claim_scope = evidence["claim_scope"] + schedule = evidence["schedule"] + candidates = evidence["candidates"] + primary = candidates["read_risk_l0_nearest"] + mse = candidates["mse_selected_nearest"] + uniform = candidates["uniform_int4_nearest"] + primary_contrast = evidence["contrasts"]["primary_vs_uniform_int4"] + random_contrast = evidence["contrasts"]["primary_vs_mean_random_equal_byte"] + + if claim_scope["phase"] != "development": + raise ValueError("MBPP README evidence must remain development-only") + if claim_scope["confirmation_touched"]: + raise ValueError("Refusing to label evidence as development after confirmation access") + if not claim_scope["teacher_forced_fidelity_only"]: + raise ValueError("README chart expects teacher-forced fidelity evidence") + if claim_scope["generated_code_executed"]: + raise ValueError("README chart must not imply generated-code execution") + if claim_scope["speed_claim_allowed"]: + raise ValueError("README chart must not imply a speed result") + if claim_scope["whole_model_memory_claim_allowed"]: + raise ValueError("README chart must not imply whole-model memory reduction") + if primary["policy"]["upgrade_layer"] != 0: + raise ValueError("README chart expects the frozen primary policy to upgrade layer 0") + if mse["policy"]["upgrade_layer"] != primary["policy"]["upgrade_layer"]: + raise ValueError("README caveat expects the MSE and primary selectors to coincide") + if primary["task_macro"] != mse["task_macro"]: + raise ValueError("Coincident layer selectors must have identical task-macro results") + if not primary["storage"]["exact_byte_gate"]: + raise ValueError("README storage claim requires the exact-byte gate") + if primary["task_macro"]["task_count"] != schedule["row_count"]: + raise ValueError("MBPP task counts disagree") + if primary["token_weighted"]["token_count"] != uniform["token_weighted"]["token_count"]: + raise ValueError("MBPP candidate token counts disagree") + + return { + "document_sha256": hashlib.sha256(raw).hexdigest(), + "canonical_evidence_sha256": document["canonical_evidence_sha256"], + "phase": claim_scope["phase"], + "confirmation_touched": claim_scope["confirmation_touched"], + "teacher_forced_fidelity_only": claim_scope["teacher_forced_fidelity_only"], + "generated_code_executed": claim_scope["generated_code_executed"], + "speed_claim_allowed": claim_scope["speed_claim_allowed"], + "whole_model_memory_claim_allowed": claim_scope["whole_model_memory_claim_allowed"], + "task_count": schedule["row_count"], + "token_count": primary["token_weighted"]["token_count"], + "uniform_macro_delta_nll": uniform["task_macro"]["delta_nll"], + "primary_macro_delta_nll": primary["task_macro"]["delta_nll"], + "relative_reduction": primary_contrast["relative_reduction"], + "primary_vs_uniform": primary_contrast["paired_bootstrap"], + "primary_vs_mean_random_equal_byte": random_contrast["paired_bootstrap"], + "primary_resident_bytes": primary["storage"]["resident_bytes"], + "uniform_resident_bytes": uniform["storage"]["resident_bytes"], + "full_precision_equivalent_bytes": primary["storage"]["full_precision_equivalent_bytes"], + "primary_upgrade_layer": primary["policy"]["upgrade_layer"], + "mse_upgrade_layer": mse["policy"]["upgrade_layer"], + } + + +def _reject_non_finite(value: str) -> None: + raise ValueError(f"Non-finite JSON constant is not permitted: {value}") + + +def _confirmation_data() -> dict[str, object]: + raw = CONFIRMATION_SOURCE.read_bytes() + document = json.loads(raw, parse_constant=_reject_non_finite) + evidence = document["evidence"] + claim_scope = evidence["claim_scope"] + schedule = evidence["schedule"] + candidates = evidence["candidates"] + primary = candidates["read_risk_l0_nearest"] + mse = candidates["mse_selected_nearest"] + uniform = candidates["uniform_int4_nearest"] + primary_contrast = evidence["contrasts"]["primary_vs_uniform_int4"] + random_contrast = evidence["contrasts"]["primary_vs_mean_random_equal_byte"] + continuation = evidence["continuation_decision"] + + if document["artifact_kind"] != "recurquant_mbpp_teacher_forced_evaluation": + raise ValueError("Unexpected MBPP confirmation artifact kind") + if document["schema_version"] != 1: + raise ValueError("Unexpected MBPP confirmation schema version") + canonical = hashlib.sha256( + (json.dumps(evidence, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + ).hexdigest() + if document["canonical_evidence_sha256"] != canonical: + raise ValueError("MBPP confirmation canonical evidence hash does not verify") + + if claim_scope["phase"] != "confirmation": + raise ValueError("MBPP confirmation chart requires confirmation-phase evidence") + if not claim_scope["confirmation_touched"]: + raise ValueError("MBPP confirmation chart requires confirmation_touched=true") + if not claim_scope["protocol_eligible"]: + raise ValueError("MBPP confirmation chart requires protocol-eligible evidence") + if not claim_scope["teacher_forced_fidelity_only"]: + raise ValueError("MBPP confirmation chart expects teacher-forced fidelity evidence") + if claim_scope["generated_code_executed"]: + raise ValueError("MBPP confirmation chart must not imply generated-code execution") + if claim_scope["speed_claim_allowed"]: + raise ValueError("MBPP confirmation chart must not imply a speed result") + if claim_scope["whole_model_memory_claim_allowed"]: + raise ValueError("MBPP confirmation chart must not imply whole-model memory reduction") + if not continuation["all_gates_pass"]: + raise ValueError("MBPP confirmation chart requires all preregistered quality gates") + + expected_task_count = 500 + expected_token_count = 30_244 + if schedule["phase"] != "confirmation": + raise ValueError("MBPP confirmation schedule phase disagrees with claim scope") + if schedule["row_count"] != expected_task_count: + raise ValueError("MBPP confirmation must contain exactly 500 held-out tasks") + if evidence["source"]["dataset_manifest"]["row_count"] != expected_task_count: + raise ValueError("MBPP confirmation dataset manifest task count disagrees") + for label, candidate in ( + ("uniform INT4", uniform), + ("read-risk mixed", primary), + ("MSE mixed", mse), + ): + if candidate["task_macro"]["task_count"] != expected_task_count: + raise ValueError(f"{label} task count disagrees with the confirmation run") + if candidate["token_weighted"]["token_count"] != expected_token_count: + raise ValueError(f"{label} token count disagrees with the confirmation run") + + if primary["policy"]["upgrade_layer"] != 0: + raise ValueError("MBPP confirmation chart expects the frozen policy to upgrade layer 0") + if mse["policy"]["upgrade_layer"] != primary["policy"]["upgrade_layer"]: + raise ValueError("Read-risk and MSE selectors no longer choose the same layer") + for result_key in ("task_macro", "token_weighted", "storage"): + if mse[result_key] != primary[result_key]: + raise ValueError(f"Coincident read-risk and MSE selectors disagree on {result_key}") + + expected_storage = { + "uniform": 2_433_024, + "primary": 2_564_096, + "full_precision": 18_874_368, + } + for label, candidate, expected_bytes in ( + ("uniform INT4", uniform, expected_storage["uniform"]), + ("mixed", primary, expected_storage["primary"]), + ): + storage = candidate["storage"] + if not storage["exact_byte_gate"] or not storage["physical_reduction_realized"]: + raise ValueError(f"{label} storage is not exact and physically realized") + if storage["resident_bytes"] != expected_bytes: + raise ValueError(f"{label} resident storage changed unexpectedly") + if storage["expected_resident_bytes"] != storage["resident_bytes"]: + raise ValueError(f"{label} measured and expected resident bytes disagree") + if storage["full_precision_equivalent_bytes"] != expected_storage["full_precision"]: + raise ValueError(f"{label} full-precision storage reference disagrees") + if ( + evidence["validity"]["reference_recurrent_state_bytes"] + != expected_storage["full_precision"] + ): + raise ValueError("Confirmation validity record has a different storage reference") + + uniform_macro = Decimal(str(uniform["task_macro"]["delta_nll"])) + primary_macro = Decimal(str(primary["task_macro"]["delta_nll"])) + relative_reduction = Decimal(str(primary_contrast["relative_reduction"])) + computed_reduction = (uniform_macro - primary_macro) / uniform_macro + if abs(relative_reduction - computed_reduction) > Decimal("1e-15"): + raise ValueError("MBPP confirmation relative reduction does not match candidates") + if f"{relative_reduction * Decimal(100):.2f}" != "72.75": + raise ValueError("MBPP confirmation headline reduction changed unexpectedly") + + for label, contrast in ( + ("uniform", primary_contrast["paired_bootstrap"]), + ("equal-byte random", random_contrast["paired_bootstrap"]), + ): + if contrast["paired_examples"] != expected_task_count: + raise ValueError(f"{label} bootstrap task count disagrees with confirmation") + if contrast["confidence"] != 0.95: + raise ValueError(f"{label} confirmation interval must be a 95% interval") + low, high = contrast["confidence_interval"] + if not (0 < low <= contrast["mean_improvement"] <= high): + raise ValueError(f"{label} paired interval does not support the chart claim") + + return { + "document_sha256": hashlib.sha256(raw).hexdigest(), + "canonical_evidence_sha256": canonical, + "phase": claim_scope["phase"], + "confirmation_touched": claim_scope["confirmation_touched"], + "protocol_eligible": claim_scope["protocol_eligible"], + "teacher_forced_fidelity_only": claim_scope["teacher_forced_fidelity_only"], + "generated_code_executed": claim_scope["generated_code_executed"], + "speed_claim_allowed": claim_scope["speed_claim_allowed"], + "whole_model_memory_claim_allowed": claim_scope["whole_model_memory_claim_allowed"], + "all_quality_gates_pass": continuation["all_gates_pass"], + "task_count": expected_task_count, + "token_count": expected_token_count, + "uniform_macro_delta_nll": uniform["task_macro"]["delta_nll"], + "primary_macro_delta_nll": primary["task_macro"]["delta_nll"], + "relative_reduction": primary_contrast["relative_reduction"], + "primary_vs_uniform": primary_contrast["paired_bootstrap"], + "primary_vs_mean_random_equal_byte": random_contrast["paired_bootstrap"], + "primary_resident_bytes": primary["storage"]["resident_bytes"], + "uniform_resident_bytes": uniform["storage"]["resident_bytes"], + "full_precision_equivalent_bytes": expected_storage["full_precision"], + "primary_upgrade_layer": primary["policy"]["upgrade_layer"], + "mse_upgrade_layer": mse["policy"]["upgrade_layer"], + } + + +def _metadata( + payload: dict[str, object], + *, + source_document: str = "research/EXPERIMENT_002_SCALE_CORRECTION.md", +) -> str: + record = { + "generated_by": "scripts/generate_readme_assets.py", + "source_document": source_document, + **payload, + } + encoded = json.dumps(record, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + return html.escape(encoded, quote=False) + + +def _storage_svg(data: dict[str, object]) -> str: + rows = data["storage"] + assert isinstance(rows, list) + evidence_hashes = data["evidence_hashes"] + assert isinstance(evidence_hashes, dict) + + width = Decimal(600) + axis_max = Decimal(20_000_000) + bar_x = Decimal(250) + y_positions = (Decimal(95), Decimal(171), Decimal(247)) + row_markup = [] + colors = ("reference", "uniform", "mixed") + for row, y, color in zip(rows, y_positions, colors, strict=True): + assert isinstance(row, dict) + resident = _number(str(row["bytes"])) + bar_width = resident / axis_max * width + ratio = str(row["ratio"]) + ratio_label = "1.000x reference" if color == "reference" else ratio + row_markup.append( + f''' + {html.escape(str(row["label"]))} + {html.escape(str(row["detail"]))} · {html.escape(ratio_label)} + + {html.escape(str(row["label"]))}: {resident:,} resident bytes; {html.escape(ratio_label)} + + {resident:,} + ''' + ) + + ticks = [] + for million in range(0, 21, 5): + x = bar_x + Decimal(million) / Decimal(20) * width + ticks.append( + f''' + {million}''' + ) + + metadata = _metadata( + { + "chart": "resident-recurrent-state-bytes", + "source_document_sha256": data["document_sha256"], + "canonical_evidence_sha256": { + "packed_multilingual_replay": evidence_hashes["multilingual_packed"] + }, + "values": rows, + } + ) + fp32, uniform, mixed = rows + storage_desc = ( + f"At batch one, FP32 recurrent states occupy {fp32['bytes']} bytes. " + f"Uniform INT4 with FP16 scales occupies {uniform['bytes']} bytes, " + f"{_number(str(uniform['ratio']))} times smaller. Keeping layer zero at INT8 " + f"and the other 17 layers at INT4 occupies {mixed['bytes']} bytes, " + f"{_number(str(mixed['ratio']))} times smaller. These are persistent state " + "bytes only, not whole-model or peak memory." + ) + return f""" + + Resident recurrent-state storage on Qwen3.5-0.8B-Base + {html.escape(storage_desc)} + {metadata} + + Resident recurrent-state storage + Qwen3.5-0.8B-Base · batch 1 · resident bytes (millions) · lower is better +{chr(10).join(ticks)} +{chr(10).join(row_markup)} + Persistent recurrent state only; excludes model weights and six standard KV caches. + The Python path materializes one layer state; no peak-memory or speed claim. + +""" + + +def _diagnostic_svg(data: dict[str, object]) -> str: + rows = data["diagnostics"] + assert isinstance(rows, list) + evidence_hashes = data["evidence_hashes"] + assert isinstance(evidence_hashes, dict) + + plot_top = Decimal(96) + plot_bottom = Decimal(350) + plot_height = plot_bottom - plot_top + axis_max = Decimal(8) + centers = (Decimal(230), Decimal(480), Decimal(730)) + bar_width = Decimal(58) + series_gap = Decimal(9) + groups = [] + for row, center in zip(rows, centers, strict=True): + assert isinstance(row, dict) + uniform = _number(str(row["uniform"])) + mixed = _number(str(row["mixed"])) + uniform_height = uniform / axis_max * plot_height + mixed_height = mixed / axis_max * plot_height + uniform_x = center - bar_width - series_gap + mixed_x = center + series_gap + uniform_y = plot_bottom - uniform_height + mixed_y = plot_bottom - mixed_height + groups.append( + f''' + + {html.escape(str(row["label"]))}, uniform INT4: {uniform} CVaR95 KL + + {uniform} + + {html.escape(str(row["label"]))}, mixed layer zero INT8: {mixed} CVaR95 KL + + {mixed} + {html.escape(str(row["label"]))} + −{html.escape(str(row["reduction"]))} mixed vs uniform + ''' + ) + + grid = [] + for tick in range(0, 9, 2): + y = plot_bottom - Decimal(tick) / axis_max * plot_height + grid.append( + f''' + {tick}''' + ) + + metadata = _metadata( + { + "chart": "corrected-diagnostic-cvar95-kl", + "source_document_sha256": data["document_sha256"], + "canonical_evidence_sha256": { + "retrieval_sweep": evidence_hashes["retrieval_sweep"], + "code_sweep": evidence_hashes["code_sweep"], + "multilingual_qdq": evidence_hashes["multilingual_qdq"], + "multilingual_packed": evidence_hashes["multilingual_packed"], + }, + "values": rows, + } + ) + comparisons = ", ".join( + f"{row['uniform']} versus {row['mixed']} for {str(row['label']).lower()}" for row in rows + ) + diagnostic_desc = ( + "On short diagnostic traces, uniform INT4 versus layer zero INT8 and the " + f"remaining layers INT4 produced CVaR95 KL of {comparisons}. Lower is better. " + "These diagnostics are not a public benchmark or breakthrough evidence." + ) + return f""" + + Corrected diagnostic tail KL by recurrent-state precision policy + {html.escape(diagnostic_desc)} + {metadata} + + Corrected diagnostic tail KL + CVaR95 token KL · stored FP16 scales · lower is better + + Uniform INT4 + + L0 INT8, rest INT4 +{chr(10).join(grid)} +{chr(10).join(groups)} + CVaR95 KL + Short paired diagnostics, not a public benchmark or breakthrough claim. + Multilingual is a correction replay; packed and QDQ token metrics matched exactly. + +""" + + +def _development_svg(data: dict[str, object]) -> str: + uniform = Decimal(str(data["uniform_macro_delta_nll"])) + primary = Decimal(str(data["primary_macro_delta_nll"])) + reduction = Decimal(str(data["relative_reduction"])) * Decimal(100) + primary_contrast = data["primary_vs_uniform"] + random_contrast = data["primary_vs_mean_random_equal_byte"] + assert isinstance(primary_contrast, dict) + assert isinstance(random_contrast, dict) + + task_count = int(data["task_count"]) + token_count = int(data["token_count"]) + for label, contrast in ( + ("uniform", primary_contrast), + ("equal-byte random", random_contrast), + ): + if contrast["paired_examples"] != task_count: + raise ValueError(f"{label} bootstrap task count disagrees with MBPP run") + if contrast["confidence"] != 0.95: + raise ValueError(f"{label} README interval must be a 95% interval") + + uniform_ci = [Decimal(str(value)) for value in primary_contrast["confidence_interval"]] + random_ci = [Decimal(str(value)) for value in random_contrast["confidence_interval"]] + uniform_mean = Decimal(str(primary_contrast["mean_improvement"])) + random_mean = Decimal(str(random_contrast["mean_improvement"])) + + axis_max = Decimal("3.2") + bar_x = Decimal(250) + plot_width = Decimal(600) + rows = ( + ("Uniform INT4", "all 18 recurrent layers INT4", uniform, "uniform", Decimal(110)), + ( + "Mixed L0 INT8", + "layer 0 INT8; other 17 layers INT4", + primary, + "mixed", + Decimal(190), + ), + ) + row_markup = [] + for label, detail, value, color, y in rows: + bar_width = value / axis_max * plot_width + row_markup.append( + f''' + {label} + {detail} + + {label}: {value:.6f} task-macro excess NLL + + {value:.4f} + ''' + ) + + ticks = [] + for tick in range(4): + x = bar_x + Decimal(tick) / axis_max * plot_width + ticks.append( + f''' + {tick}''' + ) + + metadata = _metadata( + { + "chart": "mbpp-development-task-macro-excess-nll", + "source_document_sha256": data["document_sha256"], + "canonical_evidence_sha256": data["canonical_evidence_sha256"], + "phase": data["phase"], + "confirmation_touched": data["confirmation_touched"], + "teacher_forced_fidelity_only": data["teacher_forced_fidelity_only"], + "generated_code_executed": data["generated_code_executed"], + "speed_claim_allowed": data["speed_claim_allowed"], + "whole_model_memory_claim_allowed": data["whole_model_memory_claim_allowed"], + "task_count": task_count, + "token_count": token_count, + "uniform_macro_delta_nll": data["uniform_macro_delta_nll"], + "primary_macro_delta_nll": data["primary_macro_delta_nll"], + "relative_reduction": data["relative_reduction"], + "primary_vs_uniform": primary_contrast, + "primary_vs_mean_random_equal_byte": random_contrast, + "primary_upgrade_layer": data["primary_upgrade_layer"], + "mse_upgrade_layer": data["mse_upgrade_layer"], + }, + source_document="evidence/mbpp-v02-development.json", + ) + description = ( + f"On the MBPP validation development split, uniform INT4 had task-macro " + f"excess negative log likelihood {uniform:.6f}, while keeping recurrent layer " + f"zero at INT8 and the other 17 recurrent layers at INT4 had {primary:.6f}, " + f"a {reduction:.2f} percent reduction. Across {task_count} paired tasks, the " + f"mean improvement was {uniform_mean:.6f} nats per token with a 95 percent " + f"bootstrap interval from {uniform_ci[0]:.6f} to {uniform_ci[1]:.6f}. This is " + f"teacher-forced development evidence over {token_count} reference code tokens, " + "not confirmation or generated-code correctness evidence." + ) + return f""" + + MBPP development task-macro excess negative log likelihood + {html.escape(description)} + {metadata} + + MBPP development fidelity + Task-macro excess NLL (nats/token) · lower is better +{chr(10).join(ticks)} +{chr(10).join(row_markup)} + {reduction:.2f}% lower macro excess NLL + Paired mixed-vs-uniform improvement: {uniform_mean:.4f}; 95% bootstrap CI {uniform_ci[0]:.4f}–{uniform_ci[1]:.4f}. + Equal-byte mixed-vs-mean-random improvement: {random_mean:.4f}; 95% bootstrap CI {random_ci[0]:.4f}–{random_ci[1]:.4f}. + MBPP validation development split · {task_count} tasks · {token_count:,} teacher-forced reference code tokens. + Development evidence only; no confirmation, generated-code correctness, speed, or whole-model memory claim. + +""" + + +def _confirmation_svg(data: dict[str, object]) -> str: + uniform = Decimal(str(data["uniform_macro_delta_nll"])) + primary = Decimal(str(data["primary_macro_delta_nll"])) + reduction = Decimal(str(data["relative_reduction"])) * Decimal(100) + primary_contrast = data["primary_vs_uniform"] + random_contrast = data["primary_vs_mean_random_equal_byte"] + assert isinstance(primary_contrast, dict) + assert isinstance(random_contrast, dict) + + task_count = int(data["task_count"]) + token_count = int(data["token_count"]) + uniform_ci = [Decimal(str(value)) for value in primary_contrast["confidence_interval"]] + random_ci = [Decimal(str(value)) for value in random_contrast["confidence_interval"]] + uniform_mean = Decimal(str(primary_contrast["mean_improvement"])) + random_mean = Decimal(str(random_contrast["mean_improvement"])) + + axis_max = Decimal("3.2") + bar_x = Decimal(250) + plot_width = Decimal(600) + rows = ( + ( + "Uniform INT4", + "all 18 recurrent layers INT4", + uniform, + "uniform", + Decimal(110), + ), + ( + "Mixed L0 INT8", + "layer 0 INT8; other 17 layers INT4", + primary, + "mixed", + Decimal(190), + ), + ) + row_markup = [] + for label, detail, value, color, y in rows: + bar_width = value / axis_max * plot_width + row_markup.append( + f''' + {label} + {detail} + + {label}: {value:.6f} task-macro excess NLL + + {value:.4f} + ''' + ) + + ticks = [] + for tick in range(4): + x = bar_x + Decimal(tick) / axis_max * plot_width + ticks.append( + f''' + {tick}''' + ) + + exact_storage = { + "mixed_resident_bytes": data["primary_resident_bytes"], + "uniform_int4_resident_bytes": data["uniform_resident_bytes"], + "fp32_recurrent_reference_bytes": data["full_precision_equivalent_bytes"], + } + metadata = _metadata( + { + "chart": "mbpp-confirmation-task-macro-excess-nll", + "source_document_sha256": data["document_sha256"], + "canonical_evidence_sha256": data["canonical_evidence_sha256"], + "phase": data["phase"], + "confirmation_touched": data["confirmation_touched"], + "protocol_eligible": data["protocol_eligible"], + "teacher_forced_fidelity_only": data["teacher_forced_fidelity_only"], + "generated_code_executed": data["generated_code_executed"], + "speed_claim_allowed": data["speed_claim_allowed"], + "whole_model_memory_claim_allowed": data["whole_model_memory_claim_allowed"], + "all_quality_gates_pass": data["all_quality_gates_pass"], + "task_count": task_count, + "token_count": token_count, + "uniform_macro_delta_nll": data["uniform_macro_delta_nll"], + "primary_macro_delta_nll": data["primary_macro_delta_nll"], + "relative_reduction": data["relative_reduction"], + "primary_vs_uniform": primary_contrast, + "primary_vs_mean_random_equal_byte": random_contrast, + "exact_storage": exact_storage, + "primary_upgrade_layer": data["primary_upgrade_layer"], + "mse_upgrade_layer": data["mse_upgrade_layer"], + }, + source_document="evidence/mbpp-v02-confirmation.json", + ) + description = ( + f"On the held-out MBPP confirmation split, uniform INT4 had task-macro " + f"excess negative log likelihood {uniform:.6f}, while keeping recurrent layer " + f"zero at INT8 and the other 17 recurrent layers at INT4 had {primary:.6f}, " + f"a {reduction:.2f} percent reduction. Across {task_count} paired tasks, the " + f"mean improvement was {uniform_mean:.6f} nats per token with a 95 percent " + f"bootstrap interval from {uniform_ci[0]:.6f} to {uniform_ci[1]:.6f}. This is " + f"teacher-forced fidelity evidence over {token_count} reference code tokens. " + "Generated code was not executed, and the evidence supports no speed or " + "whole-model memory claim." + ) + return f""" + + MBPP held-out confirmation task-macro excess negative log likelihood + {html.escape(description)} + {metadata} + + MBPP held-out confirmation fidelity + Task-macro excess NLL (nats/token) · teacher-forced · lower is better +{chr(10).join(ticks)} +{chr(10).join(row_markup)} + {reduction:.2f}% lower macro excess NLL + Paired mixed-vs-uniform improvement: {uniform_mean:.4f}; 95% bootstrap CI {uniform_ci[0]:.4f}–{uniform_ci[1]:.4f}. + Equal-byte mixed-vs-mean-random improvement: {random_mean:.4f}; 95% bootstrap CI {random_ci[0]:.4f}–{random_ci[1]:.4f}. + Read-risk and MSE selectors both chose recurrent layer 0. + Exact resident recurrent-state bytes: mixed {int(data["primary_resident_bytes"]):,}; uniform INT4 {int(data["uniform_resident_bytes"]):,}; FP32 reference {int(data["full_precision_equivalent_bytes"]):,}. + Held-out MBPP confirmation · {task_count} tasks · {token_count:,} teacher-forced reference code tokens. + Generated code was not executed; no speed or whole-model memory claim. + +""" + + +def _outputs() -> dict[Path, str]: + data = _source_data() + development_data = _development_data() + confirmation_data = _confirmation_data() + return { + ASSETS / "recurrent-state-storage.svg": _storage_svg(data), + ASSETS / "diagnostic-tail-kl.svg": _diagnostic_svg(data), + ASSETS / "mbpp-development-fidelity.svg": _development_svg(development_data), + ASSETS / "mbpp-confirmation-fidelity.svg": _confirmation_svg(confirmation_data), + } + + +def _validate_svg(content: str, path: Path) -> None: + try: + ET.fromstring(content) + except ET.ParseError as exc: + raise ValueError(f"Generated invalid SVG for {path}: {exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="fail if generated assets differ from the files on disk", + ) + args = parser.parse_args() + + outputs = _outputs() + stale = [] + for path, content in outputs.items(): + _validate_svg(content, path) + if args.check: + if not path.exists() or path.read_text(encoding="utf-8") != content: + stale.append(path.relative_to(ROOT)) + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + digest = hashlib.sha256(content.encode("utf-8")).hexdigest() + print(f"wrote {path.relative_to(ROOT)} sha256={digest}") + + if stale: + for path in stale: + print(f"stale: {path}", file=sys.stderr) + return 1 + if args.check: + print("README assets are up to date and valid XML.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_qwen35_smoke.py b/scripts/run_qwen35_smoke.py index ff240da..4de41c3 100644 --- a/scripts/run_qwen35_smoke.py +++ b/scripts/run_qwen35_smoke.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run a small paired FP32-state versus QDQ-state Qwen3.5 smoke experiment.""" +"""Run a paired FP32-state versus quantized-state Qwen3.5 experiment.""" from __future__ import annotations @@ -24,6 +24,7 @@ token_kl_divergence, top1_agreement, ) +from recurquant.packed_cache import PackedRecurrentStateCache from recurquant.quantization import QuantizationSpec from recurquant.signals import GatedDeltaSignalRecorder from recurquant.transformers_cache import RecurrentStateQDQCache @@ -74,6 +75,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--bits", type=int, nargs="+", default=[8, 4]) parser.add_argument("--group-size", type=int, default=128) parser.add_argument("--rounding", choices=("nearest", "stochastic"), default="nearest") + parser.add_argument("--cache-mode", choices=("qdq", "packed"), default="qdq") parser.add_argument("--sensitivity-sweep", action="store_true") parser.add_argument("--low-bits", type=int, default=4) parser.add_argument("--high-bits", type=int, default=8) @@ -222,11 +224,15 @@ def main() -> int: } reference_cache = DynamicCache(config=model.config) + cache_class = ( + PackedRecurrentStateCache if args.cache_mode == "packed" else RecurrentStateQDQCache + ) candidates = { - label: RecurrentStateQDQCache( + label: cache_class( model.config, spec=definition["default_spec"], layer_specs=definition["layer_specs"], + **({"record_evidence": True} if args.cache_mode == "packed" else {}), ) for label, definition in candidate_definitions.items() } @@ -287,6 +293,9 @@ def main() -> int: first_by_layer.setdefault(row.layer_index, row) total_baseline = sum(row.baseline_bytes for row in first_by_layer.values()) total_estimated = sum(row.estimated_bytes for row in first_by_layer.values()) + physical_reduction_realized = ( + args.cache_mode == "packed" and total_estimated < total_baseline + ) definition = candidate_definitions[label] default_spec = definition["default_spec"] layer_specs = definition["layer_specs"] @@ -306,7 +315,9 @@ def main() -> int: "estimated_bytes": total_estimated, "compression_ratio": total_baseline / total_estimated, "includes_scale_overhead": True, - "physical_reduction_realized": False, + "resident_bytes": total_estimated if args.cache_mode == "packed" else None, + "physical_reduction_realized": physical_reduction_realized, + "scope": "persistent_recurrent_state_payload_and_scales", }, "first_update_by_layer": [ row.evidence_dict() @@ -321,7 +332,11 @@ def main() -> int: evidence: dict[str, object] = { "claim_scope": { "diagnostic_only": True, - "memory_reduction_realized": False, + "memory_reduction_realized": all( + result["persistent_state_storage"]["physical_reduction_realized"] + for result in candidate_results.values() + ), + "memory_scope": "persistent recurrent-state payload and scales only", "latency_claim_allowed": False, "research_result": False, }, @@ -347,6 +362,7 @@ def main() -> int: "candidate_order": list(candidate_definitions), "group_size": args.group_size, "rounding": args.rounding, + "cache_mode": args.cache_mode, "prompt_profile": args.prompt_profile, "requested_upgrade_layers": args.upgrade_layers, "token_ids_sha256": sha256_bytes(tokens.detach().cpu().numpy().tobytes()), @@ -462,7 +478,11 @@ def main() -> int: canonical_evidence_sha256 = sha256_bytes(canonical_json_bytes(evidence)) artifact: dict[str, object] = { "schema_version": 1, - "artifact_kind": "qwen35_recurrent_state_qdq_smoke", + "artifact_kind": ( + "qwen35_recurrent_state_packed_smoke" + if args.cache_mode == "packed" + else "qwen35_recurrent_state_qdq_smoke" + ), "created_at_utc": datetime.now(UTC).isoformat(), "canonical_evidence_sha256": canonical_evidence_sha256, "evidence": evidence, diff --git a/scripts/verify_mbpp_confirmation.py b/scripts/verify_mbpp_confirmation.py new file mode 100644 index 0000000..ddd91c8 --- /dev/null +++ b/scripts/verify_mbpp_confirmation.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify the frozen RecurQuant v0.2 MBPP confirmation evidence offline.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence +from pathlib import Path + +from recurquant.confirmation import verify_mbpp_confirmation + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Verify a RecurQuant v0.2 confirmation artifact against its frozen prepared " + "manifest, optionally including the final raw checkpoint. Artifact-only quality " + "outcomes require both expected artifact hashes." + ) + ) + parser.add_argument("artifact", type=Path, help="Final confirmation evidence JSON") + parser.add_argument("prepared_manifest", type=Path, help="Frozen prepared-manifest JSON") + parser.add_argument( + "--checkpoint", + type=Path, + help="Final evaluator checkpoint for full raw-array and state verification", + ) + parser.add_argument( + "--expect-artifact-sha256", + help="Optional published SHA256 of the complete artifact file", + ) + parser.add_argument( + "--expect-artifact-evidence-sha256", + help="Optional published SHA256 of the artifact's canonical evidence object", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + report = verify_mbpp_confirmation( + args.artifact, + args.prepared_manifest, + checkpoint_path=args.checkpoint, + expected_artifact_sha256=args.expect_artifact_sha256, + expected_artifact_evidence_sha256=args.expect_artifact_evidence_sha256, + ) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 if report["valid"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/recurquant/__init__.py b/src/recurquant/__init__.py index c886dd1..f7f4704 100644 --- a/src/recurquant/__init__.py +++ b/src/recurquant/__init__.py @@ -1,11 +1,26 @@ -"""RecurQuant research primitives.""" +"""RecurQuant recurrent-state quantization primitives.""" -from .quantization import QuantizationResult, QuantizationSpec, quantize_dequantize +from .evidence import verify_evidence_artifact +from .packed_cache import PackedRecurrentStateCache +from .quantization import ( + PackedQuantizedTensor, + QuantizationResult, + QuantizationSpec, + quantize_dequantize, + quantize_pack, +) +from .qwen35 import create_qwen35_packed_cache, create_qwen35_v02_mixed_cache __all__ = [ + "PackedQuantizedTensor", + "PackedRecurrentStateCache", "QuantizationResult", "QuantizationSpec", + "create_qwen35_packed_cache", + "create_qwen35_v02_mixed_cache", "quantize_dequantize", + "quantize_pack", + "verify_evidence_artifact", ] -__version__ = "0.1.0.dev0" +__version__ = "0.2.0a1" diff --git a/src/recurquant/cli.py b/src/recurquant/cli.py index 926b4bd..c059f22 100644 --- a/src/recurquant/cli.py +++ b/src/recurquant/cli.py @@ -1,13 +1,17 @@ -"""Command-line entry points that do not require a model download.""" +"""Command-line entry points for RecurQuant.""" from __future__ import annotations import argparse import json +from pathlib import Path import torch +from .confirmation import verify_mbpp_confirmation +from .evidence import verify_evidence_artifact from .quantization import QuantizationSpec, quantize_dequantize +from .qwen35_quickstart import add_qwen35_arguments, run_qwen35_quickstart def _demo(args: argparse.Namespace) -> int: @@ -31,6 +35,28 @@ def _demo(args: argparse.Namespace) -> int: return 0 +def _verify_artifact(args: argparse.Namespace) -> int: + report = verify_evidence_artifact( + args.artifact, + expected_file_sha256=args.expect_file_sha256, + expected_canonical_evidence_sha256=args.expect_canonical_evidence_sha256, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["valid"] else 1 + + +def _verify_confirmation(args: argparse.Namespace) -> int: + report = verify_mbpp_confirmation( + args.artifact, + args.prepared_manifest, + checkpoint_path=args.checkpoint, + expected_artifact_sha256=args.expect_artifact_sha256, + expected_artifact_evidence_sha256=args.expect_artifact_evidence_sha256, + ) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 if report["valid"] else 1 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="recurquant", @@ -47,6 +73,33 @@ def build_parser() -> argparse.ArgumentParser: demo.add_argument("--key-dim", type=int, default=16) demo.add_argument("--value-dim", type=int, default=16) demo.set_defaults(handler=_demo) + + qwen35 = subparsers.add_parser( + "qwen35", + help="Run the pinned Qwen3.5 model with the packed recurrent-state cache.", + ) + add_qwen35_arguments(qwen35) + qwen35.set_defaults(handler=run_qwen35_quickstart) + + verify = subparsers.add_parser( + "verify-artifact", + help="Verify a JSON artifact's file and canonical evidence hashes.", + ) + verify.add_argument("artifact", type=Path) + verify.add_argument("--expect-file-sha256") + verify.add_argument("--expect-canonical-evidence-sha256") + verify.set_defaults(handler=_verify_artifact) + + confirmation = subparsers.add_parser( + "verify-confirmation", + help="Verify the frozen v0.2 MBPP confirmation and its quality decision.", + ) + confirmation.add_argument("artifact", type=Path) + confirmation.add_argument("prepared_manifest", type=Path) + confirmation.add_argument("--checkpoint", type=Path) + confirmation.add_argument("--expect-artifact-sha256") + confirmation.add_argument("--expect-artifact-evidence-sha256") + confirmation.set_defaults(handler=_verify_confirmation) return parser diff --git a/src/recurquant/confirmation.py b/src/recurquant/confirmation.py new file mode 100644 index 0000000..d2697f5 --- /dev/null +++ b/src/recurquant/confirmation.py @@ -0,0 +1,1532 @@ +"""Strict offline verification for the frozen RecurQuant v0.2 confirmation run.""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from statistics import fmean +from typing import Any + +import numpy as np +import torch + +from .evaluation import TokenFidelity, fidelity_summary, paired_bootstrap_mean_improvement +from .evidence import canonical_json_bytes + + +@dataclass(frozen=True, slots=True) +class ConfirmationSpec: + """Immutable inputs that identify one preregistered confirmation run.""" + + source_commit: str + prepared_manifest_file_sha256: str + prepared_manifest_evidence_sha256: str + dataset_manifest_sha256: str + token_manifest_sha256: str + calibration_evidence_sha256: str + model_id: str + model_revision: str + task_ids: tuple[int, ...] + prompt_token_count: int + code_token_count: int + combined_token_count: int + gdn_layer_indices: tuple[int, ...] + reference_state_bytes: int + uniform_int4_bytes: int + mixed_int4_int8_bytes: int + uniform_int8_bytes: int + largest_transient_state_bytes: int + preflight_task_id: int + seed: int = 2339 + group_size: int = 128 + bootstrap_samples: int = 10_000 + confidence: float = 0.95 + preflight_tolerance: float = 1e-6 + + +FROZEN_V02_CONFIRMATION = ConfirmationSpec( + source_commit="6bd5bed2b61e192526ba8fdbec8232801cbea843", + prepared_manifest_file_sha256=( + "c6a7d0db6ef7577a66ac19fbbc0be166279488f6a6be432b364bd9eb6833f7b0" + ), + prepared_manifest_evidence_sha256=( + "21a6d18c6a0887b1499d156a3d610d4bfafdd59d3557713485b62038e263b96a" + ), + dataset_manifest_sha256=( + "060aaff7117dc47af6c01253a912f34b6956241c336bbc7216e73bca8624d2d4" + ), + token_manifest_sha256=( + "199a8836489af9bd0af3fec027e85d57df356bd9919492b24015de51d143f525" + ), + calibration_evidence_sha256=( + "7aa8227dd0b19bb7494963c0b590c8ec53cee29d3b696ccd4087c71a5ac461ee" + ), + model_id="Qwen/Qwen3.5-0.8B-Base", + model_revision="dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", + task_ids=tuple(range(11, 511)), + prompt_token_count=68_904, + code_token_count=30_244, + combined_token_count=99_148, + gdn_layer_indices=(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22), + reference_state_bytes=18_874_368, + uniform_int4_bytes=2_433_024, + mixed_int4_int8_bytes=2_564_096, + uniform_int8_bytes=4_792_320, + largest_transient_state_bytes=1_048_576, + preflight_task_id=945, +) + + +class _DuplicateKeyError(ValueError): + pass + + +@dataclass(slots=True) +class _LoadedJSON: + document: Any + file_sha256: str + + +@dataclass(slots=True) +class _Audit: + errors: list[str] + + def check(self, condition: bool, message: str) -> None: + if not condition: + self.errors.append(message) + + def equal(self, actual: Any, expected: Any, label: str) -> None: + if not _deep_equal(actual, expected): + self.errors.append(f"{label} does not match the frozen value") + + +def _deep_equal(left: Any, right: Any) -> bool: + if type(left) is not type(right): + return False + if isinstance(left, dict): + return left.keys() == right.keys() and all( + _deep_equal(left[key], right[key]) for key in left + ) + if isinstance(left, list): + return len(left) == len(right) and all( + _deep_equal(left_value, right_value) + for left_value, right_value in zip(left, right, strict=True) + ) + return bool(left == right) + + +def _pairs_to_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _DuplicateKeyError(f"duplicate object key: {key!r}") + result[key] = value + return result + + +def _reject_non_finite_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant is not permitted: {value}") + + +def _check_finite_tree(value: Any, *, path: str = "$") -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"non-finite number at {path}") + if isinstance(value, dict): + for key, child in value.items(): + _check_finite_tree(child, path=f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _check_finite_tree(child, path=f"{path}[{index}]") + + +def _load_strict_json(path: str | Path, *, label: str, audit: _Audit) -> _LoadedJSON | None: + resolved = Path(path) + try: + raw = resolved.read_bytes() + except OSError as exc: + audit.errors.append(f"could not read {label}: {exc}") + return None + digest = hashlib.sha256(raw).hexdigest() + try: + document = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_pairs_to_object, + parse_constant=_reject_non_finite_constant, + ) + _check_finite_tree(document) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + audit.errors.append(f"{label} is not strict finite UTF-8 JSON: {exc}") + return None + return _LoadedJSON(document=document, file_sha256=digest) + + +def _sha256_canonical(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def _sha256_compact_json(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _is_finite_number(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(float(value)) + except (OverflowError, ValueError): + return False + + +def _valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and value == value.lower() + and all(character in "0123456789abcdef" for character in value) + ) + + +def _mapping(value: Any, *, label: str, audit: _Audit) -> dict[str, Any]: + if not isinstance(value, dict): + audit.errors.append(f"{label} must be an object") + return {} + return value + + +def _list(value: Any, *, label: str, audit: _Audit) -> list[Any]: + if not isinstance(value, list): + audit.errors.append(f"{label} must be an array") + return [] + return value + + +def _verify_envelope( + loaded: _LoadedJSON, + *, + label: str, + artifact_kind: str, + audit: _Audit, +) -> dict[str, Any]: + document = _mapping(loaded.document, label=f"{label} root", audit=audit) + audit.equal(document.get("schema_version"), 1, f"{label} schema_version") + audit.equal(document.get("artifact_kind"), artifact_kind, f"{label} artifact_kind") + evidence = _mapping(document.get("evidence"), label=f"{label} evidence", audit=audit) + computed = _sha256_canonical(evidence) + recorded = document.get("canonical_evidence_sha256") + audit.check(_valid_sha256(recorded), f"{label} canonical evidence SHA256 is invalid") + audit.equal(recorded, computed, f"{label} canonical evidence SHA256") + return evidence + + +def _expected_candidate_bytes(policy: Mapping[str, Any], spec: ConfirmationSpec) -> int: + default_bits = policy.get("default_bits") + upgrade_layer = policy.get("upgrade_layer") + if default_bits == 8 and upgrade_layer is None: + return spec.uniform_int8_bytes + if default_bits == 4 and upgrade_layer is None: + return spec.uniform_int4_bytes + if default_bits == 4 and isinstance(upgrade_layer, int) and not isinstance( + upgrade_layer, bool + ): + return spec.mixed_int4_int8_bytes + raise ValueError(f"unsupported frozen candidate layout: {dict(policy)!r}") + + +def _quartile_membership(token_manifest: Sequence[Mapping[str, Any]]) -> dict[int, str]: + ordered = sorted( + ((int(row["code_tokens"]), int(row["task_id"])) for row in token_manifest), + key=lambda item: (item[0], item[1]), + ) + quotient, remainder = divmod(len(ordered), 4) + membership: dict[int, str] = {} + offset = 0 + for index in range(4): + size = quotient + (1 if index < remainder else 0) + for _, task_id in ordered[offset : offset + size]: + membership[task_id] = f"Q{index + 1}" + offset += size + return membership + + +def _quantiles(values: Sequence[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "minimum": float(array.min()), + "q25": float(np.quantile(array, 0.25)), + "median": float(np.quantile(array, 0.5)), + "q75": float(np.quantile(array, 0.75)), + "maximum": float(array.max()), + } + + +def _candidate_task_aggregates( + rows: Sequence[Mapping[str, Any]], + membership: Mapping[int, str], +) -> tuple[dict[str, Any], dict[str, float], dict[str, Any]]: + task_macro = { + "task_count": len(rows), + "reference_nll": fmean(float(row["reference_nll"]) for row in rows), + "candidate_nll": fmean(float(row["candidate_nll"]) for row in rows), + "delta_nll": fmean(float(row["delta_nll"]) for row in rows), + "mean_kl": fmean(float(row["mean_kl"]) for row in rows), + "top1_agreement": fmean(float(row["top1_agreement"]) for row in rows), + } + distribution = _quantiles([float(row["delta_nll"]) for row in rows]) + by_quartile: dict[str, Any] = {} + for quartile in ("Q1", "Q2", "Q3", "Q4"): + selected = [row for row in rows if membership[int(row["task_id"])] == quartile] + if not selected: + by_quartile[quartile] = { + "task_count": 0, + "minimum_code_tokens": None, + "maximum_code_tokens": None, + "macro_delta_nll": None, + "macro_mean_kl": None, + "macro_top1_agreement": None, + } + continue + by_quartile[quartile] = { + "task_count": len(selected), + "minimum_code_tokens": min(int(row["code_tokens"]) for row in selected), + "maximum_code_tokens": max(int(row["code_tokens"]) for row in selected), + "macro_delta_nll": fmean(float(row["delta_nll"]) for row in selected), + "macro_mean_kl": fmean(float(row["mean_kl"]) for row in selected), + "macro_top1_agreement": fmean( + float(row["top1_agreement"]) for row in selected + ), + } + return task_macro, distribution, by_quartile + + +def _approximately_equal(left: Any, right: Any, *, tolerance: float = 1e-7) -> bool: + if isinstance(left, bool) or isinstance(right, bool): + return type(left) is type(right) and left == right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + try: + return math.isclose( + float(left), float(right), rel_tol=tolerance, abs_tol=tolerance + ) + except (OverflowError, ValueError): + return False + if type(left) is not type(right): + return False + if isinstance(left, dict): + return left.keys() == right.keys() and all( + _approximately_equal(left[key], right[key], tolerance=tolerance) for key in left + ) + if isinstance(left, list): + return len(left) == len(right) and all( + _approximately_equal(a, b, tolerance=tolerance) + for a, b in zip(left, right, strict=True) + ) + return left == right + + +def _check_approximately_equal( + audit: _Audit, + actual: Any, + expected: Any, + label: str, + *, + tolerance: float = 1e-7, +) -> None: + if not _approximately_equal(actual, expected, tolerance=tolerance): + audit.errors.append(f"{label} does not match values recomputed from task data") + + +def _verify_manifest_and_source( + manifest_evidence: dict[str, Any], + artifact_evidence: dict[str, Any], + *, + spec: ConfirmationSpec, + audit: _Audit, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + manifest_claim = _mapping( + manifest_evidence.get("claim_scope"), label="prepared manifest claim_scope", audit=audit + ) + expected_manifest_claim = { + "phase": "confirmation", + "protocol_eligible": True, + "outcomes_computed": False, + "confirmation_touched": True, + } + audit.equal(manifest_claim, expected_manifest_claim, "prepared manifest claim_scope") + + manifest_source = _mapping( + manifest_evidence.get("source"), label="prepared manifest source", audit=audit + ) + audit.equal(manifest_source.get("model_id"), spec.model_id, "prepared manifest model_id") + audit.equal( + manifest_source.get("model_revision"), + spec.model_revision, + "prepared manifest model_revision", + ) + audit.equal( + manifest_source.get("tokenizer_revision"), + spec.model_revision, + "prepared manifest tokenizer_revision", + ) + audit.equal( + manifest_source.get("dataset_manifest_sha256"), + spec.dataset_manifest_sha256, + "prepared manifest dataset manifest SHA256", + ) + audit.equal( + manifest_source.get("token_manifest_sha256"), + spec.token_manifest_sha256, + "prepared manifest token manifest SHA256", + ) + audit.equal( + manifest_source.get("calibration_evidence_sha256"), + spec.calibration_evidence_sha256, + "prepared manifest calibration evidence SHA256", + ) + + dataset_manifest = _mapping( + manifest_source.get("dataset_manifest"), label="prepared dataset manifest", audit=audit + ) + token_manifest = _list( + manifest_source.get("token_manifest"), label="prepared token manifest", audit=audit + ) + audit.equal( + _sha256_compact_json(dataset_manifest), + spec.dataset_manifest_sha256, + "recomputed dataset manifest SHA256", + ) + audit.equal( + _sha256_canonical(token_manifest), + spec.token_manifest_sha256, + "recomputed token manifest SHA256", + ) + + dataset_rows = _list( + dataset_manifest.get("rows"), label="prepared dataset manifest rows", audit=audit + ) + task_ids = list(spec.task_ids) + audit.equal( + [row.get("task_id") for row in dataset_rows if isinstance(row, dict)], + task_ids, + "prepared dataset task IDs", + ) + audit.equal( + [row.get("task_id") for row in token_manifest if isinstance(row, dict)], + task_ids, + "prepared token task IDs", + ) + audit.equal(dataset_manifest.get("phase"), "confirmation", "dataset manifest phase") + audit.equal(dataset_manifest.get("source_split"), "test", "dataset manifest source split") + audit.equal(dataset_manifest.get("row_count"), len(task_ids), "dataset manifest row_count") + + prompt_count = sum( + int(row.get("prompt_tokens", -1)) for row in token_manifest if isinstance(row, dict) + ) + code_count = sum( + int(row.get("code_tokens", -1)) for row in token_manifest if isinstance(row, dict) + ) + combined_count = sum( + int(row.get("total_tokens", -1)) for row in token_manifest if isinstance(row, dict) + ) + audit.equal(prompt_count, spec.prompt_token_count, "prepared prompt token count") + audit.equal(code_count, spec.code_token_count, "prepared code token count") + audit.equal(combined_count, spec.combined_token_count, "prepared combined token count") + for index, row_value in enumerate(token_manifest): + row = _mapping(row_value, label=f"prepared token row {index}", audit=audit) + audit.check( + row.get("total_tokens") == row.get("prompt_tokens", -1) + row.get("code_tokens", -1), + f"prepared token row {index} total_tokens is inconsistent", + ) + for digest_key in ("prompt_token_ids_sha256", "code_token_ids_sha256"): + audit.check( + _valid_sha256(row.get(digest_key)), + f"prepared token row {index} {digest_key} is invalid", + ) + + artifact_claim = _mapping( + artifact_evidence.get("claim_scope"), label="artifact claim_scope", audit=audit + ) + expected_artifact_claim = { + "phase": "confirmation", + "protocol_eligible": True, + "teacher_forced_fidelity_only": True, + "generated_code_executed": False, + "speed_claim_allowed": False, + "whole_model_memory_claim_allowed": False, + "confirmation_touched": True, + } + audit.equal(artifact_claim, expected_artifact_claim, "artifact claim_scope") + artifact_source = _mapping( + artifact_evidence.get("source"), label="artifact source", audit=audit + ) + audit.equal(artifact_source.get("model_id"), spec.model_id, "artifact model_id") + audit.equal( + artifact_source.get("model_revision"), spec.model_revision, "artifact model_revision" + ) + audit.equal( + artifact_source.get("tokenizer_revision"), + spec.model_revision, + "artifact tokenizer_revision", + ) + audit.equal( + artifact_source.get("repository_commit"), spec.source_commit, "artifact source commit" + ) + audit.equal( + artifact_source.get("dataset_manifest_sha256"), + spec.dataset_manifest_sha256, + "artifact dataset manifest SHA256", + ) + audit.equal( + artifact_source.get("token_manifest_sha256"), + spec.token_manifest_sha256, + "artifact token manifest SHA256", + ) + audit.equal( + artifact_source.get("calibration_evidence_sha256"), + spec.calibration_evidence_sha256, + "artifact calibration evidence SHA256", + ) + audit.equal( + artifact_source.get("prepared_manifest_evidence_sha256"), + spec.prepared_manifest_evidence_sha256, + "artifact prepared manifest evidence SHA256", + ) + audit.equal( + artifact_source.get("dataset_manifest"), + dataset_manifest, + "artifact/prepared dataset manifest deep equality", + ) + audit.equal( + artifact_source.get("token_manifest"), + token_manifest, + "artifact/prepared token manifest deep equality", + ) + return dataset_rows, [dict(row) for row in token_manifest if isinstance(row, dict)] + + +def _verify_schedule_and_validity( + artifact_evidence: dict[str, Any], + *, + candidate_names: list[str], + spec: ConfirmationSpec, + audit: _Audit, +) -> None: + environment = _mapping( + artifact_evidence.get("environment"), label="artifact environment", audit=audit + ) + audit.equal( + environment.get("tracked_worktree_clean"), True, "artifact tracked_worktree_clean" + ) + schedule = _mapping(artifact_evidence.get("schedule"), label="artifact schedule", audit=audit) + audit.equal(schedule.get("seed"), spec.seed, "artifact schedule seed") + audit.equal(schedule.get("phase"), "confirmation", "artifact schedule phase") + audit.equal(schedule.get("row_count"), len(spec.task_ids), "artifact schedule row_count") + audit.equal(schedule.get("group_size"), spec.group_size, "artifact schedule group_size") + audit.equal( + schedule.get("candidate_order"), candidate_names, "artifact schedule candidate order" + ) + audit.equal( + schedule.get("scored_first_code_token_from_prefill"), + True, + "artifact first-token scoring flag", + ) + audit.equal( + schedule.get("candidate_generated_tokens_fed_back"), + False, + "artifact candidate feedback flag", + ) + audit.check( + _is_finite_number(schedule.get("elapsed_wall_seconds_not_a_latency_benchmark")) + and float(schedule["elapsed_wall_seconds_not_a_latency_benchmark"]) >= 0, + "artifact elapsed wall time must be a finite non-negative number", + ) + resumed = schedule.get("resumed_task_count") + audit.check( + isinstance(resumed, int) + and not isinstance(resumed, bool) + and 0 <= resumed <= len(spec.task_ids), + "artifact resumed_task_count is outside the frozen task range", + ) + audit.check( + _valid_sha256(schedule.get("run_signature_sha256")), + "artifact run signature SHA256 is invalid", + ) + audit.check( + _valid_sha256(schedule.get("final_checkpoint_sha256")), + "artifact final checkpoint SHA256 is invalid", + ) + + validity = _mapping(artifact_evidence.get("validity"), label="artifact validity", audit=audit) + audit.equal( + validity.get("configured_gdn_layer_indices"), + list(spec.gdn_layer_indices), + "configured Gated DeltaNet layer indices", + ) + audit.equal( + validity.get("reference_recurrent_state_bytes"), + spec.reference_state_bytes, + "reference recurrent-state bytes", + ) + preflight = _mapping( + validity.get("packed_qdq_preflight"), label="packed/QDQ preflight", audit=audit + ) + audit.equal(preflight.get("passed"), True, "packed/QDQ preflight passed flag") + audit.equal( + preflight.get("absolute_tolerance"), + spec.preflight_tolerance, + "packed/QDQ preflight tolerance", + ) + audit.equal(preflight.get("task_id"), spec.preflight_task_id, "packed/QDQ preflight task") + maximums = _mapping( + preflight.get("maximum_absolute_difference_by_candidate"), + label="packed/QDQ preflight candidate differences", + audit=audit, + ) + audit.equal( + set(maximums), set(candidate_names), "packed/QDQ preflight candidate set" + ) + for name, value in maximums.items(): + audit.check( + _is_finite_number(value) and 0 <= float(value) <= spec.preflight_tolerance, + f"packed/QDQ preflight difference for {name} exceeds the frozen tolerance", + ) + + +_PER_TASK_KEYS = { + "task_id", + "code_tokens", + "token_count", + "mean_kl", + "cvar95_kl", + "max_kl", + "top1_agreement", + "reference_nll", + "candidate_nll", + "delta_nll", +} +_TOKEN_WEIGHTED_KEYS = { + "token_count", + "mean_kl", + "cvar95_kl", + "max_kl", + "top1_agreement", + "reference_nll", + "candidate_nll", + "delta_nll", +} + + +def _verify_candidates( + artifact_evidence: dict[str, Any], + *, + candidate_plan: list[dict[str, Any]], + token_manifest: list[dict[str, Any]], + spec: ConfirmationSpec, + audit: _Audit, +) -> dict[str, list[dict[str, Any]]]: + candidates = _mapping( + artifact_evidence.get("candidates"), label="artifact candidates", audit=audit + ) + candidate_names = [str(policy.get("name")) for policy in candidate_plan] + audit.equal(set(candidates), set(candidate_names), "artifact candidate set") + membership = _quartile_membership(token_manifest) + expected_task_ids = list(spec.task_ids) + rows_by_candidate: dict[str, list[dict[str, Any]]] = {} + reference_token_nll: Any = None + + for policy in candidate_plan: + name = str(policy["name"]) + candidate = _mapping(candidates.get(name), label=f"candidate {name}", audit=audit) + audit.equal(candidate.get("policy"), policy, f"candidate {name} policy") + expected_bytes = _expected_candidate_bytes(policy, spec) + storage = _mapping(candidate.get("storage"), label=f"candidate {name} storage", audit=audit) + expected_storage_keys = { + "resident_bytes", + "full_precision_equivalent_bytes", + "resident_compression_ratio", + "largest_transient_state_bytes", + "physical_reduction_realized", + "expected_resident_bytes", + "exact_byte_gate", + } + audit.equal(set(storage), expected_storage_keys, f"candidate {name} storage fields") + audit.equal( + storage.get("resident_bytes"), expected_bytes, f"candidate {name} resident bytes" + ) + audit.equal( + storage.get("expected_resident_bytes"), + expected_bytes, + f"candidate {name} expected resident bytes", + ) + audit.equal( + storage.get("full_precision_equivalent_bytes"), + spec.reference_state_bytes, + f"candidate {name} full-precision equivalent bytes", + ) + audit.equal( + storage.get("largest_transient_state_bytes"), + spec.largest_transient_state_bytes, + f"candidate {name} largest transient state bytes", + ) + audit.equal( + storage.get("physical_reduction_realized"), + True, + f"candidate {name} physical reduction flag", + ) + audit.equal(storage.get("exact_byte_gate"), True, f"candidate {name} exact-byte gate") + audit.check( + _is_finite_number(storage.get("resident_compression_ratio")) + and math.isclose( + float(storage["resident_compression_ratio"]), + spec.reference_state_bytes / expected_bytes, + rel_tol=1e-12, + abs_tol=1e-12, + ), + f"candidate {name} compression ratio is inconsistent with resident bytes", + ) + + task_rows_value = _list( + candidate.get("per_task"), label=f"candidate {name} per_task", audit=audit + ) + task_rows = [dict(row) for row in task_rows_value if isinstance(row, dict)] + audit.equal(len(task_rows), len(spec.task_ids), f"candidate {name} task count") + audit.equal( + [row.get("task_id") for row in task_rows], + expected_task_ids, + f"candidate {name} task IDs", + ) + for index, (row, token_row) in enumerate( + zip(task_rows, token_manifest, strict=False) + ): + audit.equal(set(row), _PER_TASK_KEYS, f"candidate {name} task {index} fields") + audit.equal( + row.get("code_tokens"), + token_row.get("code_tokens"), + f"candidate {name} task {index} code token count", + ) + audit.equal( + row.get("token_count"), + token_row.get("code_tokens"), + f"candidate {name} task {index} scored token count", + ) + for metric in _PER_TASK_KEYS - {"task_id", "code_tokens", "token_count"}: + audit.check( + _is_finite_number(row.get(metric)), + f"candidate {name} task {index} {metric} is not finite numeric data", + ) + if all( + _is_finite_number(row.get(key)) + for key in ("candidate_nll", "reference_nll", "delta_nll") + ): + audit.check( + math.isclose( + float(row["candidate_nll"]) - float(row["reference_nll"]), + float(row["delta_nll"]), + rel_tol=1e-6, + abs_tol=1e-6, + ), + f"candidate {name} task {index} delta_nll is inconsistent", + ) + rows_by_candidate[name] = task_rows + + token_weighted = _mapping( + candidate.get("token_weighted"), + label=f"candidate {name} token_weighted", + audit=audit, + ) + audit.equal( + set(token_weighted), _TOKEN_WEIGHTED_KEYS, f"candidate {name} token metrics fields" + ) + audit.equal( + token_weighted.get("token_count"), + spec.code_token_count, + f"candidate {name} scored token count", + ) + for metric in _TOKEN_WEIGHTED_KEYS - {"token_count"}: + audit.check( + _is_finite_number(token_weighted.get(metric)), + f"candidate {name} token metric {metric} is not finite numeric data", + ) + if reference_token_nll is None: + reference_token_nll = token_weighted.get("reference_nll") + else: + audit.equal( + token_weighted.get("reference_nll"), + reference_token_nll, + f"candidate {name} token reference NLL", + ) + if all( + _is_finite_number(token_weighted.get(key)) + for key in ("candidate_nll", "reference_nll", "delta_nll") + ): + audit.check( + math.isclose( + float(token_weighted["candidate_nll"]) + - float(token_weighted["reference_nll"]), + float(token_weighted["delta_nll"]), + rel_tol=1e-6, + abs_tol=1e-6, + ), + f"candidate {name} token-weighted delta_nll is inconsistent", + ) + + if task_rows: + expected_macro, expected_distribution, expected_quartiles = ( + _candidate_task_aggregates(task_rows, membership) + ) + _check_approximately_equal( + audit, + candidate.get("task_macro"), + expected_macro, + f"candidate {name} task macro", + ) + _check_approximately_equal( + audit, + candidate.get("task_delta_nll_distribution"), + expected_distribution, + f"candidate {name} task delta distribution", + ) + _check_approximately_equal( + audit, + candidate.get("by_code_length_quartile"), + expected_quartiles, + f"candidate {name} length quartiles", + ) + return rows_by_candidate + + +def _recompute_contrasts_and_outcome( + artifact_evidence: dict[str, Any], + rows_by_candidate: Mapping[str, Sequence[Mapping[str, Any]]], + *, + token_metrics_by_candidate: Mapping[str, Mapping[str, Any]] | None = None, + all_values_finite_gate: bool | None = None, + exact_resident_bytes_gate: bool | None = None, + spec: ConfirmationSpec, + audit: _Audit, +) -> dict[str, Any] | None: + required = { + "uniform_int4_nearest", + "read_risk_l0_nearest", + "random_l18_nearest", + "random_l4_nearest", + "random_l13_nearest", + } + if not required.issubset(rows_by_candidate): + audit.errors.append("artifact is missing candidates required for frozen contrasts") + return None + if any(len(rows_by_candidate[name]) != len(spec.task_ids) for name in required): + audit.errors.append("frozen contrasts cannot be recomputed from incomplete task rows") + return None + + uniform_delta = [ + float(row["delta_nll"]) for row in rows_by_candidate["uniform_int4_nearest"] + ] + primary_delta = [ + float(row["delta_nll"]) for row in rows_by_candidate["read_risk_l0_nearest"] + ] + random_names = ("random_l18_nearest", "random_l4_nearest", "random_l13_nearest") + random_mean_delta = [ + fmean(float(rows_by_candidate[name][index]["delta_nll"]) for name in random_names) + for index in range(len(spec.task_ids)) + ] + macro_uniform = fmean(uniform_delta) + macro_primary = fmean(primary_delta) + relative_reduction = ( + (macro_uniform - macro_primary) / macro_uniform if macro_uniform > 0 else None + ) + uniform_bootstrap = paired_bootstrap_mean_improvement( + uniform_delta, + primary_delta, + samples=spec.bootstrap_samples, + confidence=spec.confidence, + seed=spec.seed, + ) + random_bootstrap = paired_bootstrap_mean_improvement( + random_mean_delta, + primary_delta, + samples=spec.bootstrap_samples, + confidence=spec.confidence, + seed=spec.seed, + ) + expected_contrasts = { + "primary_vs_uniform_int4": { + "uniform_macro_delta_nll": macro_uniform, + "primary_macro_delta_nll": macro_primary, + "relative_reduction": relative_reduction, + "paired_bootstrap": uniform_bootstrap, + }, + "primary_vs_mean_random_equal_byte": {"paired_bootstrap": random_bootstrap}, + } + _check_approximately_equal( + audit, + artifact_evidence.get("contrasts"), + expected_contrasts, + "artifact contrasts", + tolerance=1e-12, + ) + + candidates = _mapping( + artifact_evidence.get("candidates"), label="artifact candidates", audit=audit + ) + if token_metrics_by_candidate is None: + token_metrics_by_candidate = { + name: _mapping( + _mapping(candidate, label=f"candidate {name}", audit=audit).get( + "token_weighted" + ), + label=f"candidate {name} token metrics", + audit=audit, + ) + for name, candidate in candidates.items() + } + primary_token = _mapping( + token_metrics_by_candidate.get("read_risk_l0_nearest"), + label="primary token metrics", + audit=audit, + ) + uniform_token = _mapping( + token_metrics_by_candidate.get("uniform_int4_nearest"), + label="uniform INT4 token metrics", + audit=audit, + ) + if all_values_finite_gate is None: + all_values_finite_gate = True + if exact_resident_bytes_gate is None: + exact_resident_bytes_gate = all( + _mapping(candidate, label=f"candidate {name}", audit=audit) + .get("storage", {}) + .get("exact_byte_gate") + is True + for name, candidate in candidates.items() + ) + legacy_gates = { + "all_values_finite": all_values_finite_gate, + "exact_resident_bytes": exact_resident_bytes_gate, + "primary_macro_delta_nll_reduction_at_least_15_percent": ( + relative_reduction is not None and relative_reduction >= 0.15 + ), + "equal_byte_bootstrap_interval_above_zero": ( + random_bootstrap["confidence_interval"][0] > 0 + ), + "primary_mean_token_kl_lower_than_uniform_int4": ( + float(primary_token["mean_kl"]) < float(uniform_token["mean_kl"]) + ), + "primary_cvar95_token_kl_lower_than_uniform_int4": ( + float(primary_token["cvar95_kl"]) < float(uniform_token["cvar95_kl"]) + ), + "primary_top1_not_lower_than_uniform_int4": ( + float(primary_token["top1_agreement"]) + >= float(uniform_token["top1_agreement"]) + ), + } + recorded = _mapping( + artifact_evidence.get("continuation_decision"), + label="artifact continuation decision", + audit=audit, + ) + audit.equal(recorded.get("gates"), legacy_gates, "recorded continuation gates") + audit.equal( + recorded.get("all_gates_pass"), + all(legacy_gates.values()), + "recorded all_gates_pass", + ) + audit.equal( + recorded.get("confirmation_permitted"), + False, + "confirmation-phase confirmation_permitted", + ) + + quality_gates = { + **legacy_gates, + "primary_vs_uniform_bootstrap_interval_above_zero": ( + uniform_bootstrap["confidence_interval"][0] > 0 + ), + } + return { + "quality_gates": quality_gates, + "quality_hypothesis_pass": all(quality_gates.values()), + "primary_vs_uniform_int4": expected_contrasts["primary_vs_uniform_int4"], + "primary_vs_mean_random_equal_byte": expected_contrasts[ + "primary_vs_mean_random_equal_byte" + ], + } + + +def _verify_checkpoint( + checkpoint_loaded: _LoadedJSON, + artifact_evidence: dict[str, Any], + manifest_evidence: dict[str, Any], + rows_by_candidate: Mapping[str, Sequence[Mapping[str, Any]]], + token_manifest: Sequence[Mapping[str, Any]], + *, + spec: ConfirmationSpec, + audit: _Audit, +) -> tuple[ + dict[str, list[dict[str, Any]]], + dict[str, dict[str, Any]], + bool, + bool, +]: + checkpoint = _mapping(checkpoint_loaded.document, label="checkpoint root", audit=audit) + audit.equal( + set(checkpoint), + {"schema_version", "run_signature_sha256", "state_sha256", "state"}, + "checkpoint root fields", + ) + audit.equal(checkpoint.get("schema_version"), 1, "checkpoint schema_version") + audit.check( + _valid_sha256(checkpoint.get("run_signature_sha256")), + "checkpoint run signature SHA256 is invalid", + ) + state = _mapping(checkpoint.get("state"), label="checkpoint state", audit=audit) + audit.equal( + set(state), + { + "completed_task_ids", + "global_values", + "per_task", + "storage_by_candidate", + "reference_state_bytes", + "elapsed_wall_seconds", + }, + "checkpoint state fields", + ) + computed_state_sha256 = _sha256_canonical(state) + audit.check( + _valid_sha256(checkpoint.get("state_sha256")), "checkpoint state SHA256 is invalid" + ) + audit.equal( + checkpoint.get("state_sha256"), computed_state_sha256, "checkpoint canonical state SHA256" + ) + + schedule = _mapping(artifact_evidence.get("schedule"), label="artifact schedule", audit=audit) + audit.equal( + checkpoint_loaded.file_sha256, + schedule.get("final_checkpoint_sha256"), + "checkpoint file SHA256", + ) + audit.equal( + checkpoint.get("run_signature_sha256"), + schedule.get("run_signature_sha256"), + "checkpoint/artifact run signature", + ) + + manifest_source = _mapping( + manifest_evidence.get("source"), label="prepared manifest source", audit=audit + ) + artifact_source = _mapping( + artifact_evidence.get("source"), label="artifact source", audit=audit + ) + candidate_plan = _list( + _mapping( + manifest_evidence.get("schedule"), label="prepared manifest schedule", audit=audit + ).get("candidate_plan"), + label="prepared candidate plan", + audit=audit, + ) + signature_evidence = { + "phase": "confirmation", + "model_id": spec.model_id, + "model_revision": spec.model_revision, + "repository_commit": spec.source_commit, + "calibration_evidence_sha256": spec.calibration_evidence_sha256, + "prepared_manifest_evidence_sha256": spec.prepared_manifest_evidence_sha256, + "token_manifest_sha256": spec.token_manifest_sha256, + "group_size": spec.group_size, + "candidate_plan": candidate_plan, + } + del manifest_source, artifact_source + audit.equal( + _sha256_canonical(signature_evidence), + schedule.get("run_signature_sha256"), + "recomputed frozen run signature", + ) + + audit.equal( + state.get("completed_task_ids"), list(spec.task_ids), "checkpoint completed task IDs" + ) + audit.equal( + state.get("reference_state_bytes"), + spec.reference_state_bytes, + "checkpoint reference recurrent-state bytes", + ) + audit.check( + _is_finite_number(state.get("elapsed_wall_seconds")) + and float(state["elapsed_wall_seconds"]) >= 0, + "checkpoint elapsed wall time must be finite and non-negative", + ) + + plan_names = [str(policy["name"]) for policy in candidate_plan] + checkpoint_tasks = _mapping(state.get("per_task"), label="checkpoint per_task", audit=audit) + checkpoint_storage = _mapping( + state.get("storage_by_candidate"), label="checkpoint storage", audit=audit + ) + global_values = _mapping( + state.get("global_values"), label="checkpoint global_values", audit=audit + ) + audit.equal(set(checkpoint_tasks), set(plan_names), "checkpoint per-task candidate set") + audit.equal(set(checkpoint_storage), set(plan_names), "checkpoint storage candidate set") + audit.equal(set(global_values), set(plan_names), "checkpoint global-values candidate set") + + reference_values: list[Any] | None = None + code_counts = [int(row["code_tokens"]) for row in token_manifest] + raw_rows_by_candidate: dict[str, list[dict[str, Any]]] = {} + raw_token_metrics_by_candidate: dict[str, dict[str, Any]] = {} + raw_values_finite = True + checkpoint_exact_bytes = True + for policy in candidate_plan: + name = str(policy["name"]) + artifact_rows = list(rows_by_candidate.get(name, [])) + audit.equal( + checkpoint_tasks.get(name), artifact_rows, f"checkpoint/artifact task rows for {name}" + ) + artifact_storage = _mapping( + _mapping( + _mapping( + artifact_evidence.get("candidates"), label="artifact candidates", audit=audit + ).get(name), + label=f"candidate {name}", + audit=audit, + ).get("storage"), + label=f"candidate {name} storage", + audit=audit, + ) + base_storage = { + key: artifact_storage.get(key) + for key in ( + "resident_bytes", + "full_precision_equivalent_bytes", + "resident_compression_ratio", + "largest_transient_state_bytes", + "physical_reduction_realized", + ) + } + audit.equal( + checkpoint_storage.get(name), base_storage, f"checkpoint/artifact storage for {name}" + ) + checkpoint_storage_value = checkpoint_storage.get(name) + expected_bytes = _expected_candidate_bytes(policy, spec) + storage_is_exact = ( + isinstance(checkpoint_storage_value, dict) + and checkpoint_storage_value.get("resident_bytes") == expected_bytes + and checkpoint_storage_value.get("full_precision_equivalent_bytes") + == spec.reference_state_bytes + and checkpoint_storage_value.get("largest_transient_state_bytes") + == spec.largest_transient_state_bytes + and checkpoint_storage_value.get("physical_reduction_realized") is True + ) + checkpoint_exact_bytes = checkpoint_exact_bytes and storage_is_exact + + values = _mapping( + global_values.get(name), label=f"checkpoint global values for {name}", audit=audit + ) + audit.equal( + set(values), + {"kl", "reference_nll", "candidate_nll", "top1"}, + f"checkpoint global metric fields for {name}", + ) + candidate_values_valid = True + for metric in ("kl", "reference_nll", "candidate_nll", "top1"): + metric_values = _list( + values.get(metric), label=f"checkpoint {name} {metric}", audit=audit + ) + audit.equal( + len(metric_values), spec.code_token_count, f"checkpoint {name} {metric} length" + ) + if metric == "top1": + metric_is_valid = all(isinstance(value, bool) for value in metric_values) + audit.check( + metric_is_valid, + f"checkpoint {name} top1 values must all be booleans", + ) + else: + metric_is_valid = all(_is_finite_number(value) for value in metric_values) + audit.check( + metric_is_valid, + f"checkpoint {name} {metric} contains non-finite or non-numeric data", + ) + raw_values_finite = raw_values_finite and metric_is_valid + candidate_values_valid = candidate_values_valid and metric_is_valid + if reference_values is None: + reference_values = list(values.get("reference_nll", [])) + else: + audit.equal( + values.get("reference_nll"), + reference_values, + f"checkpoint shared reference NLL for {name}", + ) + + if all( + isinstance(values.get(key), list) + and len(values[key]) == spec.code_token_count + for key in ("kl", "reference_nll", "candidate_nll", "top1") + ) and candidate_values_valid: + fidelity = TokenFidelity( + kl=torch.tensor(values["kl"], dtype=torch.float32), + reference_nll=torch.tensor(values["reference_nll"], dtype=torch.float32), + candidate_nll=torch.tensor(values["candidate_nll"], dtype=torch.float32), + top1_agreement=torch.tensor(values["top1"], dtype=torch.bool), + ) + expected_token = fidelity_summary(fidelity) + raw_token_metrics_by_candidate[name] = dict(expected_token) + actual_token = _mapping( + _mapping( + _mapping( + artifact_evidence.get("candidates"), + label="artifact candidates", + audit=audit, + ).get(name), + label=f"candidate {name}", + audit=audit, + ).get("token_weighted"), + label=f"candidate {name} token metrics", + audit=audit, + ) + _check_approximately_equal( + audit, + actual_token, + expected_token, + f"candidate {name} token metrics", + ) + offset = 0 + reconstructed_rows: list[dict[str, Any]] = [] + for task_id, code_count in zip(spec.task_ids, code_counts, strict=True): + task_fidelity = TokenFidelity( + kl=fidelity.kl[offset : offset + code_count], + reference_nll=fidelity.reference_nll[offset : offset + code_count], + candidate_nll=fidelity.candidate_nll[offset : offset + code_count], + top1_agreement=fidelity.top1_agreement[offset : offset + code_count], + ) + reconstructed_rows.append( + { + "task_id": task_id, + "code_tokens": code_count, + **fidelity_summary(task_fidelity), + } + ) + offset += code_count + _check_approximately_equal( + audit, + artifact_rows, + reconstructed_rows, + f"candidate {name} per-task metrics", + ) + raw_rows_by_candidate[name] = reconstructed_rows + + return ( + raw_rows_by_candidate, + raw_token_metrics_by_candidate, + raw_values_finite, + checkpoint_exact_bytes, + ) + + +def verify_mbpp_confirmation( + artifact_path: str | Path, + prepared_manifest_path: str | Path, + *, + checkpoint_path: str | Path | None = None, + expected_artifact_sha256: str | None = None, + expected_artifact_evidence_sha256: str | None = None, + spec: ConfirmationSpec = FROZEN_V02_CONFIRMATION, +) -> dict[str, Any]: + """Verify the frozen confirmation artifact without model or network access. + + Supplying ``checkpoint_path`` additionally verifies the raw token arrays, + canonical checkpoint state, final file hash, and frozen run signature. In + artifact-only mode, both external artifact hashes are required before a + quality outcome is trusted. A valid negative confirmation result remains + valid evidence; only malformed or inconsistent inputs are invalid. + """ + + base_audit = _Audit(errors=[]) + artifact_loaded = _load_strict_json( + artifact_path, label="confirmation artifact", audit=base_audit + ) + manifest_loaded = _load_strict_json( + prepared_manifest_path, label="prepared manifest", audit=base_audit + ) + hashes: dict[str, Any] = { + "artifact_file_sha256": ( + artifact_loaded.file_sha256 if artifact_loaded is not None else None + ), + "artifact_canonical_evidence_sha256": None, + "prepared_manifest_file_sha256": ( + manifest_loaded.file_sha256 if manifest_loaded is not None else None + ), + "prepared_manifest_canonical_evidence_sha256": None, + "checkpoint_file_sha256": None, + "checkpoint_state_sha256": None, + } + expected_file_hash = ( + expected_artifact_sha256.lower() + if isinstance(expected_artifact_sha256, str) + else expected_artifact_sha256 + ) + expected_evidence_hash = ( + expected_artifact_evidence_sha256.lower() + if isinstance(expected_artifact_evidence_sha256, str) + else expected_artifact_evidence_sha256 + ) + both_artifact_hashes_supplied = ( + expected_artifact_sha256 is not None + and expected_artifact_evidence_sha256 is not None + ) + artifact_outcome: dict[str, Any] | None = None + outcome: dict[str, Any] | None = None + artifact_evidence: dict[str, Any] = {} + manifest_evidence: dict[str, Any] = {} + rows_by_candidate: dict[str, list[dict[str, Any]]] = {} + token_manifest: list[dict[str, Any]] = [] + + if artifact_loaded is not None and manifest_loaded is not None: + try: + artifact_evidence = _verify_envelope( + artifact_loaded, + label="confirmation artifact", + artifact_kind="recurquant_mbpp_teacher_forced_evaluation", + audit=base_audit, + ) + manifest_evidence = _verify_envelope( + manifest_loaded, + label="prepared manifest", + artifact_kind="recurquant_mbpp_prepared_manifest", + audit=base_audit, + ) + hashes["artifact_canonical_evidence_sha256"] = _sha256_canonical( + artifact_evidence + ) + hashes["prepared_manifest_canonical_evidence_sha256"] = _sha256_canonical( + manifest_evidence + ) + base_audit.equal( + manifest_loaded.file_sha256, + spec.prepared_manifest_file_sha256, + "prepared manifest file SHA256", + ) + base_audit.equal( + hashes["prepared_manifest_canonical_evidence_sha256"], + spec.prepared_manifest_evidence_sha256, + "prepared manifest frozen evidence SHA256", + ) + if expected_artifact_sha256 is not None: + base_audit.check( + _valid_sha256(expected_file_hash), + "expected artifact file SHA256 is invalid", + ) + base_audit.equal( + artifact_loaded.file_sha256, + expected_file_hash, + "expected artifact file SHA256", + ) + if expected_artifact_evidence_sha256 is not None: + base_audit.check( + _valid_sha256(expected_evidence_hash), + "expected artifact evidence SHA256 is invalid", + ) + base_audit.equal( + hashes["artifact_canonical_evidence_sha256"], + expected_evidence_hash, + "expected artifact evidence SHA256", + ) + + _, token_manifest = _verify_manifest_and_source( + manifest_evidence, + artifact_evidence, + spec=spec, + audit=base_audit, + ) + manifest_schedule = _mapping( + manifest_evidence.get("schedule"), + label="prepared manifest schedule", + audit=base_audit, + ) + candidate_plan_value = _list( + manifest_schedule.get("candidate_plan"), + label="prepared candidate plan", + audit=base_audit, + ) + candidate_plan = [ + dict(policy) for policy in candidate_plan_value if isinstance(policy, dict) + ] + base_audit.equal( + manifest_schedule.get("row_count"), + len(spec.task_ids), + "prepared manifest schedule row_count", + ) + candidate_names = [str(policy.get("name")) for policy in candidate_plan] + base_audit.check( + len(candidate_names) == len(set(candidate_names)) and all(candidate_names), + "prepared candidate names must be unique and non-empty", + ) + _verify_schedule_and_validity( + artifact_evidence, + candidate_names=candidate_names, + spec=spec, + audit=base_audit, + ) + rows_by_candidate = _verify_candidates( + artifact_evidence, + candidate_plan=candidate_plan, + token_manifest=token_manifest, + spec=spec, + audit=base_audit, + ) + if checkpoint_path is None: + artifact_outcome = _recompute_contrasts_and_outcome( + artifact_evidence, + rows_by_candidate, + spec=spec, + audit=base_audit, + ) + except ( + KeyError, + IndexError, + OverflowError, + TypeError, + ValueError, + ZeroDivisionError, + ) as exc: + base_audit.errors.append(f"could not complete artifact verification: {exc}") + + checkpoint_audit = _Audit(errors=[]) + checkpoint_loaded: _LoadedJSON | None = None + if checkpoint_path is not None: + checkpoint_loaded = _load_strict_json( + checkpoint_path, label="final checkpoint", audit=checkpoint_audit + ) + if checkpoint_loaded is not None: + hashes["checkpoint_file_sha256"] = checkpoint_loaded.file_sha256 + checkpoint_document = checkpoint_loaded.document + if isinstance(checkpoint_document, dict) and isinstance( + checkpoint_document.get("state"), dict + ): + hashes["checkpoint_state_sha256"] = _sha256_canonical( + checkpoint_document["state"] + ) + if artifact_evidence and manifest_evidence and token_manifest: + try: + ( + raw_rows_by_candidate, + raw_token_metrics_by_candidate, + raw_values_finite, + checkpoint_exact_bytes, + ) = _verify_checkpoint( + checkpoint_loaded, + artifact_evidence, + manifest_evidence, + rows_by_candidate, + token_manifest, + spec=spec, + audit=checkpoint_audit, + ) + outcome = _recompute_contrasts_and_outcome( + artifact_evidence, + raw_rows_by_candidate, + token_metrics_by_candidate=raw_token_metrics_by_candidate, + all_values_finite_gate=raw_values_finite, + exact_resident_bytes_gate=checkpoint_exact_bytes, + spec=spec, + audit=checkpoint_audit, + ) + except ( + KeyError, + IndexError, + OverflowError, + RuntimeError, + TypeError, + ValueError, + ) as exc: + checkpoint_audit.errors.append( + f"could not complete checkpoint verification: {exc}" + ) + else: + checkpoint_audit.errors.append( + "checkpoint cross-checks require a valid artifact and prepared manifest " + "structure" + ) + + artifact_hashes_matched = ( + both_artifact_hashes_supplied + and artifact_loaded is not None + and expected_file_hash == artifact_loaded.file_sha256 + and expected_evidence_hash == hashes["artifact_canonical_evidence_sha256"] + ) + errors = [*base_audit.errors, *checkpoint_audit.errors] + artifact_manifest_verified = not base_audit.errors + checkpoint_verified = ( + checkpoint_path is not None + and checkpoint_loaded is not None + and not base_audit.errors + and not checkpoint_audit.errors + ) + valid = not errors + if checkpoint_path is None: + if artifact_hashes_matched and artifact_manifest_verified and artifact_outcome is not None: + outcome = { + **artifact_outcome, + "outcome_verified": True, + "verification_basis": "externally_anchored_artifact", + } + else: + outcome = { + "quality_gates": None, + "quality_hypothesis_pass": None, + "primary_vs_uniform_int4": None, + "primary_vs_mean_random_equal_byte": None, + "outcome_verified": False, + "verification_basis": "unanchored_artifact", + } + elif outcome is not None: + outcome = { + **outcome, + "outcome_verified": checkpoint_verified, + "verification_basis": "checkpoint_raw_arrays", + } + outcome_verified = bool(outcome is not None and outcome.get("outcome_verified") is True) + quality_pass = outcome.get("quality_hypothesis_pass") if outcome_verified else None + if not valid: + result = "invalid" + elif not outcome_verified: + result = "unverified" + elif quality_pass is True: + result = "pass" + elif quality_pass is False: + result = "fail" + else: + result = "unverified" + warnings: list[str] = [] + if checkpoint_path is None: + warnings.append( + "checkpoint not supplied; raw token arrays, canonical state, and final checkpoint " + "file hash were not independently verified" + ) + if not both_artifact_hashes_supplied: + warnings.append( + "artifact-only quality outcomes require both externally published artifact " + "file and canonical evidence SHA256 values" + ) + return { + "artifact_path": str(Path(artifact_path)), + "prepared_manifest_path": str(Path(prepared_manifest_path)), + "checkpoint_path": str(Path(checkpoint_path)) if checkpoint_path is not None else None, + "artifact_manifest_verified": artifact_manifest_verified, + "checkpoint_verified": checkpoint_verified, + "outcome_verified": outcome_verified, + "errors": errors, + "hashes": hashes, + "outcome": outcome, + "result": result, + "valid": valid, + "warnings": warnings, + } diff --git a/src/recurquant/evaluation.py b/src/recurquant/evaluation.py new file mode 100644 index 0000000..4c517fe --- /dev/null +++ b/src/recurquant/evaluation.py @@ -0,0 +1,120 @@ +"""Teacher-forced fidelity metrics and paired uncertainty estimates.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from .metrics import tail_mean, token_kl_divergence, top1_agreement + + +@dataclass(frozen=True, slots=True) +class TokenFidelity: + """Per-token comparison between reference and candidate distributions.""" + + kl: torch.Tensor + reference_nll: torch.Tensor + candidate_nll: torch.Tensor + top1_agreement: torch.Tensor + + def to_cpu(self) -> TokenFidelity: + return TokenFidelity( + kl=self.kl.detach().to(torch.float32).cpu(), + reference_nll=self.reference_nll.detach().to(torch.float32).cpu(), + candidate_nll=self.candidate_nll.detach().to(torch.float32).cpu(), + top1_agreement=self.top1_agreement.detach().cpu(), + ) + + +def token_fidelity( + reference_logits: torch.Tensor, + candidate_logits: torch.Tensor, + target_ids: torch.Tensor, +) -> TokenFidelity: + """Compare logits while scoring the supplied ground-truth next tokens.""" + + if reference_logits.shape != candidate_logits.shape: + raise ValueError("reference and candidate logits must have identical shapes") + if target_ids.shape != reference_logits.shape[:-1]: + raise ValueError("target_ids must match every logits dimension except vocabulary") + if target_ids.dtype not in (torch.int32, torch.int64): + raise TypeError("target_ids must use an integer dtype") + + reference_log_probs = torch.log_softmax(reference_logits.to(torch.float32), dim=-1) + candidate_log_probs = torch.log_softmax(candidate_logits.to(torch.float32), dim=-1) + targets = target_ids.to(torch.int64).unsqueeze(-1) + return TokenFidelity( + kl=token_kl_divergence(reference_logits, candidate_logits), + reference_nll=-reference_log_probs.gather(-1, targets).squeeze(-1), + candidate_nll=-candidate_log_probs.gather(-1, targets).squeeze(-1), + top1_agreement=top1_agreement(reference_logits, candidate_logits), + ) + + +def fidelity_summary(fidelity: TokenFidelity) -> dict[str, float | int]: + """Aggregate token metrics without retaining vocabulary-sized logits.""" + + kl = fidelity.kl.detach().to(torch.float32).flatten() + reference_nll = fidelity.reference_nll.detach().to(torch.float32).flatten() + candidate_nll = fidelity.candidate_nll.detach().to(torch.float32).flatten() + agreement = fidelity.top1_agreement.detach().to(torch.float32).flatten() + if not kl.numel(): + raise ValueError("fidelity must contain at least one token") + tensors = (kl, reference_nll, candidate_nll, agreement) + if any(not torch.isfinite(values).all() for values in tensors): + raise ValueError("fidelity metrics must be finite") + + mean_reference_nll = reference_nll.mean() + mean_candidate_nll = candidate_nll.mean() + return { + "token_count": int(kl.numel()), + "mean_kl": float(kl.mean().item()), + "cvar95_kl": float(tail_mean(kl, fraction=0.05).item()), + "max_kl": float(kl.max().item()), + "top1_agreement": float(agreement.mean().item()), + "reference_nll": float(mean_reference_nll.item()), + "candidate_nll": float(mean_candidate_nll.item()), + "delta_nll": float((mean_candidate_nll - mean_reference_nll).item()), + } + + +def paired_bootstrap_mean_improvement( + baseline_values: list[float], + candidate_values: list[float], + *, + samples: int = 10_000, + confidence: float = 0.95, + seed: int = 2339, +) -> dict[str, Any]: + """Bootstrap paired mean improvement, defined as baseline minus candidate.""" + + if len(baseline_values) != len(candidate_values): + raise ValueError("baseline_values and candidate_values must have equal length") + if not baseline_values: + raise ValueError("at least one paired example is required") + if samples <= 0: + raise ValueError("samples must be positive") + if not 0 < confidence < 1: + raise ValueError("confidence must be between 0 and 1") + + baseline = np.asarray(baseline_values, dtype=np.float64) + candidate = np.asarray(candidate_values, dtype=np.float64) + if not np.isfinite(baseline).all() or not np.isfinite(candidate).all(): + raise ValueError("paired values must be finite") + differences = baseline - candidate + generator = np.random.default_rng(seed) + indices = generator.integers(0, differences.size, size=(samples, differences.size)) + sampled_means = differences[indices].mean(axis=1) + alpha = (1.0 - confidence) / 2.0 + lower, upper = np.quantile(sampled_means, [alpha, 1.0 - alpha]) + return { + "paired_examples": int(differences.size), + "mean_improvement": float(differences.mean()), + "confidence": confidence, + "confidence_interval": [float(lower), float(upper)], + "bootstrap_samples": samples, + "seed": seed, + } diff --git a/src/recurquant/evidence.py b/src/recurquant/evidence.py new file mode 100644 index 0000000..079833d --- /dev/null +++ b/src/recurquant/evidence.py @@ -0,0 +1,133 @@ +"""Verification helpers for RecurQuant JSON evidence artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import string +from pathlib import Path +from typing import Any + + +def canonical_json_bytes(value: Any) -> bytes: + """Serialize a value with the canonical format used by research artifacts.""" + + return ( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _reject_non_finite(value: str) -> None: + raise ValueError(f"non-finite JSON constant is not permitted: {value}") + + +def _valid_sha256(value: str) -> bool: + return len(value) == 64 and all(character in string.hexdigits for character in value) + + +def verify_evidence_artifact( + path: str | Path, + *, + expected_file_sha256: str | None = None, + expected_canonical_evidence_sha256: str | None = None, +) -> dict[str, Any]: + """Verify an artifact's file bytes and canonical ``evidence`` payload. + + The function always returns a JSON-serializable report. ``valid`` is false + when the document is malformed, its recorded canonical hash is wrong, or an + optional expected hash does not match. + """ + + artifact_path = Path(path) + errors: list[str] = [] + try: + raw = artifact_path.read_bytes() + except OSError as exc: + return { + "artifact_path": str(artifact_path), + "computed_canonical_evidence_sha256": None, + "errors": [f"could not read artifact: {exc}"], + "file_sha256": None, + "recorded_canonical_evidence_sha256": None, + "valid": False, + } + + file_sha256 = hashlib.sha256(raw).hexdigest() + expected_file = expected_file_sha256.lower() if expected_file_sha256 else None + expected_canonical = ( + expected_canonical_evidence_sha256.lower() + if expected_canonical_evidence_sha256 + else None + ) + if expected_file is not None and not _valid_sha256(expected_file): + errors.append("expected file SHA256 must contain exactly 64 hexadecimal characters") + elif expected_file is not None and file_sha256 != expected_file: + errors.append( + f"file SHA256 mismatch: expected {expected_file}, computed {file_sha256}" + ) + if expected_canonical is not None and not _valid_sha256(expected_canonical): + errors.append( + "expected canonical evidence SHA256 must contain exactly 64 hexadecimal characters" + ) + + try: + document = json.loads(raw.decode("utf-8"), parse_constant=_reject_non_finite) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + errors.append(f"artifact is not strict UTF-8 JSON: {exc}") + return { + "artifact_path": str(artifact_path), + "computed_canonical_evidence_sha256": None, + "errors": errors, + "file_sha256": file_sha256, + "recorded_canonical_evidence_sha256": None, + "valid": False, + } + + if not isinstance(document, dict): + errors.append("artifact root must be a JSON object") + evidence = None + recorded_canonical = None + else: + evidence = document.get("evidence") + recorded_canonical = document.get("canonical_evidence_sha256") + + computed_canonical: str | None = None + if not isinstance(evidence, dict): + errors.append("artifact must contain an evidence object") + else: + computed_canonical = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + + if not isinstance(recorded_canonical, str) or not _valid_sha256(recorded_canonical): + errors.append( + "artifact canonical_evidence_sha256 must contain exactly 64 hexadecimal characters" + ) + recorded_canonical_value = ( + recorded_canonical if isinstance(recorded_canonical, str) else None + ) + else: + recorded_canonical_value = recorded_canonical.lower() + if computed_canonical is not None and recorded_canonical_value != computed_canonical: + errors.append( + "canonical evidence SHA256 mismatch: " + f"recorded {recorded_canonical_value}, computed {computed_canonical}" + ) + + if ( + expected_canonical is not None + and _valid_sha256(expected_canonical) + and computed_canonical is not None + and computed_canonical != expected_canonical + ): + errors.append( + "expected canonical evidence SHA256 mismatch: " + f"expected {expected_canonical}, computed {computed_canonical}" + ) + + return { + "artifact_path": str(artifact_path), + "computed_canonical_evidence_sha256": computed_canonical, + "errors": errors, + "file_sha256": file_sha256, + "recorded_canonical_evidence_sha256": recorded_canonical_value, + "valid": not errors, + } diff --git a/src/recurquant/packed_cache.py b/src/recurquant/packed_cache.py new file mode 100644 index 0000000..c42e1e6 --- /dev/null +++ b/src/recurquant/packed_cache.py @@ -0,0 +1,419 @@ +"""Physically packed persistent-state cache for Transformers linear attention.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping, MutableMapping +from dataclasses import asdict, dataclass + +import torch +from transformers import DynamicCache +from transformers.cache_utils import LinearAttentionLayer + +from .quantization import ( + PackedQuantizedTensor, + QuantizationSpec, + quantize_pack, + scheduled_quantization_spec, +) + + +class _PackedStateView(MutableMapping[int, torch.Tensor | None]): + """Compatibility view that materializes a packed state only when read.""" + + def __init__(self, owner: PackedLinearAttentionLayer) -> None: + self._owner = owner + + def __getitem__(self, key: int) -> torch.Tensor | None: + packed = self._owner.packed_states[key] + return None if packed is None else packed.dequantize() + + def __setitem__(self, key: int, value: torch.Tensor | None) -> None: + if value is None: + self._owner.packed_states[key] = None + self._owner.is_recurrent_states_initialized[key] = False + else: + self._owner._store(value, key) + + def __delitem__(self, key: int) -> None: + self.__setitem__(key, None) + + def __iter__(self) -> Iterator[int]: + return iter(self._owner.packed_states) + + def __len__(self) -> int: + return len(self._owner.packed_states) + + +@dataclass(frozen=True, slots=True) +class PackedCacheUpdateEvidence: + update_index: int + layer_index: int + state_index: int + bits: int + group_size: int + scale_bits: int + rounding: str + source_dtype: str + shape: tuple[int, ...] + baseline_bytes: int + estimated_bytes: int + resident_bytes: int + relative_l2_error: float + mean_squared_error: float + max_absolute_error: float + + def evidence_dict(self) -> dict[str, object]: + return asdict(self) + + +UpdateCallback = Callable[ + [int, int, torch.Tensor, PackedQuantizedTensor, torch.Tensor], + None, +] + + +def _reorder_packed_batch( + packed: PackedQuantizedTensor, + beam_idx: torch.LongTensor, +) -> PackedQuantizedTensor: + """Select packed batch entries without another quantization round trip.""" + + if len(packed.original_shape) <= packed.spec.flatten_last_dims: + raise RuntimeError( + "Cannot reorder a packed state when its quantization groups include the " + "batch dimension. Use flatten_last_dims smaller than the state rank." + ) + + batch_size = packed.original_shape[0] + if packed.rows % batch_size: + raise RuntimeError("packed row metadata is inconsistent with its batch dimension") + rows_per_batch = packed.rows // batch_size + selected = beam_idx.to(packed.payload.device) + selected_batch_size = selected.numel() + + scales = packed.scales.reshape( + batch_size, + rows_per_batch, + packed.groups_per_row, + ) + reordered_scales = scales.index_select(0, selected).reshape( + selected_batch_size * rows_per_batch, + packed.groups_per_row, + ) + + codes_per_batch = rows_per_batch * packed.padded_size + if packed.spec.bits == 8: + codes = packed.payload.reshape(batch_size, codes_per_batch) + reordered_payload = codes.index_select(0, selected).reshape(-1) + elif packed.spec.bits == 4: + payload = packed.payload.reshape(-1) + low = torch.bitwise_and(payload, 0x0F) + high = torch.bitwise_right_shift(payload, 4) + nibbles = torch.empty( + payload.numel() * 2, + dtype=torch.uint8, + device=payload.device, + ) + nibbles[0::2] = low + nibbles[1::2] = high + codes = nibbles[: packed.rows * packed.padded_size].reshape( + batch_size, + codes_per_batch, + ) + reordered_codes = codes.index_select(0, selected).reshape(-1) + if reordered_codes.numel() % 2: + reordered_codes = torch.cat( + [ + reordered_codes, + torch.zeros(1, dtype=torch.uint8, device=reordered_codes.device), + ] + ) + reordered_payload = torch.bitwise_or( + reordered_codes[0::2], + torch.bitwise_left_shift(reordered_codes[1::2], 4), + ) + else: + raise RuntimeError(f"Packed reordering does not support INT{packed.spec.bits}") + + return PackedQuantizedTensor( + payload=reordered_payload.contiguous(), + scales=reordered_scales.contiguous(), + spec=packed.spec, + original_shape=(selected_batch_size, *packed.original_shape[1:]), + original_dtype=packed.original_dtype, + flattened_size=packed.flattened_size, + padded_size=packed.padded_size, + rows=selected_batch_size * rows_per_batch, + groups_per_row=packed.groups_per_row, + ) + + +class PackedLinearAttentionLayer(LinearAttentionLayer): + """Linear-attention cache layer with integer recurrent-state residency.""" + + is_compileable = False + + def __init__( + self, + *, + spec: QuantizationSpec, + layer_index: int, + number_of_states: int = 1, + on_update: UpdateCallback | None = None, + ) -> None: + super().__init__(number_of_states=number_of_states) + if spec.bits not in (4, 8): + raise ValueError("packed cache layers currently support only INT4 and INT8") + self.spec = spec + self.layer_index = layer_index + self.on_update = on_update + self.packed_states: dict[int, PackedQuantizedTensor | None] = dict.fromkeys( + range(number_of_states) + ) + self.recurrent_states = _PackedStateView(self) + self._update_count = 0 + + def _selected_spec(self) -> QuantizationSpec: + return scheduled_quantization_spec( + self.spec, + layer_index=self.layer_index, + layer_update_index=self._update_count, + ) + + def _store(self, recurrent_states: torch.Tensor, state_idx: int) -> torch.Tensor: + if torch.is_grad_enabled() and recurrent_states.requires_grad: + raise RuntimeError( + "Packed recurrent states are inference-only and cannot accept an " + "autograd-tracked tensor. Wrap every prefill and decode forward in " + "torch.inference_mode() or torch.no_grad(); model.eval() alone does " + "not disable autograd." + ) + selected_spec = self._selected_spec() + packed = quantize_pack(recurrent_states, selected_spec) + self.packed_states[state_idx] = packed + self.is_recurrent_states_initialized[state_idx] = True + if self.device is None: + self.dtype = recurrent_states.dtype + self.device = recurrent_states.device + materialized = packed.dequantize() + if self.on_update is not None: + self.on_update( + self.layer_index, + state_idx, + recurrent_states, + packed, + materialized, + ) + self._update_count += 1 + return materialized + + def update_recurrent_state( + self, + recurrent_states: torch.Tensor, + state_idx: int = 0, + **kwargs: object, + ) -> torch.Tensor: + del kwargs + return self._store(recurrent_states, state_idx) + + def resident_recurrent_state_bytes(self) -> int: + return sum( + packed.storage_bytes for packed in self.packed_states.values() if packed is not None + ) + + def full_precision_equivalent_recurrent_state_bytes(self) -> int: + return sum( + packed.elements * torch.empty((), dtype=packed.original_dtype).element_size() + for packed in self.packed_states.values() + if packed is not None + ) + + def largest_materialized_recurrent_state_bytes(self) -> int: + """Return bytes in the largest single dequantized recurrent state. + + This is an exact tensor-size count, not a device allocator peak or a + measurement of the layer's total temporary workspace. + """ + + return max( + ( + packed.elements * torch.empty((), dtype=packed.original_dtype).element_size() + for packed in self.packed_states.values() + if packed is not None + ), + default=0, + ) + + def reset(self) -> None: + for state_idx in range(self.number_of_states): + if self.is_conv_states_initialized[state_idx]: + self.conv_states[state_idx].zero_() + packed = self.packed_states[state_idx] + if packed is not None: + packed.payload.zero_() + packed.scales.zero_() + self.has_previous_state[state_idx] = False + + def reorder_cache(self, beam_idx: torch.LongTensor) -> None: + for state_idx in range(self.number_of_states): + if self.is_conv_states_initialized[state_idx]: + self.conv_states[state_idx] = self.conv_states[state_idx].index_select( + 0, beam_idx.to(self.device) + ) + packed = self.packed_states[state_idx] + if packed is not None: + self.packed_states[state_idx] = _reorder_packed_batch(packed, beam_idx) + + def offload(self) -> None: + for state_idx in range(self.number_of_states): + if self.is_conv_states_initialized[state_idx]: + self.conv_states[state_idx] = self.conv_states[state_idx].to( + "cpu", non_blocking=True + ) + packed = self.packed_states[state_idx] + if packed is not None: + self.packed_states[state_idx] = packed.to("cpu") + + def prefetch(self) -> None: + for state_idx in range(self.number_of_states): + if ( + self.is_conv_states_initialized[state_idx] + and self.conv_states[state_idx].device != self.device + ): + self.conv_states[state_idx] = self.conv_states[state_idx].to( + self.device, non_blocking=True + ) + packed = self.packed_states[state_idx] + if packed is not None and packed.payload.device != self.device: + self.packed_states[state_idx] = packed.to(self.device) + + +class PackedRecurrentStateCache(DynamicCache): + """Drop-in Qwen3.5 cache that keeps Gated DeltaNet states physically packed. + + Whether physical packing reduces resident bytes depends on the state shape and + quantization spec; inspect ``storage_summary()`` after a cache update. The + current PyTorch implementation materializes one recurrent state while its layer + executes. It makes no speed or whole-model peak-memory claim and is not + compatible with ``torch.compile``. + """ + + def __init__( + self, + config: object, + *, + spec: QuantizationSpec, + layer_specs: Mapping[int, QuantizationSpec] | None = None, + record_evidence: bool = False, + ) -> None: + super().__init__(config=config) + self.spec = spec + self.layer_specs = dict(layer_specs or {}) + self.record_evidence = record_evidence + self.update_evidence: list[PackedCacheUpdateEvidence] = [] + self._update_index = 0 + + linear_layer_indices = { + layer_index + for layer_index, layer in enumerate(self.layers) + if isinstance(layer, LinearAttentionLayer) + } + if any(isinstance(index, bool) or not isinstance(index, int) for index in self.layer_specs): + raise TypeError("layer_specs keys must be integer model-layer indices") + invalid_overrides = set(self.layer_specs) - linear_layer_indices + if invalid_overrides: + raise ValueError( + "layer_specs contains non-linear or unknown layer indices: " + f"{sorted(invalid_overrides)}" + ) + + replaced = 0 + for layer_index, layer in enumerate(self.layers): + if isinstance(layer, LinearAttentionLayer): + selected_spec = self.layer_specs.get(layer_index, self.spec) + self.layers[layer_index] = PackedLinearAttentionLayer( + spec=selected_spec, + layer_index=layer_index, + number_of_states=layer.number_of_states, + on_update=self._record_update if record_evidence else None, + ) + replaced += 1 + if replaced == 0: + raise TypeError("config did not create any Transformers linear-attention layers") + + def _record_update( + self, + layer_index: int, + state_index: int, + source: torch.Tensor, + packed: PackedQuantizedTensor, + materialized: torch.Tensor, + ) -> None: + error = materialized.to(torch.float32) - source.detach().to(torch.float32) + source_norm = torch.linalg.vector_norm(source.detach().to(torch.float32)) + relative_l2 = torch.linalg.vector_norm(error) / source_norm.clamp_min(1e-12) + self.update_evidence.append( + PackedCacheUpdateEvidence( + update_index=self._update_index, + layer_index=layer_index, + state_index=state_index, + bits=packed.spec.bits, + group_size=packed.spec.group_size, + scale_bits=packed.spec.scale_bits, + rounding=packed.spec.rounding, + source_dtype=str(source.dtype), + shape=tuple(source.shape), + baseline_bytes=source.numel() * source.element_size(), + estimated_bytes=packed.storage_bytes, + resident_bytes=packed.storage_bytes, + relative_l2_error=float(relative_l2.item()), + mean_squared_error=float(error.square().mean().item()), + max_absolute_error=float(error.abs().max().item()), + ) + ) + self._update_index += 1 + + def packed_layers(self) -> Iterator[tuple[int, PackedLinearAttentionLayer]]: + for layer_index, layer in enumerate(self.layers): + if isinstance(layer, PackedLinearAttentionLayer): + yield layer_index, layer + + def resident_recurrent_state_bytes(self) -> int: + return sum(layer.resident_recurrent_state_bytes() for _, layer in self.packed_layers()) + + def full_precision_equivalent_recurrent_state_bytes(self) -> int: + return sum( + layer.full_precision_equivalent_recurrent_state_bytes() + for _, layer in self.packed_layers() + ) + + def largest_materialized_recurrent_state_bytes(self) -> int: + """Return bytes in the largest single dequantized recurrent state. + + This does not measure allocator peak memory or the full layer workspace. + """ + + return max( + ( + layer.largest_materialized_recurrent_state_bytes() + for _, layer in self.packed_layers() + ), + default=0, + ) + + def storage_summary(self) -> dict[str, int | float | bool]: + resident = self.resident_recurrent_state_bytes() + full_precision_equivalent = self.full_precision_equivalent_recurrent_state_bytes() + return { + "resident_bytes": resident, + "full_precision_equivalent_bytes": full_precision_equivalent, + "largest_materialized_state_bytes": ( + self.largest_materialized_recurrent_state_bytes() + ), + "resident_compression_ratio": ( + full_precision_equivalent / resident if resident else 0.0 + ), + "physical_reduction_realized": ( + resident > 0 and resident < full_precision_equivalent + ), + } diff --git a/src/recurquant/public_data.py b/src/recurquant/public_data.py new file mode 100644 index 0000000..78b387f --- /dev/null +++ b/src/recurquant/public_data.py @@ -0,0 +1,354 @@ +"""Pinned, contamination-aware utilities for the public MBPP evaluation data.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from numbers import Integral +from typing import Any, Final, TypedDict + +MBPP_DATASET_ID: Final = "google-research-datasets/mbpp" +MBPP_CONFIG: Final = "full" +MBPP_REVISION: Final = "4bb6404fdc6cacfda99d4ac4205087b89d32030c" +MBPP_CALIBRATION_SIZE: Final = 128 +MBPP_SELECTION_NAMESPACE: Final = "rq-v0.2" +MBPP_MANIFEST_SCHEMA: Final = "recurquant.mbpp-manifest.v1" +MBPP_FORMATTER_VERSION: Final = "recurquant.mbpp-prompt-code.v1" + +# This is an acknowledgement token, not a secret. Requiring it makes opening the +# untouched public test split an explicit action instead of an accidental code path. +MBPP_CONFIRMATION_TOKEN: Final = "recurquant:unlock-mbpp-confirmation:rq-v0.2" +MBPP_CONFIRMATION_LOCK: Final = MBPP_CONFIRMATION_TOKEN + + +class MBPPPhase(StrEnum): + """Evaluation phases and their permitted source splits.""" + + CALIBRATION = "calibration" + DEVELOPMENT = "development" + CONFIRMATION = "confirmation" + + +MBPP_PHASE_SPLITS: Final[Mapping[MBPPPhase, str]] = { + MBPPPhase.CALIBRATION: "train", + MBPPPhase.DEVELOPMENT: "validation", + MBPPPhase.CONFIRMATION: "test", +} + + +class MBPPRow(TypedDict): + """Canonical fields exposed by the pinned MBPP ``full`` configuration.""" + + task_id: int + text: str + code: str + test_list: list[str] + test_setup_code: str + challenge_test_list: list[str] + + +class MBPPManifestRow(TypedDict): + task_id: int + sha256: str + + +class MBPPManifest(TypedDict): + schema: str + dataset_id: str + config: str + revision: str + phase: str + source_split: str + selection_namespace: str | None + formatter_version: str + row_count: int + rows: list[MBPPManifestRow] + + +class ConfirmationLockedError(PermissionError): + """Raised before the untouched confirmation split can be loaded.""" + + +@dataclass(frozen=True, slots=True) +class MBPPPromptCode: + """A stable prompt/target pair suitable for teacher-forced code scoring.""" + + prompt: str + code: str + + @property + def combined(self) -> str: + return self.prompt + self.code + + @property + def code_start(self) -> int: + """Character offset at which target code starts in :attr:`combined`.""" + + return len(self.prompt) + + +LoadDataset = Callable[..., Iterable[Mapping[str, Any]]] + + +def mbpp_source_split(phase: MBPPPhase | str) -> str: + """Return the official MBPP split assigned to an evaluation phase.""" + + normalized = _coerce_phase(phase) + return MBPP_PHASE_SPLITS[normalized] + + +def load_mbpp_rows( + phase: MBPPPhase | str, + *, + limit: int | None = None, + confirmation_lock: str | None = None, +) -> tuple[MBPPRow, ...]: + """Load a frozen phase population through the simple script-facing API. + + ``limit`` only truncates calibration or development: calibration is always + selected from the protocol's 128 ranked training rows before truncation. + Confirmation requires ``MBPP_CONFIRMATION_LOCK`` and always loads the entire + pinned test split, preventing partial inspection or cherry-picking. + """ + + normalized_phase = _coerce_phase(phase) + if limit is not None and limit < 1: + raise ValueError("limit must be at least 1") + if normalized_phase is MBPPPhase.CONFIRMATION and limit is not None: + raise ValueError("limit is forbidden for the untouched MBPP confirmation phase") + rows = load_mbpp_phase( + normalized_phase, + confirmation_token=confirmation_lock, + ) + return rows if limit is None else rows[:limit] + + +def mbpp_calibration_key(row_or_task_id: Mapping[str, Any] | int) -> str: + """Return the frozen SHA-256 ranking key for calibration selection.""" + + task_id = _task_id(row_or_task_id) + payload = f"{MBPP_SELECTION_NAMESPACE}|{task_id}".encode() + return hashlib.sha256(payload).hexdigest() + + +def select_mbpp_calibration( + rows: Iterable[Mapping[str, Any]], + *, + size: int = MBPP_CALIBRATION_SIZE, +) -> tuple[MBPPRow, ...]: + """Select the rows with the lowest frozen calibration ranking hashes.""" + + if size < 1: + raise ValueError("calibration size must be at least 1") + + canonical_rows = tuple(canonical_mbpp_row(row) for row in rows) + _ensure_unique_task_ids(canonical_rows) + if size > len(canonical_rows): + raise ValueError( + f"calibration size {size} exceeds the available {len(canonical_rows)} rows" + ) + + ranked = sorted( + canonical_rows, + key=lambda row: (mbpp_calibration_key(row), row["task_id"]), + ) + return tuple(ranked[:size]) + + +def canonical_mbpp_row(row: Mapping[str, Any]) -> MBPPRow: + """Validate and normalize one MBPP row for platform-stable hashing.""" + + return { + "task_id": _task_id(row), + "text": _string_field(row, "text"), + "code": _string_field(row, "code"), + "test_list": _string_list_field(row, "test_list"), + "test_setup_code": _string_field(row, "test_setup_code"), + "challenge_test_list": _string_list_field(row, "challenge_test_list"), + } + + +def mbpp_row_sha256(row: Mapping[str, Any]) -> str: + """Hash the canonical content of one MBPP row.""" + + return hashlib.sha256(_canonical_json(canonical_mbpp_row(row))).hexdigest() + + +def build_mbpp_manifest( + rows: Iterable[Mapping[str, Any]], + *, + phase: MBPPPhase | str, +) -> MBPPManifest: + """Build an order-independent manifest containing IDs and canonical row hashes.""" + + normalized_phase = _coerce_phase(phase) + canonical_rows = tuple(canonical_mbpp_row(row) for row in rows) + _ensure_unique_task_ids(canonical_rows) + manifest_rows: list[MBPPManifestRow] = [ + {"task_id": row["task_id"], "sha256": mbpp_row_sha256(row)} + for row in sorted(canonical_rows, key=lambda item: item["task_id"]) + ] + return { + "schema": MBPP_MANIFEST_SCHEMA, + "dataset_id": MBPP_DATASET_ID, + "config": MBPP_CONFIG, + "revision": MBPP_REVISION, + "phase": normalized_phase.value, + "source_split": mbpp_source_split(normalized_phase), + "selection_namespace": ( + MBPP_SELECTION_NAMESPACE if normalized_phase is MBPPPhase.CALIBRATION else None + ), + "formatter_version": MBPP_FORMATTER_VERSION, + "row_count": len(manifest_rows), + "rows": manifest_rows, + } + + +def mbpp_manifest( + rows: Iterable[Mapping[str, Any]], + *, + phase: MBPPPhase | str, +) -> MBPPManifest: + """Return the canonical provenance manifest used by evaluation scripts.""" + + return build_mbpp_manifest(rows, phase=phase) + + +def mbpp_manifest_sha256( + rows: Iterable[Mapping[str, Any]], + *, + phase: MBPPPhase | str, +) -> str: + """Hash a canonical, provenance-bearing MBPP phase manifest.""" + + return hashlib.sha256(_canonical_json(build_mbpp_manifest(rows, phase=phase))).hexdigest() + + +def format_mbpp_prompt_code(row: Mapping[str, Any]) -> MBPPPromptCode: + """Format a Base-model prompt and a separate code target without a chat template.""" + + canonical = canonical_mbpp_row(row) + task = canonical["text"] + tests = "\n".join(canonical["test_list"]) + return MBPPPromptCode( + prompt=( + f"You are an expert Python programmer, and here is your task: {task}\n" + f"Your code should pass these tests:\n\n{tests}\n[BEGIN]\n" + ), + code=canonical["code"], + ) + + +def format_mbpp_example(row: Mapping[str, Any]) -> MBPPPromptCode: + """Return the stable prompt and separate reference-code target for one row.""" + + return format_mbpp_prompt_code(row) + + +def load_mbpp_phase( + phase: MBPPPhase | str, + *, + confirmation_token: str | None = None, + calibration_size: int = MBPP_CALIBRATION_SIZE, + load_dataset_fn: LoadDataset | None = None, +) -> tuple[MBPPRow, ...]: + """Load one pinned MBPP phase, guarding confirmation before any data access. + + ``datasets`` is an optional dependency and is imported only when this function + is called without an injected loader. + """ + + normalized_phase = _coerce_phase(phase) + if normalized_phase is MBPPPhase.CONFIRMATION: + _require_confirmation_token(confirmation_token) + + loader = load_dataset_fn or _load_dataset_lazily + loaded = loader( + MBPP_DATASET_ID, + MBPP_CONFIG, + revision=MBPP_REVISION, + split=mbpp_source_split(normalized_phase), + streaming=True, + ) + if normalized_phase is MBPPPhase.CALIBRATION: + return select_mbpp_calibration(loaded, size=calibration_size) + + canonical_rows = tuple(canonical_mbpp_row(row) for row in loaded) + _ensure_unique_task_ids(canonical_rows) + return canonical_rows + + +def _load_dataset_lazily(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + try: + from datasets import load_dataset + except ModuleNotFoundError as error: + raise RuntimeError( + "MBPP loading requires the optional evaluation dependencies; " + "install recurquant[eval]" + ) from error + return load_dataset(*args, **kwargs) + + +def _require_confirmation_token(token: str | None) -> None: + if token is None or not hmac.compare_digest(token, MBPP_CONFIRMATION_TOKEN): + raise ConfirmationLockedError( + "MBPP confirmation is locked; pass the explicit MBPP_CONFIRMATION_TOKEN " + "only after freezing the candidate and development manifest" + ) + + +def _coerce_phase(phase: MBPPPhase | str) -> MBPPPhase: + try: + return MBPPPhase(phase) + except ValueError as error: + choices = ", ".join(item.value for item in MBPPPhase) + raise ValueError(f"unknown MBPP phase {phase!r}; expected one of: {choices}") from error + + +def _task_id(row_or_task_id: Mapping[str, Any] | int) -> int: + raw = row_or_task_id.get("task_id") if isinstance(row_or_task_id, Mapping) else row_or_task_id + if isinstance(raw, bool) or not isinstance(raw, Integral): + raise TypeError("MBPP task_id must be an integer") + return int(raw) + + +def _string_field(row: Mapping[str, Any], name: str) -> str: + value = row.get(name) + if not isinstance(value, str): + raise TypeError(f"MBPP {name} must be a string") + return value.replace("\r\n", "\n").replace("\r", "\n") + + +def _string_list_field(row: Mapping[str, Any], name: str) -> list[str]: + value = row.get(name) + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise TypeError(f"MBPP {name} must be a sequence of strings") + normalized: list[str] = [] + for item in value: + if not isinstance(item, str): + raise TypeError(f"MBPP {name} must contain only strings") + normalized.append(item.replace("\r\n", "\n").replace("\r", "\n")) + return normalized + + +def _ensure_unique_task_ids(rows: Iterable[MBPPRow]) -> None: + seen: set[int] = set() + for row in rows: + task_id = row["task_id"] + if task_id in seen: + raise ValueError(f"duplicate MBPP task_id in manifest: {task_id}") + seen.add(task_id) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") diff --git a/src/recurquant/quantization.py b/src/recurquant/quantization.py index 55109fb..18129c2 100644 --- a/src/recurquant/quantization.py +++ b/src/recurquant/quantization.py @@ -1,14 +1,14 @@ -"""Deterministic quantize/dequantize emulation for recurrent matrix states. +"""Grouped quantization and physical INT4/INT8 packing for recurrent states. -The functions in this module measure numerical effects and estimated storage. -They do not claim real memory or latency savings: tensors are dequantized back -to their original dtype so an unmodified PyTorch model can consume them. +``quantize_dequantize`` emulates numerical effects in floating-point storage; +``quantize_pack`` creates integer payload and scale tensors with exact resident +bytes. Neither function alone establishes end-to-end memory or latency gains. """ from __future__ import annotations import math -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from typing import Literal import torch @@ -33,8 +33,8 @@ def __post_init__(self) -> None: raise ValueError("bits must be between 2 and 16") if self.group_size <= 0: raise ValueError("group_size must be positive") - if self.scale_bits <= 0: - raise ValueError("scale_bits must be positive") + if self.scale_bits not in (16, 32): + raise ValueError("scale_bits must be 16 or 32") if self.flatten_last_dims <= 0: raise ValueError("flatten_last_dims must be positive") if self.rounding not in ("nearest", "stochastic"): @@ -85,6 +85,74 @@ def evidence_dict(self) -> dict[str, object]: } +@dataclass(frozen=True, slots=True) +class PackedQuantizedTensor: + """Physically packed grouped INT4 or INT8 tensor plus stored scales.""" + + payload: torch.Tensor + scales: torch.Tensor + spec: QuantizationSpec + original_shape: tuple[int, ...] + original_dtype: torch.dtype + flattened_size: int + padded_size: int + rows: int + groups_per_row: int + + @property + def elements(self) -> int: + return math.prod(self.original_shape) + + @property + def storage_bytes(self) -> int: + return self.payload.numel() * self.payload.element_size() + ( + self.scales.numel() * self.scales.element_size() + ) + + def dequantize(self) -> torch.Tensor: + """Materialize the packed tensor in its original dtype and shape.""" + + packed_elements = self.rows * self.padded_size + if self.spec.bits == 8: + codes = self.payload.to(torch.int8).reshape(-1).to(torch.float32) + elif self.spec.bits == 4: + payload = self.payload.reshape(-1) + low = torch.bitwise_and(payload, 0x0F).to(torch.int16) + high = torch.bitwise_right_shift(payload, 4).to(torch.int16) + low = torch.where(low >= 8, low - 16, low) + high = torch.where(high >= 8, high - 16, high) + interleaved = torch.empty( + payload.numel() * 2, + dtype=torch.int16, + device=payload.device, + ) + interleaved[0::2] = low + interleaved[1::2] = high + codes = interleaved[:packed_elements].to(torch.float32) + else: + raise RuntimeError(f"Packed dequantization does not support INT{self.spec.bits}") + + grouped = codes.reshape(self.rows, self.groups_per_row, self.spec.group_size) + restored = grouped * self.scales.to(torch.float32).unsqueeze(-1) + restored = restored.reshape(self.rows, self.padded_size)[:, : self.flattened_size] + return restored.reshape(self.original_shape).to(self.original_dtype) + + def to(self, device: torch.device | str) -> PackedQuantizedTensor: + """Move packed payload and scales without materializing the tensor.""" + + return PackedQuantizedTensor( + payload=self.payload.to(device), + scales=self.scales.to(device), + spec=self.spec, + original_shape=self.original_shape, + original_dtype=self.original_dtype, + flattened_size=self.flattened_size, + padded_size=self.padded_size, + rows=self.rows, + groups_per_row=self.groups_per_row, + ) + + def _stochastic_round(values: torch.Tensor, *, seed: int) -> torch.Tensor: lower = torch.floor(values) probability_up = values - lower @@ -99,15 +167,36 @@ def _stochastic_round(values: torch.Tensor, *, seed: int) -> torch.Tensor: return lower + (draw < probability_up).to(values.dtype) -def quantize_dequantize(tensor: torch.Tensor, spec: QuantizationSpec) -> QuantizationResult: - """Round-trip ``tensor`` through grouped symmetric integer quantization. +def _scale_dtype(scale_bits: int) -> torch.dtype: + if scale_bits == 16: + return torch.float16 + if scale_bits == 32: + return torch.float32 + raise ValueError("scale_bits must be 16 or 32") - The last ``spec.flatten_last_dims`` dimensions are flattened into one row. - All earlier dimensions remain independent rows. For a Gated DeltaNet state - shaped ``[batch, heads, key_dim, value_dim]``, the default therefore uses - independent scales within each batch item and head. - """ +def scheduled_quantization_spec( + spec: QuantizationSpec, + *, + layer_index: int, + layer_update_index: int, +) -> QuantizationSpec: + """Return the reproducible per-layer stream used for stochastic rounding.""" + + if layer_index < 0 or layer_update_index < 0: + raise ValueError("layer and update indices must be non-negative") + if spec.rounding == "nearest": + return spec + return replace( + spec, + seed=spec.seed + layer_index * 1_000_000 + layer_update_index, + ) + + +def _group_tensor( + tensor: torch.Tensor, + spec: QuantizationSpec, +) -> tuple[torch.Tensor, torch.dtype, int, int, int, torch.Tensor]: if not tensor.is_floating_point(): raise TypeError("tensor must use a floating-point dtype") if tensor.ndim < spec.flatten_last_dims: @@ -120,41 +209,76 @@ def quantize_dequantize(tensor: torch.Tensor, spec: QuantizationSpec) -> Quantiz original_dtype = tensor.dtype working = tensor.detach().to(torch.float32) + if not torch.isfinite(working).all().item(): + raise ValueError("tensor must contain only finite values") flattened_size = math.prod(working.shape[-spec.flatten_last_dims :]) rows = working.reshape(-1, flattened_size) - groups_per_row = math.ceil(flattened_size / spec.group_size) padded_size = groups_per_row * spec.group_size if padded_size != flattened_size: rows = torch.nn.functional.pad(rows, (0, padded_size - flattened_size)) grouped = rows.reshape(rows.shape[0], groups_per_row, spec.group_size) + return working, original_dtype, flattened_size, padded_size, groups_per_row, grouped + +def _quantize_groups( + grouped: torch.Tensor, + spec: QuantizationSpec, +) -> tuple[torch.Tensor, torch.Tensor]: qmax = (1 << (spec.bits - 1)) - 1 absmax = grouped.abs().amax(dim=-1, keepdim=True) - scales = torch.where( + ideal_scales = torch.where( absmax > spec.epsilon, absmax / qmax, torch.ones_like(absmax), ) - normalized = grouped / scales + scale_dtype = _scale_dtype(spec.scale_bits) + if scale_dtype == torch.float16: + # FP16 scale storage cannot represent arbitrarily small non-zero scales. + # Clamp to its smallest positive subnormal so division never produces + # infinities while still emulating the bytes that are actually stored. + ideal_scales = ideal_scales.clamp( + min=2.0**-24, + max=torch.finfo(torch.float16).max, + ) + stored_scales = ideal_scales.to(scale_dtype) + working_scales = stored_scales.to(torch.float32) + normalized = grouped / working_scales if spec.rounding == "nearest": quantized = torch.round(normalized) else: quantized = _stochastic_round(normalized, seed=spec.seed) - quantized = quantized.clamp(-qmax, qmax) - dequantized = quantized * scales + return quantized.clamp(-qmax, qmax), stored_scales + + +def quantize_dequantize(tensor: torch.Tensor, spec: QuantizationSpec) -> QuantizationResult: + """Round-trip ``tensor`` through grouped symmetric integer quantization. + + The last ``spec.flatten_last_dims`` dimensions are flattened into one row. + All earlier dimensions remain independent rows. For a Gated DeltaNet state + shaped ``[batch, heads, key_dim, value_dim]``, the default therefore uses + independent scales within each batch item and head. + """ + + working, original_dtype, flattened_size, padded_size, groups_per_row, grouped = ( + _group_tensor(tensor, spec) + ) + quantized, scales = _quantize_groups(grouped, spec) + dequantized = quantized * scales.to(torch.float32) - restored = dequantized.reshape(rows.shape[0], padded_size)[:, :flattened_size] + restored = dequantized.reshape(grouped.shape[0], padded_size)[:, :flattened_size] restored = restored.reshape(working.shape).to(original_dtype) error = restored.to(torch.float32) - working error_l2 = torch.linalg.vector_norm(error) source_l2 = torch.linalg.vector_norm(working) relative_l2 = error_l2 / source_l2.clamp_min(spec.epsilon) - boundary_fraction = (quantized.abs() == qmax).to(torch.float32).mean() + qmax = (1 << (spec.bits - 1)) - 1 + valid_quantized = quantized.reshape(grouped.shape[0], padded_size)[:, :flattened_size] + boundary_fraction = (valid_quantized.abs() == qmax).to(torch.float32).mean() number_of_groups = grouped.shape[0] * grouped.shape[1] - payload_bits = tensor.numel() * spec.bits + payload_bits = number_of_groups * spec.group_size * spec.bits scale_bits = number_of_groups * spec.scale_bits estimated_bytes = math.ceil((payload_bits + scale_bits) / 8) @@ -174,3 +298,37 @@ def quantize_dequantize(tensor: torch.Tensor, spec: QuantizationSpec) -> Quantiz scale_min=float(scales.min().item()), scale_max=float(scales.max().item()), ) + + +def quantize_pack(tensor: torch.Tensor, spec: QuantizationSpec) -> PackedQuantizedTensor: + """Quantize and physically pack a tensor using INT4 or INT8 payloads.""" + + if spec.bits not in (4, 8): + raise ValueError("physical packing currently supports only INT4 and INT8") + working, original_dtype, flattened_size, padded_size, groups_per_row, grouped = ( + _group_tensor(tensor, spec) + ) + quantized, scales = _quantize_groups(grouped, spec) + flat_codes = quantized.to(torch.int16).reshape(-1) + if spec.bits == 8: + payload = flat_codes.to(torch.int8) + else: + nibbles = torch.bitwise_and(flat_codes, 0x0F).to(torch.uint8) + if nibbles.numel() % 2: + nibbles = torch.nn.functional.pad(nibbles, (0, 1)) + payload = torch.bitwise_or( + nibbles[0::2], + torch.bitwise_left_shift(nibbles[1::2], 4), + ) + + return PackedQuantizedTensor( + payload=payload.contiguous(), + scales=scales.squeeze(-1).contiguous(), + spec=spec, + original_shape=tuple(working.shape), + original_dtype=original_dtype, + flattened_size=flattened_size, + padded_size=padded_size, + rows=grouped.shape[0], + groups_per_row=groups_per_row, + ) diff --git a/src/recurquant/qwen35.py b/src/recurquant/qwen35.py new file mode 100644 index 0000000..3ae56b9 --- /dev/null +++ b/src/recurquant/qwen35.py @@ -0,0 +1,248 @@ +"""Validated public factory for the supported Qwen3.5 cache integration.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +import torch +import transformers +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version + +from .packed_cache import PackedRecurrentStateCache +from .quantization import QuantizationSpec, RoundingMode + +if TYPE_CHECKING: + from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig + + Qwen35Source = Qwen3_5ForCausalLM | Qwen3_5TextConfig +else: + Qwen35Source = object + +_SUPPORTED_TRANSFORMERS_REQUIREMENT = "transformers==5.14.1" +_SUPPORTED_TRANSFORMERS_SPEC = SpecifierSet("==5.14.1") +_SUPPORTED_LAYER_TYPES = frozenset({"linear_attention", "full_attention"}) + + +def _validate_transformers_compatibility() -> Version: + installed = transformers.__version__ + try: + parsed = Version(installed) + except InvalidVersion: + raise RuntimeError( + f"RecurQuant could not interpret Transformers version {installed!r}. " + "Install a stable tested release with: " + f"pip install '{_SUPPORTED_TRANSFORMERS_REQUIREMENT}'." + ) from None + if ( + parsed.is_prerelease + or parsed.is_devrelease + or parsed not in _SUPPORTED_TRANSFORMERS_SPEC + ): + raise RuntimeError( + "RecurQuant's Qwen3.5 cache is tested only with " + f"{_SUPPORTED_TRANSFORMERS_REQUIREMENT}; " + f"found transformers=={installed}. Install the tested release before creating " + "the cache." + ) + return parsed + + +def _load_qwen_classes() -> tuple[type[Any], type[Any]]: + try: + from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig + except ImportError as exc: + raise RuntimeError( + "The installed Transformers build does not provide the tested Qwen3.5 " + "classes. Install the supported build with: " + f"pip install '{_SUPPORTED_TRANSFORMERS_REQUIREMENT}'." + ) from exc + return Qwen3_5ForCausalLM, Qwen3_5TextConfig + + +def _mapped_device_label(device: object) -> str: + if isinstance(device, int) and not isinstance(device, bool): + return f"cuda:{device}" + return str(device).lower() + + +def _validate_model_runtime(model: torch.nn.Module, config: object) -> None: + if model.training: + raise ValueError( + "The packed Qwen3.5 cache is inference-only. Call model.eval() before " + "create_qwen35_packed_cache(model, ...), and run every prefill/decode " + "forward inside torch.inference_mode() or torch.no_grad(); eval mode alone " + "does not disable autograd." + ) + + attention_implementation = getattr(config, "_attn_implementation", None) + if attention_implementation != "eager": + raise ValueError( + "This Qwen3.5 model is not configured for the tested eager attention path " + f"(found {attention_implementation!r}). Reload it with " + "attn_implementation='eager' and then create the cache." + ) + + device_map = getattr(model, "hf_device_map", None) + if isinstance(device_map, Mapping) and device_map: + mapped_devices = {_mapped_device_label(device) for device in device_map.values()} + if mapped_devices & {"disk", "meta"}: + raise ValueError( + "RecurQuant does not support disk-offloaded or meta-device Qwen3.5 " + "models. Reload the complete model on one CPU or one CUDA device " + "without disk offload." + ) + if len(mapped_devices) > 1: + raise ValueError( + "RecurQuant does not support sharded or multi-device Qwen3.5 models. " + "Reload the complete model on one CPU or one CUDA device without " + "device_map='auto'." + ) + + actual_devices = {parameter.device for parameter in model.parameters()} + actual_devices.update(buffer.device for buffer in model.buffers()) + if any(device.type == "meta" for device in actual_devices): + raise ValueError( + "RecurQuant cannot use a Qwen3.5 model containing meta tensors. Materialize " + "all weights on one CPU or one CUDA device before creating the cache." + ) + if len(actual_devices) != 1: + rendered_devices = sorted(str(device) for device in actual_devices) or ["none"] + raise ValueError( + "RecurQuant requires the complete Qwen3.5 model on exactly one device; " + f"found {rendered_devices}. Move all parameters and buffers to one CPU or " + "one CUDA device before creating the cache." + ) + + +def _validated_text_config(model_or_config: Qwen35Source) -> Any: + qwen_model_class, qwen_config_class = _load_qwen_classes() + if isinstance(model_or_config, qwen_model_class): + config = model_or_config.config + _validate_model_runtime(model_or_config, config) + elif isinstance(model_or_config, qwen_config_class): + config = model_or_config + else: + actual = type(model_or_config).__name__ + raise TypeError( + "create_qwen35_packed_cache expects a transformers.Qwen3_5ForCausalLM " + f"model or Qwen3_5TextConfig, got {actual}. Load a text-only Qwen3.5 causal " + "language model and pass the evaluated model or its config. Passing only " + "a config validates structure, not the model's runtime placement or " + "attention backend." + ) + + layer_types = getattr(config, "layer_types", None) + number_of_layers = getattr(config, "num_hidden_layers", None) + if not isinstance(layer_types, (list, tuple)) or not isinstance(number_of_layers, int): + raise ValueError( + "The Qwen3.5 config must define layer_types and num_hidden_layers. Reload the " + "model with the tested Transformers version instead of modifying its config." + ) + if len(layer_types) != number_of_layers: + raise ValueError( + "The Qwen3.5 config is inconsistent: len(layer_types) must equal " + f"num_hidden_layers ({len(layer_types)} != {number_of_layers}). Reload the " + "original model config." + ) + + unknown_layer_types = set(layer_types) - _SUPPORTED_LAYER_TYPES + if unknown_layer_types: + raise ValueError( + "The Qwen3.5 config contains unsupported layer types " + f"{sorted(unknown_layer_types)}. RecurQuant currently supports only " + "linear_attention and full_attention layers." + ) + if "linear_attention" not in layer_types: + raise ValueError( + "This Qwen3.5 config has no linear_attention layers, so it has no supported " + "recurrent state to pack. Use a hybrid Qwen3.5 text checkpoint such as " + "Qwen/Qwen3.5-0.8B-Base." + ) + return config + + +def create_qwen35_packed_cache( + model_or_config: Qwen35Source, + *, + bits: int = 4, + group_size: int = 128, + scale_bits: int = 16, + rounding: RoundingMode = "nearest", + seed: int = 2339, + layer_specs: Mapping[int, QuantizationSpec] | None = None, + record_evidence: bool = False, +) -> PackedRecurrentStateCache: + """Create a validated packed recurrent-state cache for Qwen3.5 inference. + + This factory covers text-only ``Qwen3_5ForCausalLM`` models on the tested stable + Transformers release. Passing a model additionally validates evaluation + mode, eager attention, and single-device materialization. Passing only a config + validates its layer structure; it cannot validate the eventual model runtime. + Every model forward must run under ``torch.inference_mode()`` or + ``torch.no_grad()``. ``bits`` may be 4 or 8. Layer-specific specs use model-layer + indices and are checked against the model's linear-attention layers. + + The returned cache physically packs persistent recurrent states. Whether this + uses fewer resident bytes depends on tensor shape, grouping, payload width, and + scale overhead; inspect ``storage_summary()`` and its + ``physical_reduction_realized`` field after an update. The current PyTorch path + materializes one state while that layer executes and makes no latency or + whole-model peak-memory claim. + """ + + _validate_transformers_compatibility() + config = _validated_text_config(model_or_config) + if bits not in (4, 8): + raise ValueError( + f"The packed cache supports bits=4 or bits=8, got bits={bits}. " + "Use quantize_dequantize for numerical experiments at other bit widths." + ) + + spec = QuantizationSpec( + bits=bits, + group_size=group_size, + scale_bits=scale_bits, + rounding=rounding, + seed=seed, + ) + return PackedRecurrentStateCache( + config, + spec=spec, + layer_specs=layer_specs, + record_evidence=record_evidence, + ) + + +def create_qwen35_v02_mixed_cache( + model_or_config: Qwen35Source, + *, + record_evidence: bool = False, +) -> PackedRecurrentStateCache: + """Create the frozen v0.2 mixed-precision Qwen3.5 cache. + + The policy stores model layer 0 at INT8 and every other supported recurrent + layer at INT4, with group size 128, FP16 scales, nearest rounding, and seed + 2339. It is the fixed development policy reported by the repository, not an + automatic selector for other checkpoints. + """ + + return create_qwen35_packed_cache( + model_or_config, + bits=4, + group_size=128, + scale_bits=16, + rounding="nearest", + seed=2339, + layer_specs={ + 0: QuantizationSpec( + bits=8, + group_size=128, + scale_bits=16, + rounding="nearest", + seed=2339, + ) + }, + record_evidence=record_evidence, + ) diff --git a/src/recurquant/qwen35_quickstart.py b/src/recurquant/qwen35_quickstart.py new file mode 100644 index 0000000..d3be398 --- /dev/null +++ b/src/recurquant/qwen35_quickstart.py @@ -0,0 +1,180 @@ +"""Shared installed workflow for the pinned Qwen3.5 RecurQuant quickstart.""" + +from __future__ import annotations + +import argparse +import json +import warnings +from collections.abc import Sequence + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from .qwen35 import create_qwen35_packed_cache, create_qwen35_v02_mixed_cache + +MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" +MODEL_REVISION = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +DEFAULT_PROMPT = "Explain recurrent-state quantization in two sentences." +MIXED_POLICY = "mixed-v02" +UNIFORM_INT4_STRESS_POLICY = "uniform-int4-stress" + + +def add_qwen35_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Add the shared Qwen3.5 quickstart arguments to ``parser``.""" + + parser.add_argument("--prompt", default=DEFAULT_PROMPT) + parser.add_argument("--max-new-tokens", type=int, default=32) + parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto") + parser.add_argument( + "--policy", + choices=(MIXED_POLICY, UNIFORM_INT4_STRESS_POLICY), + default=MIXED_POLICY, + help=( + "mixed-v02 uses the frozen layer-0 INT8/rest INT4 policy; " + "uniform-int4-stress retains uniform INT4 only as a stress baseline" + ), + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Use an already-cached copy of the pinned model and tokenizer.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Print one machine-readable JSON document instead of human-readable output.", + ) + return parser + + +def _device(requested: str) -> torch.device: + if requested == "auto": + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + if requested == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("--device cuda was requested, but CUDA is not available") + return torch.device(requested) + + +def _model_dtype(device: torch.device) -> torch.dtype: + if device.type != "cuda": + return torch.float32 + if torch.cuda.is_bf16_supported(): + return torch.bfloat16 + warnings.warn( + "CUDA BF16 is unavailable; falling back to FP16. RecurQuant's public " + "full-model fidelity evidence has not been validated for FP16 weights.", + RuntimeWarning, + stacklevel=2, + ) + return torch.float16 + + +def _create_cache(model: torch.nn.Module, policy: str): + if policy == MIXED_POLICY: + return create_qwen35_v02_mixed_cache(model) + if policy == UNIFORM_INT4_STRESS_POLICY: + warnings.warn( + "Uniform INT4 is retained only as a stress baseline; the default mixed-v02 " + "policy is the repository's frozen development policy.", + RuntimeWarning, + stacklevel=2, + ) + return create_qwen35_packed_cache(model, bits=4, group_size=128) + raise ValueError(f"unknown Qwen3.5 quickstart policy: {policy!r}") + + +def run_qwen35_quickstart(args: argparse.Namespace) -> int: + """Run the pinned model and print generated text plus recurrent-state bytes.""" + + if args.max_new_tokens <= 0: + raise ValueError("--max-new-tokens must be positive") + + device = _device(args.device) + dtype = _model_dtype(device) + tokenizer = AutoTokenizer.from_pretrained( + MODEL_ID, + revision=MODEL_REVISION, + local_files_only=args.local_files_only, + trust_remote_code=False, + ) + model = AutoModelForCausalLM.from_pretrained( + MODEL_ID, + revision=MODEL_REVISION, + dtype=dtype, + attn_implementation="eager", + low_cpu_mem_usage=True, + use_safetensors=True, + local_files_only=args.local_files_only, + ).to(device) + model.eval() + + cache = _create_cache(model, args.policy) + encoded = tokenizer(args.prompt, return_tensors="pt").to(device) + generated: list[torch.Tensor] = [] + + with torch.inference_mode(): + output = model( + **encoded, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ) + for step in range(args.max_new_tokens): + next_token = output.logits[:, -1, :].argmax(dim=-1, keepdim=True) + generated.append(next_token) + reached_eos = ( + tokenizer.eos_token_id is not None + and bool((next_token == tokenizer.eos_token_id).all().item()) + ) + if reached_eos or step + 1 == args.max_new_tokens: + break + output = model( + input_ids=next_token, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ) + + generated_ids = torch.cat(generated, dim=1) + generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + summary = cache.storage_summary() + if args.json: + print( + json.dumps( + { + "generated_text": generated_text, + "model_id": MODEL_ID, + "model_revision": MODEL_REVISION, + "policy": args.policy, + "storage_summary": summary, + }, + sort_keys=True, + ) + ) + return 0 + + print(f"policy={args.policy}") + print(generated_text) + print(f"resident_recurrent_state_bytes={summary['resident_bytes']}") + print( + "full_precision_equivalent_recurrent_state_bytes=" + f"{summary['full_precision_equivalent_bytes']}" + ) + print( + "largest_materialized_recurrent_state_bytes=" + f"{summary['largest_materialized_state_bytes']}" + ) + print(f"resident_compression_ratio={summary['resident_compression_ratio']:.3f}x") + print(f"physical_reduction_realized={summary['physical_reduction_realized']}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + add_qwen35_arguments(parser) + parsed = parser.parse_args(None if argv is None else list(argv)) + return run_qwen35_quickstart(parsed) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/recurquant/transformers_cache.py b/src/recurquant/transformers_cache.py index 4d2ce7b..8b36f40 100644 --- a/src/recurquant/transformers_cache.py +++ b/src/recurquant/transformers_cache.py @@ -3,12 +3,17 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from dataclasses import asdict, dataclass, replace +from dataclasses import asdict, dataclass import torch from transformers import DynamicCache +from transformers.cache_utils import LinearAttentionLayer -from .quantization import QuantizationSpec, quantize_dequantize +from .quantization import ( + QuantizationSpec, + quantize_dequantize, + scheduled_quantization_spec, +) @dataclass(frozen=True, slots=True) @@ -45,13 +50,37 @@ def __init__( spec: QuantizationSpec, layer_specs: Mapping[int, QuantizationSpec] | None = None, enabled_layers: Iterable[int] | None = None, + record_evidence: bool = True, ) -> None: super().__init__(config=config) self.spec = spec self.layer_specs = dict(layer_specs or {}) self.enabled_layers = None if enabled_layers is None else frozenset(enabled_layers) + self.record_evidence = record_evidence self.update_evidence: list[CacheUpdateEvidence] = [] self._update_index = 0 + self._layer_update_counts: dict[int, int] = {} + + linear_layer_indices = { + layer_index + for layer_index, layer in enumerate(self.layers) + if isinstance(layer, LinearAttentionLayer) + } + if any(isinstance(index, bool) or not isinstance(index, int) for index in self.layer_specs): + raise TypeError("layer_specs keys must be integer model-layer indices") + invalid_overrides = set(self.layer_specs) - linear_layer_indices + if invalid_overrides: + raise ValueError( + "layer_specs contains non-linear or unknown layer indices: " + f"{sorted(invalid_overrides)}" + ) + if self.enabled_layers is not None: + invalid_enabled = set(self.enabled_layers) - linear_layer_indices + if invalid_enabled: + raise ValueError( + "enabled_layers contains non-linear or unknown layer indices: " + f"{sorted(invalid_enabled)}" + ) def update_recurrent_state( self, @@ -64,29 +93,32 @@ def update_recurrent_state( stored = recurrent_states if enabled: selected_spec = self.layer_specs.get(layer_idx, self.spec) - if selected_spec.rounding == "stochastic": - selected_spec = replace( - selected_spec, - seed=selected_spec.seed + self._update_index, - ) + layer_update_index = self._layer_update_counts.get(layer_idx, 0) + selected_spec = scheduled_quantization_spec( + selected_spec, + layer_index=layer_idx, + layer_update_index=layer_update_index, + ) result = quantize_dequantize(recurrent_states, selected_spec) stored = result.tensor - self.update_evidence.append( - CacheUpdateEvidence( - update_index=self._update_index, - layer_index=layer_idx, - state_index=state_idx, - shape=tuple(recurrent_states.shape), - source_dtype=str(recurrent_states.dtype), - bits=selected_spec.bits, - group_size=selected_spec.group_size, - rounding=selected_spec.rounding, - baseline_bytes=result.baseline_bytes, - estimated_bytes=result.estimated_bytes, - relative_l2_error=result.relative_l2_error, - mean_squared_error=result.mean_squared_error, - max_absolute_error=result.max_absolute_error, + if self.record_evidence: + self.update_evidence.append( + CacheUpdateEvidence( + update_index=self._update_index, + layer_index=layer_idx, + state_index=state_idx, + shape=tuple(recurrent_states.shape), + source_dtype=str(recurrent_states.dtype), + bits=selected_spec.bits, + group_size=selected_spec.group_size, + rounding=selected_spec.rounding, + baseline_bytes=result.baseline_bytes, + estimated_bytes=result.estimated_bytes, + relative_l2_error=result.relative_l2_error, + mean_squared_error=result.mean_squared_error, + max_absolute_error=result.max_absolute_error, + ) ) - ) + self._layer_update_counts[layer_idx] = layer_update_index + 1 self._update_index += 1 return super().update_recurrent_state(stored, layer_idx, state_idx, **kwargs) diff --git a/tests/test_confirmation_verifier.py b/tests/test_confirmation_verifier.py new file mode 100644 index 0000000..a7e6ef1 --- /dev/null +++ b/tests/test_confirmation_verifier.py @@ -0,0 +1,761 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from statistics import fmean +from typing import Any + +import numpy as np +import torch + +from recurquant.cli import build_parser +from recurquant.confirmation import ConfirmationSpec, verify_mbpp_confirmation +from recurquant.evaluation import ( + TokenFidelity, + fidelity_summary, + paired_bootstrap_mean_improvement, +) +from recurquant.evidence import canonical_json_bytes + + +@dataclass(frozen=True, slots=True) +class _Fixture: + artifact: Path + manifest: Path + checkpoint: Path + artifact_sha256: str + artifact_evidence_sha256: str + spec: ConfirmationSpec + + +def test_installed_cli_exposes_confirmation_verifier() -> None: + args = build_parser().parse_args( + [ + "verify-confirmation", + "confirmation.json", + "manifest.json", + "--checkpoint", + "checkpoint.json", + ] + ) + + assert args.command == "verify-confirmation" + assert args.artifact == Path("confirmation.json") + assert args.prepared_manifest == Path("manifest.json") + assert args.checkpoint == Path("checkpoint.json") + + +def _compact_sha256(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _quantiles(values: list[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "minimum": float(array.min()), + "q25": float(np.quantile(array, 0.25)), + "median": float(np.quantile(array, 0.5)), + "q75": float(np.quantile(array, 0.75)), + "maximum": float(array.max()), + } + + +def _candidate_artifact( + policy: dict[str, Any], + *, + task_ids: tuple[int, ...], + code_counts: list[int], + task_delta: list[float], + mean_kl: float, + top1_pair: tuple[bool, bool], + resident_bytes: int, + reference_bytes: int, + transient_bytes: int, +) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: + kl: list[float] = [] + reference_nll: list[float] = [] + candidate_nll: list[float] = [] + top1: list[bool] = [] + per_task: list[dict[str, Any]] = [] + for task_id, count, delta in zip(task_ids, code_counts, task_delta, strict=True): + task_kl = [mean_kl] * count + task_reference = [1.0] * count + task_candidate = [1.0 + delta] * count + task_top1 = list(top1_pair[:count]) + fidelity = TokenFidelity( + kl=torch.tensor(task_kl, dtype=torch.float32), + reference_nll=torch.tensor(task_reference, dtype=torch.float32), + candidate_nll=torch.tensor(task_candidate, dtype=torch.float32), + top1_agreement=torch.tensor(task_top1, dtype=torch.bool), + ) + per_task.append( + { + "task_id": task_id, + "code_tokens": count, + **fidelity_summary(fidelity), + } + ) + kl.extend(task_kl) + reference_nll.extend(task_reference) + candidate_nll.extend(task_candidate) + top1.extend(task_top1) + + global_values = { + "kl": kl, + "reference_nll": reference_nll, + "candidate_nll": candidate_nll, + "top1": top1, + } + token_fidelity = TokenFidelity( + kl=torch.tensor(kl, dtype=torch.float32), + reference_nll=torch.tensor(reference_nll, dtype=torch.float32), + candidate_nll=torch.tensor(candidate_nll, dtype=torch.float32), + top1_agreement=torch.tensor(top1, dtype=torch.bool), + ) + task_macro = { + "task_count": len(per_task), + "reference_nll": fmean(float(row["reference_nll"]) for row in per_task), + "candidate_nll": fmean(float(row["candidate_nll"]) for row in per_task), + "delta_nll": fmean(float(row["delta_nll"]) for row in per_task), + "mean_kl": fmean(float(row["mean_kl"]) for row in per_task), + "top1_agreement": fmean(float(row["top1_agreement"]) for row in per_task), + } + quartiles = { + f"Q{index + 1}": { + "task_count": 1, + "minimum_code_tokens": row["code_tokens"], + "maximum_code_tokens": row["code_tokens"], + "macro_delta_nll": row["delta_nll"], + "macro_mean_kl": row["mean_kl"], + "macro_top1_agreement": row["top1_agreement"], + } + for index, row in enumerate(per_task) + } + storage = { + "resident_bytes": resident_bytes, + "full_precision_equivalent_bytes": reference_bytes, + "resident_compression_ratio": reference_bytes / resident_bytes, + "largest_transient_state_bytes": transient_bytes, + "physical_reduction_realized": True, + "expected_resident_bytes": resident_bytes, + "exact_byte_gate": True, + } + candidate = { + "policy": policy, + "storage": storage, + "token_weighted": fidelity_summary(token_fidelity), + "task_macro": task_macro, + "task_delta_nll_distribution": _quantiles( + [float(row["delta_nll"]) for row in per_task] + ), + "by_code_length_quartile": quartiles, + "per_task": per_task, + } + checkpoint_storage = { + key: storage[key] + for key in ( + "resident_bytes", + "full_precision_equivalent_bytes", + "resident_compression_ratio", + "largest_transient_state_bytes", + "physical_reduction_realized", + ) + } + return candidate, global_values, checkpoint_storage + + +def _write_fixture(tmp_path: Path, *, weak_uniform_interval: bool = False) -> _Fixture: + seed = 2339 + model_id = "example/model" + model_revision = "1" * 40 + source_commit = "2" * 40 + calibration_sha256 = "3" * 64 + task_ids = (11, 12, 13, 14) + code_counts = [2, 2, 2, 2] + token_manifest = [ + { + "task_id": task_id, + "prompt_tokens": 5, + "code_tokens": count, + "total_tokens": 5 + count, + "prompt_token_ids_sha256": f"{task_id:064x}", + "code_token_ids_sha256": f"{task_id + 100:064x}", + } + for task_id, count in zip(task_ids, code_counts, strict=True) + ] + dataset_manifest = { + "config": "full", + "dataset_id": "example/mbpp", + "formatter_version": "recurquant.mbpp-prompt-code.v1", + "phase": "confirmation", + "revision": "4" * 40, + "row_count": len(task_ids), + "rows": [ + {"sha256": f"{task_id + 200:064x}", "task_id": task_id} + for task_id in task_ids + ], + "schema": "recurquant.mbpp-manifest.v1", + "selection_namespace": None, + "source_split": "test", + } + dataset_sha256 = _compact_sha256(dataset_manifest) + token_sha256 = hashlib.sha256(canonical_json_bytes(token_manifest)).hexdigest() + candidate_plan = [ + { + "name": "uniform_int4_nearest", + "default_bits": 4, + "upgrade_layer": None, + "rounding": "nearest", + "seed": seed, + }, + { + "name": "uniform_int8_nearest", + "default_bits": 8, + "upgrade_layer": None, + "rounding": "nearest", + "seed": seed, + }, + { + "name": "read_risk_l0_nearest", + "default_bits": 4, + "upgrade_layer": 0, + "rounding": "nearest", + "seed": seed, + }, + { + "name": "random_l18_nearest", + "default_bits": 4, + "upgrade_layer": 18, + "rounding": "nearest", + "seed": seed, + }, + { + "name": "random_l4_nearest", + "default_bits": 4, + "upgrade_layer": 4, + "rounding": "nearest", + "seed": seed, + }, + { + "name": "random_l13_nearest", + "default_bits": 4, + "upgrade_layer": 13, + "rounding": "nearest", + "seed": seed, + }, + ] + manifest_evidence = { + "claim_scope": { + "phase": "confirmation", + "protocol_eligible": True, + "outcomes_computed": False, + "confirmation_touched": True, + }, + "source": { + "model_id": model_id, + "model_revision": model_revision, + "tokenizer_revision": model_revision, + "dataset_manifest": dataset_manifest, + "dataset_manifest_sha256": dataset_sha256, + "token_manifest": token_manifest, + "token_manifest_sha256": token_sha256, + "calibration_evidence_sha256": calibration_sha256, + "repository_commit": "5" * 40, + }, + "environment": {"tracked_worktree_clean": True}, + "schedule": {"row_count": len(task_ids), "candidate_plan": candidate_plan}, + } + manifest_evidence_sha256 = hashlib.sha256( + canonical_json_bytes(manifest_evidence) + ).hexdigest() + manifest_document = { + "schema_version": 1, + "artifact_kind": "recurquant_mbpp_prepared_manifest", + "created_at_utc": "2026-07-22T00:00:00+00:00", + "canonical_evidence_sha256": manifest_evidence_sha256, + "evidence": manifest_evidence, + } + manifest_path = tmp_path / "manifest.json" + manifest_raw = canonical_json_bytes(manifest_document) + manifest_path.write_bytes(manifest_raw) + manifest_file_sha256 = hashlib.sha256(manifest_raw).hexdigest() + + reference_bytes = 1_000 + uniform4_bytes = 100 + mixed_bytes = 110 + uniform8_bytes = 200 + transient_bytes = 50 + primary_delta = [0.0, 0.0, 0.0, 2.8] if weak_uniform_interval else [0.5] * 4 + candidate_inputs = { + "uniform_int4_nearest": ([1.0] * 4, 1.0, (True, False), uniform4_bytes), + "uniform_int8_nearest": ([0.1] * 4, 0.1, (True, True), uniform8_bytes), + "read_risk_l0_nearest": (primary_delta, 0.25, (True, True), mixed_bytes), + "random_l18_nearest": ( + [value + 1.0 for value in primary_delta], + 0.8, + (True, False), + mixed_bytes, + ), + "random_l4_nearest": ( + [value + 1.0 for value in primary_delta], + 0.8, + (True, False), + mixed_bytes, + ), + "random_l13_nearest": ( + [value + 1.0 for value in primary_delta], + 0.8, + (True, False), + mixed_bytes, + ), + } + candidates: dict[str, Any] = {} + global_values: dict[str, Any] = {} + per_task: dict[str, Any] = {} + storage_by_candidate: dict[str, Any] = {} + for policy in candidate_plan: + name = policy["name"] + delta, kl, top1_pair, resident_bytes = candidate_inputs[name] + candidate, values, storage = _candidate_artifact( + policy, + task_ids=task_ids, + code_counts=code_counts, + task_delta=delta, + mean_kl=kl, + top1_pair=top1_pair, + resident_bytes=resident_bytes, + reference_bytes=reference_bytes, + transient_bytes=transient_bytes, + ) + candidates[name] = candidate + global_values[name] = values + per_task[name] = candidate["per_task"] + storage_by_candidate[name] = storage + + uniform_delta = [ + float(row["delta_nll"]) + for row in candidates["uniform_int4_nearest"]["per_task"] + ] + primary_delta_recorded = [ + float(row["delta_nll"]) + for row in candidates["read_risk_l0_nearest"]["per_task"] + ] + random_mean = [ + fmean( + float(candidates[name]["per_task"][index]["delta_nll"]) + for name in ("random_l18_nearest", "random_l4_nearest", "random_l13_nearest") + ) + for index in range(len(task_ids)) + ] + uniform_bootstrap = paired_bootstrap_mean_improvement( + uniform_delta, primary_delta_recorded, samples=10_000, seed=seed + ) + random_bootstrap = paired_bootstrap_mean_improvement( + random_mean, primary_delta_recorded, samples=10_000, seed=seed + ) + macro_uniform = fmean(uniform_delta) + macro_primary = fmean(primary_delta_recorded) + relative_reduction = (macro_uniform - macro_primary) / macro_uniform + contrasts = { + "primary_vs_uniform_int4": { + "uniform_macro_delta_nll": macro_uniform, + "primary_macro_delta_nll": macro_primary, + "relative_reduction": relative_reduction, + "paired_bootstrap": uniform_bootstrap, + }, + "primary_vs_mean_random_equal_byte": {"paired_bootstrap": random_bootstrap}, + } + legacy_gates = { + "all_values_finite": True, + "exact_resident_bytes": True, + "primary_macro_delta_nll_reduction_at_least_15_percent": ( + relative_reduction >= 0.15 + ), + "equal_byte_bootstrap_interval_above_zero": ( + random_bootstrap["confidence_interval"][0] > 0 + ), + "primary_mean_token_kl_lower_than_uniform_int4": True, + "primary_cvar95_token_kl_lower_than_uniform_int4": True, + "primary_top1_not_lower_than_uniform_int4": True, + } + + signature_evidence = { + "phase": "confirmation", + "model_id": model_id, + "model_revision": model_revision, + "repository_commit": source_commit, + "calibration_evidence_sha256": calibration_sha256, + "prepared_manifest_evidence_sha256": manifest_evidence_sha256, + "token_manifest_sha256": token_sha256, + "group_size": 128, + "candidate_plan": candidate_plan, + } + run_signature = hashlib.sha256(canonical_json_bytes(signature_evidence)).hexdigest() + checkpoint_state = { + "completed_task_ids": list(task_ids), + "global_values": global_values, + "per_task": per_task, + "storage_by_candidate": storage_by_candidate, + "reference_state_bytes": reference_bytes, + "elapsed_wall_seconds": 12.5, + } + checkpoint_document = { + "schema_version": 1, + "run_signature_sha256": run_signature, + "state_sha256": hashlib.sha256(canonical_json_bytes(checkpoint_state)).hexdigest(), + "state": checkpoint_state, + } + checkpoint_path = tmp_path / "checkpoint.json" + checkpoint_raw = canonical_json_bytes(checkpoint_document) + checkpoint_path.write_bytes(checkpoint_raw) + checkpoint_file_sha256 = hashlib.sha256(checkpoint_raw).hexdigest() + + artifact_evidence = { + "claim_scope": { + "phase": "confirmation", + "protocol_eligible": True, + "teacher_forced_fidelity_only": True, + "generated_code_executed": False, + "speed_claim_allowed": False, + "whole_model_memory_claim_allowed": False, + "confirmation_touched": True, + }, + "source": { + "model_id": model_id, + "model_revision": model_revision, + "tokenizer_revision": model_revision, + "dataset_manifest": dataset_manifest, + "dataset_manifest_sha256": dataset_sha256, + "token_manifest": token_manifest, + "token_manifest_sha256": token_sha256, + "calibration_artifact_path": "calibration.json", + "calibration_evidence_sha256": calibration_sha256, + "repository_commit": source_commit, + "prepared_manifest_evidence_sha256": manifest_evidence_sha256, + }, + "environment": {"tracked_worktree_clean": True}, + "schedule": { + "seed": seed, + "phase": "confirmation", + "row_count": len(task_ids), + "group_size": 128, + "candidate_order": [policy["name"] for policy in candidate_plan], + "scored_first_code_token_from_prefill": True, + "candidate_generated_tokens_fed_back": False, + "elapsed_wall_seconds_not_a_latency_benchmark": 12.6, + "run_signature_sha256": run_signature, + "checkpoint_path": "checkpoint.json", + "resumed_task_count": 0, + "final_checkpoint_sha256": checkpoint_file_sha256, + }, + "validity": { + "configured_gdn_layer_indices": [0], + "reference_recurrent_state_bytes": reference_bytes, + "packed_qdq_preflight": { + "absolute_tolerance": 1e-6, + "maximum_absolute_difference_by_candidate": { + policy["name"]: 0.0 for policy in candidate_plan + }, + "passed": True, + "task_id": 945, + }, + }, + "candidates": candidates, + "contrasts": contrasts, + "continuation_decision": { + "gates": legacy_gates, + "all_gates_pass": all(legacy_gates.values()), + "confirmation_permitted": False, + }, + } + artifact_evidence_sha256 = hashlib.sha256( + canonical_json_bytes(artifact_evidence) + ).hexdigest() + artifact_document = { + "schema_version": 1, + "artifact_kind": "recurquant_mbpp_teacher_forced_evaluation", + "created_at_utc": "2026-07-22T00:00:01+00:00", + "canonical_evidence_sha256": artifact_evidence_sha256, + "evidence": artifact_evidence, + } + artifact_path = tmp_path / "artifact.json" + artifact_raw = canonical_json_bytes(artifact_document) + artifact_path.write_bytes(artifact_raw) + artifact_sha256 = hashlib.sha256(artifact_raw).hexdigest() + + spec = ConfirmationSpec( + source_commit=source_commit, + prepared_manifest_file_sha256=manifest_file_sha256, + prepared_manifest_evidence_sha256=manifest_evidence_sha256, + dataset_manifest_sha256=dataset_sha256, + token_manifest_sha256=token_sha256, + calibration_evidence_sha256=calibration_sha256, + model_id=model_id, + model_revision=model_revision, + task_ids=task_ids, + prompt_token_count=20, + code_token_count=8, + combined_token_count=28, + gdn_layer_indices=(0,), + reference_state_bytes=reference_bytes, + uniform_int4_bytes=uniform4_bytes, + mixed_int4_int8_bytes=mixed_bytes, + uniform_int8_bytes=uniform8_bytes, + largest_transient_state_bytes=transient_bytes, + preflight_task_id=945, + ) + return _Fixture( + artifact=artifact_path, + manifest=manifest_path, + checkpoint=checkpoint_path, + artifact_sha256=artifact_sha256, + artifact_evidence_sha256=artifact_evidence_sha256, + spec=spec, + ) + + +def test_verifies_artifact_and_manifest_without_large_checkpoint(tmp_path: Path) -> None: + fixture = _write_fixture(tmp_path) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + expected_artifact_sha256=fixture.artifact_sha256.upper(), + expected_artifact_evidence_sha256=fixture.artifact_evidence_sha256.upper(), + spec=fixture.spec, + ) + + assert report["valid"] is True, report["errors"] + assert report["artifact_manifest_verified"] is True + assert report["checkpoint_verified"] is False + assert report["outcome_verified"] is True + assert report["outcome"]["quality_hypothesis_pass"] is True + assert report["outcome"]["verification_basis"] == "externally_anchored_artifact" + assert report["result"] == "pass" + assert report["warnings"] + + +def test_unanchored_and_one_hash_artifact_modes_do_not_claim_an_outcome( + tmp_path: Path, +) -> None: + fixture = _write_fixture(tmp_path) + + unsigned = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + spec=fixture.spec, + ) + one_hash = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + expected_artifact_sha256=fixture.artifact_sha256, + spec=fixture.spec, + ) + + for report in (unsigned, one_hash): + assert report["valid"] is True, report["errors"] + assert report["artifact_manifest_verified"] is True + assert report["outcome_verified"] is False + assert report["outcome"]["quality_hypothesis_pass"] is None + assert report["outcome"]["quality_gates"] is None + assert report["outcome"]["verification_basis"] == "unanchored_artifact" + assert report["result"] == "unverified" + + +def test_verifies_checkpoint_hash_state_signature_and_raw_arrays(tmp_path: Path) -> None: + fixture = _write_fixture(tmp_path) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + checkpoint_path=fixture.checkpoint, + spec=fixture.spec, + ) + + assert report["valid"] is True, report["errors"] + assert report["checkpoint_verified"] is True + assert report["outcome_verified"] is True + assert report["outcome"]["verification_basis"] == "checkpoint_raw_arrays" + assert report["outcome"]["quality_hypothesis_pass"] is True + assert report["hashes"]["checkpoint_file_sha256"] + assert report["hashes"]["checkpoint_state_sha256"] + assert report["warnings"] == [] + + +def test_uniform_interval_is_part_of_quality_decision_but_not_legacy_record( + tmp_path: Path, +) -> None: + fixture = _write_fixture(tmp_path, weak_uniform_interval=True) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + expected_artifact_sha256=fixture.artifact_sha256, + expected_artifact_evidence_sha256=fixture.artifact_evidence_sha256, + spec=fixture.spec, + ) + + assert report["valid"] is True, report["errors"] + assert report["outcome"]["quality_gates"][ + "primary_macro_delta_nll_reduction_at_least_15_percent" + ] is True + assert report["outcome"]["quality_gates"][ + "primary_vs_uniform_bootstrap_interval_above_zero" + ] is False + assert report["outcome"]["quality_hypothesis_pass"] is False + assert report["outcome_verified"] is True + assert report["result"] == "fail" + + +def test_checkpoint_raw_values_control_near_boundary_gate(tmp_path: Path) -> None: + fixture = _write_fixture(tmp_path) + artifact = json.loads(fixture.artifact.read_text(encoding="utf-8")) + checkpoint = json.loads(fixture.checkpoint.read_text(encoding="utf-8")) + primary_name = "read_risk_l0_nearest" + uniform_name = "uniform_int4_nearest" + uniform_boundary = 0.5 + recorded_below_boundary = 0.49999999 + checkpoint_above_boundary = 0.50000003 + checkpoint_float32_summary = float( + torch.tensor([checkpoint_above_boundary], dtype=torch.float32).mean().item() + ) + assert checkpoint_float32_summary > uniform_boundary > recorded_below_boundary + assert checkpoint_float32_summary - recorded_below_boundary < 1e-7 + + for name, recorded_value in ( + (uniform_name, uniform_boundary), + (primary_name, recorded_below_boundary), + ): + candidate = artifact["evidence"]["candidates"][name] + for key in ("mean_kl", "cvar95_kl", "max_kl"): + candidate["token_weighted"][key] = recorded_value + candidate["task_macro"]["mean_kl"] = recorded_value + for row in candidate["per_task"]: + for key in ("mean_kl", "cvar95_kl", "max_kl"): + row[key] = recorded_value + for quartile in candidate["by_code_length_quartile"].values(): + quartile["macro_mean_kl"] = recorded_value + for row in checkpoint["state"]["per_task"][name]: + for key in ("mean_kl", "cvar95_kl", "max_kl"): + row[key] = recorded_value + checkpoint["state"]["global_values"][uniform_name]["kl"] = [ + uniform_boundary + ] * fixture.spec.code_token_count + checkpoint["state"]["global_values"][primary_name]["kl"] = [ + checkpoint_above_boundary + ] * fixture.spec.code_token_count + checkpoint["state_sha256"] = hashlib.sha256( + canonical_json_bytes(checkpoint["state"]) + ).hexdigest() + checkpoint_raw = canonical_json_bytes(checkpoint) + fixture.checkpoint.write_bytes(checkpoint_raw) + artifact["evidence"]["schedule"]["final_checkpoint_sha256"] = hashlib.sha256( + checkpoint_raw + ).hexdigest() + artifact["canonical_evidence_sha256"] = hashlib.sha256( + canonical_json_bytes(artifact["evidence"]) + ).hexdigest() + fixture.artifact.write_bytes(canonical_json_bytes(artifact)) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + checkpoint_path=fixture.checkpoint, + spec=fixture.spec, + ) + + assert report["valid"] is False + assert report["checkpoint_verified"] is False + assert report["outcome_verified"] is False + assert report["result"] == "invalid" + assert report["outcome"]["verification_basis"] == "checkpoint_raw_arrays" + assert report["outcome"]["quality_gates"][ + "primary_mean_token_kl_lower_than_uniform_int4" + ] is False + assert report["outcome"]["quality_hypothesis_pass"] is False + assert any("recorded continuation gates" in error for error in report["errors"]) + + +def test_huge_integer_metric_is_rejected_without_overflowing(tmp_path: Path) -> None: + fixture = _write_fixture(tmp_path) + artifact = json.loads(fixture.artifact.read_text(encoding="utf-8")) + artifact["evidence"]["candidates"]["read_risk_l0_nearest"]["token_weighted"][ + "mean_kl" + ] = 10**400 + artifact["canonical_evidence_sha256"] = hashlib.sha256( + canonical_json_bytes(artifact["evidence"]) + ).hexdigest() + fixture.artifact.write_bytes(canonical_json_bytes(artifact)) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + spec=fixture.spec, + ) + + assert report["valid"] is False + assert report["outcome_verified"] is False + assert report["result"] == "invalid" + assert any( + "not finite numeric data" in error or "could not complete" in error + for error in report["errors"] + ) + + +def test_rejects_non_finite_json_even_when_json_parser_accepts_overflow( + tmp_path: Path, +) -> None: + fixture = _write_fixture(tmp_path) + raw = fixture.artifact.read_text(encoding="utf-8") + fixture.artifact.write_text( + raw.replace('"mean_kl": 0.25', '"mean_kl": 1e400', 1), + encoding="utf-8", + ) + + report = verify_mbpp_confirmation( + fixture.artifact, + fixture.manifest, + spec=fixture.spec, + ) + + assert report["valid"] is False + assert any("non-finite" in error for error in report["errors"]) + + +def test_script_returns_nonzero_json_report_for_tampered_checkpoint(tmp_path: Path) -> None: + fixture = _write_fixture(tmp_path) + checkpoint = json.loads(fixture.checkpoint.read_text(encoding="utf-8")) + checkpoint["state"]["completed_task_ids"][-1] = 999 + fixture.checkpoint.write_text(json.dumps(checkpoint), encoding="utf-8") + script = Path(__file__).parents[1] / "scripts" / "verify_mbpp_confirmation.py" + + completed = subprocess.run( + [ + sys.executable, + str(script), + str(fixture.artifact), + str(fixture.manifest), + "--checkpoint", + str(fixture.checkpoint), + ], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 1 + report = json.loads(completed.stdout) + assert report["valid"] is False + assert report["checkpoint_verified"] is False + assert any("checkpoint" in error for error in report["errors"]) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..0417200 --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest +import torch + +from recurquant.evaluation import ( + fidelity_summary, + paired_bootstrap_mean_improvement, + token_fidelity, +) + + +def test_token_fidelity_scores_ground_truth_targets() -> None: + reference = torch.tensor([[[4.0, 1.0, -1.0], [0.0, 2.0, 1.0]]]) + candidate = torch.tensor([[[3.0, 2.0, -1.0], [0.0, 1.0, 2.0]]]) + targets = torch.tensor([[0, 2]]) + + result = token_fidelity(reference, candidate, targets) + summary = fidelity_summary(result) + + assert result.kl.shape == targets.shape + assert summary["token_count"] == 2 + assert summary["mean_kl"] > 0 + assert summary["candidate_nll"] > 0 + assert summary["top1_agreement"] == pytest.approx(0.5) + + +def test_token_fidelity_rejects_mismatched_targets() -> None: + logits = torch.zeros(1, 2, 3) + with pytest.raises(ValueError, match="target_ids"): + token_fidelity(logits, logits, torch.zeros(1, 1, dtype=torch.long)) + + +def test_paired_bootstrap_is_seeded_and_improvement_is_positive() -> None: + baseline = [2.0, 3.0, 4.0, 5.0] + candidate = [1.0, 2.0, 3.0, 4.0] + + first = paired_bootstrap_mean_improvement(baseline, candidate, samples=200, seed=17) + second = paired_bootstrap_mean_improvement(baseline, candidate, samples=200, seed=17) + + assert first == second + assert first["mean_improvement"] == pytest.approx(1.0) + assert first["confidence_interval"] == pytest.approx([1.0, 1.0]) + + +@pytest.mark.parametrize( + ("baseline", "candidate", "message"), + [ + ([1.0], [], "equal length"), + ([], [], "at least one"), + ([float("nan")], [1.0], "finite"), + ], +) +def test_paired_bootstrap_rejects_invalid_values( + baseline: list[float], candidate: list[float], message: str +) -> None: + with pytest.raises(ValueError, match=message): + paired_bootstrap_mean_improvement(baseline, candidate) diff --git a/tests/test_evidence_verification.py b/tests/test_evidence_verification.py new file mode 100644 index 0000000..9450c51 --- /dev/null +++ b/tests/test_evidence_verification.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from recurquant.cli import main +from recurquant.evidence import canonical_json_bytes, verify_evidence_artifact + + +def _write_artifact(path: Path, evidence: dict[str, object]) -> tuple[str, str]: + canonical_sha256 = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + payload = { + "artifact_kind": "test", + "canonical_evidence_sha256": canonical_sha256, + "evidence": evidence, + "schema_version": 1, + } + raw = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + path.write_bytes(raw) + return hashlib.sha256(raw).hexdigest(), canonical_sha256 + + +def test_verifier_accepts_exact_file_and_canonical_hashes(tmp_path: Path) -> None: + path = tmp_path / "artifact.json" + file_sha256, canonical_sha256 = _write_artifact(path, {"metric": 1.25}) + + report = verify_evidence_artifact( + path, + expected_file_sha256=file_sha256.upper(), + expected_canonical_evidence_sha256=canonical_sha256, + ) + + assert report == { + "artifact_path": str(path), + "computed_canonical_evidence_sha256": canonical_sha256, + "errors": [], + "file_sha256": file_sha256, + "recorded_canonical_evidence_sha256": canonical_sha256, + "valid": True, + } + + +def test_verifier_rejects_tampered_evidence(tmp_path: Path) -> None: + path = tmp_path / "artifact.json" + _write_artifact(path, {"metric": 1.25}) + document = json.loads(path.read_text(encoding="utf-8")) + document["evidence"]["metric"] = 9.0 + path.write_text(json.dumps(document), encoding="utf-8") + + report = verify_evidence_artifact(path) + + assert report["valid"] is False + assert any("canonical evidence SHA256 mismatch" in error for error in report["errors"]) + + +@pytest.mark.parametrize( + ("contents", "expected_error"), + [ + (b"{\"evidence\": {\"metric\": NaN}}", "strict UTF-8 JSON"), + (b"[]", "artifact root must be a JSON object"), + (b"{}", "artifact must contain an evidence object"), + ], +) +def test_verifier_rejects_invalid_artifacts( + tmp_path: Path, + contents: bytes, + expected_error: str, +) -> None: + path = tmp_path / "invalid.json" + path.write_bytes(contents) + + report = verify_evidence_artifact(path) + + assert report["valid"] is False + assert any(expected_error in error for error in report["errors"]) + + +def test_verify_artifact_cli_returns_status_and_json( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + path = tmp_path / "artifact.json" + file_sha256, _ = _write_artifact(path, {"metric": 1.25}) + + assert main(["verify-artifact", str(path), "--expect-file-sha256", file_sha256]) == 0 + accepted = json.loads(capsys.readouterr().out) + assert accepted["valid"] is True + + assert main(["verify-artifact", str(path), "--expect-file-sha256", "0" * 64]) == 1 + rejected = json.loads(capsys.readouterr().out) + assert rejected["valid"] is False + assert any("file SHA256 mismatch" in error for error in rejected["errors"]) diff --git a/tests/test_packed_cache.py b/tests/test_packed_cache.py new file mode 100644 index 0000000..ed07203 --- /dev/null +++ b/tests/test_packed_cache.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import pytest +import torch +from transformers import Qwen3_5ForCausalLM + +from recurquant.packed_cache import PackedLinearAttentionLayer, PackedRecurrentStateCache +from recurquant.quantization import QuantizationSpec +from recurquant.transformers_cache import RecurrentStateQDQCache +from tests.test_transformers_cache import tiny_config + + +def test_packed_cache_runs_prefill_and_decode_with_integer_residency() -> None: + torch.manual_seed(19) + config = tiny_config() + model = Qwen3_5ForCausalLM._from_config(config, attn_implementation="eager").eval() + cache = PackedRecurrentStateCache( + config, + spec=QuantizationSpec(bits=4, group_size=16), + record_evidence=True, + ) + prompt = torch.randint(0, config.vocab_size, (1, 6)) + next_token = torch.randint(0, config.vocab_size, (1, 1)) + + with torch.inference_mode(): + model(prompt, past_key_values=cache, use_cache=True) + output = model(next_token, past_key_values=cache, use_cache=True) + + layer = cache.layers[0] + assert isinstance(layer, PackedLinearAttentionLayer) + packed = layer.packed_states[0] + assert packed is not None + assert packed.payload.dtype == torch.uint8 + assert packed.scales.dtype == torch.float16 + assert layer.recurrent_states[0].shape == (1, 2, 8, 8) + assert output.logits.shape == (1, 1, 128) + assert len(cache.update_evidence) == 2 + + summary = cache.storage_summary() + assert summary["resident_bytes"] == 80 + assert summary["full_precision_equivalent_bytes"] == 512 + assert summary["largest_materialized_state_bytes"] == 512 + assert summary["resident_compression_ratio"] == 6.4 + assert summary["physical_reduction_realized"] is True + + +def test_packed_cache_does_not_collect_unbounded_evidence_by_default() -> None: + config = tiny_config() + cache = PackedRecurrentStateCache( + config, + spec=QuantizationSpec(bits=4, group_size=16), + ) + state = torch.linspace(-1, 1, 128).reshape(1, 2, 8, 8) + + cache.update_recurrent_state(state, layer_idx=0) + + assert cache.update_evidence == [] + assert cache.layers[0].is_compileable is False + + +def test_packed_cache_reports_when_group_padding_outweighs_packing() -> None: + cache = PackedRecurrentStateCache( + tiny_config(), + spec=QuantizationSpec(bits=4, group_size=1024), + ) + + cache.update_recurrent_state(torch.ones(1, 2, 8, 8), layer_idx=0) + + assert cache.storage_summary() == { + "resident_bytes": 1028, + "full_precision_equivalent_bytes": 512, + "largest_materialized_state_bytes": 512, + "resident_compression_ratio": 512 / 1028, + "physical_reduction_realized": False, + } + + +def test_packed_cache_rejects_autograd_tracked_state_but_allows_no_grad() -> None: + cache = PackedRecurrentStateCache( + tiny_config(), + spec=QuantizationSpec(bits=4, group_size=16), + ) + state = torch.linspace(-1, 1, 128, requires_grad=True).reshape(1, 2, 8, 8) + + with pytest.raises(RuntimeError, match=r"torch\.inference_mode\(\).*torch\.no_grad\(\)"): + cache.update_recurrent_state(state, layer_idx=0) + + with torch.no_grad(): + materialized = cache.update_recurrent_state(state, layer_idx=0) + assert materialized.requires_grad is False + + +def test_packed_cache_matches_qdq_cache_numerically() -> None: + torch.manual_seed(23) + config = tiny_config() + model = Qwen3_5ForCausalLM._from_config(config, attn_implementation="eager").eval() + spec = QuantizationSpec(bits=4, group_size=16) + qdq_cache = RecurrentStateQDQCache(config, spec=spec) + packed_cache = PackedRecurrentStateCache(config, spec=spec) + prompt = torch.randint(0, config.vocab_size, (1, 6)) + next_token = torch.randint(0, config.vocab_size, (1, 1)) + + with torch.inference_mode(): + model(prompt, past_key_values=qdq_cache, use_cache=True) + qdq_output = model(next_token, past_key_values=qdq_cache, use_cache=True) + model(prompt, past_key_values=packed_cache, use_cache=True) + packed_output = model(next_token, past_key_values=packed_cache, use_cache=True) + + assert torch.equal(qdq_output.logits, packed_output.logits) + assert torch.equal( + qdq_cache.layers[0].recurrent_states[0], + packed_cache.layers[0].recurrent_states[0], + ) + + +def test_stochastic_packed_cache_matches_qdq_across_multiple_linear_layers() -> None: + torch.manual_seed(29) + config = tiny_config( + ["linear_attention", "linear_attention", "linear_attention", "full_attention"] + ) + model = Qwen3_5ForCausalLM._from_config(config, attn_implementation="eager").eval() + spec = QuantizationSpec(bits=4, group_size=16, rounding="stochastic", seed=31) + qdq_cache = RecurrentStateQDQCache(config, spec=spec) + packed_cache = PackedRecurrentStateCache(config, spec=spec) + prompt = torch.randint(0, config.vocab_size, (1, 6)) + continuation = torch.randint(0, config.vocab_size, (1, 2)) + + with torch.inference_mode(): + model(prompt, past_key_values=qdq_cache, use_cache=True) + qdq_output = model(continuation, past_key_values=qdq_cache, use_cache=True) + model(prompt, past_key_values=packed_cache, use_cache=True) + packed_output = model(continuation, past_key_values=packed_cache, use_cache=True) + + assert torch.equal(qdq_output.logits, packed_output.logits) + for layer_index in range(3): + assert torch.equal( + qdq_cache.layers[layer_index].recurrent_states[0], + packed_cache.layers[layer_index].recurrent_states[0], + ) + + +def test_layer_override_changes_physical_payload_width() -> None: + config = tiny_config() + cache = PackedRecurrentStateCache( + config, + spec=QuantizationSpec(bits=4, group_size=16), + layer_specs={0: QuantizationSpec(bits=8, group_size=16)}, + ) + state = torch.linspace(-1, 1, 128).reshape(1, 2, 8, 8) + + cache.update_recurrent_state(state, layer_idx=0) + + layer = cache.layers[0] + assert isinstance(layer, PackedLinearAttentionLayer) + packed = layer.packed_states[0] + assert packed is not None + assert packed.payload.dtype == torch.int8 + assert packed.storage_bytes == 144 + + +def test_int8_bfloat16_reorder_preserves_dequantized_state_exactly() -> None: + state = ( + torch.randn((3, 2, 8, 8), generator=torch.Generator().manual_seed(9)) * 1e-4 + ).to(torch.bfloat16) + layer = PackedLinearAttentionLayer( + spec=QuantizationSpec(bits=8, group_size=16), + layer_index=0, + ) + before = layer.update_recurrent_state(state).clone() + beam_idx = torch.tensor([2, 0, 1]) + + layer.reorder_cache(beam_idx) + + assert torch.equal(layer.recurrent_states[0], before.index_select(0, beam_idx)) + + +def test_int4_odd_nibble_reorder_preserves_dequantized_state_exactly() -> None: + state = ( + torch.randn((3, 1, 3, 3), generator=torch.Generator().manual_seed(17)) * 1e-3 + ).to(torch.bfloat16) + layer = PackedLinearAttentionLayer( + spec=QuantizationSpec(bits=4, group_size=3), + layer_index=0, + ) + before = layer.update_recurrent_state(state).clone() + beam_idx = torch.tensor([2, 2, 0]) + + layer.reorder_cache(beam_idx) + + assert torch.equal(layer.recurrent_states[0], before.index_select(0, beam_idx)) + + +def test_packed_cache_rejects_non_linear_and_unknown_overrides() -> None: + config = tiny_config() + + for layer_index in (1, 99): + with pytest.raises(ValueError, match="non-linear or unknown"): + PackedRecurrentStateCache( + config, + spec=QuantizationSpec(bits=4, group_size=16), + layer_specs={layer_index: QuantizationSpec(bits=8, group_size=16)}, + ) diff --git a/tests/test_public_data.py b/tests/test_public_data.py new file mode 100644 index 0000000..c326394 --- /dev/null +++ b/tests/test_public_data.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Iterable, Mapping +from typing import Any + +import pytest + +from recurquant.public_data import ( + MBPP_CONFIG, + MBPP_CONFIRMATION_LOCK, + MBPP_CONFIRMATION_TOKEN, + MBPP_DATASET_ID, + MBPP_REVISION, + ConfirmationLockedError, + MBPPPhase, + build_mbpp_manifest, + format_mbpp_example, + format_mbpp_prompt_code, + load_mbpp_phase, + load_mbpp_rows, + mbpp_manifest, + mbpp_manifest_sha256, + mbpp_row_sha256, + mbpp_source_split, + select_mbpp_calibration, +) + + +def row(task_id: int, *, newline: str = "\n", code: str | None = None) -> dict[str, Any]: + return { + "task_id": task_id, + "text": f"Write task {task_id}.{newline}", + "code": code if code is not None else f"def task_{task_id}():{newline} return {task_id}", + "test_list": [f"assert task_{task_id}() == {task_id}"], + "test_setup_code": "", + "challenge_test_list": [], + } + + +def test_pins_and_phase_split_mapping_are_frozen() -> None: + assert MBPP_DATASET_ID == "google-research-datasets/mbpp" + assert MBPP_CONFIG == "full" + assert MBPP_REVISION == "4bb6404fdc6cacfda99d4ac4205087b89d32030c" + assert MBPP_CONFIRMATION_LOCK == MBPP_CONFIRMATION_TOKEN + assert mbpp_source_split(MBPPPhase.CALIBRATION) == "train" + assert mbpp_source_split("development") == "validation" + assert mbpp_source_split("confirmation") == "test" + + +def test_calibration_selection_uses_lowest_frozen_task_hashes() -> None: + candidates = [row(task_id) for task_id in range(1, 9)] + expected_ids = [ + task_id + for _, task_id in sorted( + ( + hashlib.sha256(f"rq-v0.2|{task_id}".encode()).hexdigest(), + task_id, + ) + for task_id in range(1, 9) + )[:3] + ] + + selected = select_mbpp_calibration(reversed(candidates), size=3) + + assert [item["task_id"] for item in selected] == expected_ids + + +def test_calibration_selection_rejects_invalid_size_and_duplicate_ids() -> None: + with pytest.raises(ValueError, match="at least 1"): + select_mbpp_calibration([row(1)], size=0) + with pytest.raises(ValueError, match="exceeds"): + select_mbpp_calibration([row(1)], size=2) + with pytest.raises(ValueError, match="duplicate"): + select_mbpp_calibration([row(1), row(1)], size=1) + + +def test_row_hash_is_canonical_across_newline_styles_and_mapping_order() -> None: + unix = row(7, newline="\n") + windows = dict(reversed(list(row(7, newline="\r\n").items()))) + + assert mbpp_row_sha256(unix) == mbpp_row_sha256(windows) + expected = "d5d1b650c6c244c8b0abd9526e578c08b8317b102f6e6c378cc21b919ed86bed" + assert mbpp_row_sha256(unix) == expected + assert mbpp_row_sha256(unix) != mbpp_row_sha256(row(7, code="def changed():\n pass")) + + +def test_manifest_hash_is_order_independent_but_phase_bound() -> None: + rows = [row(3), row(1), row(2)] + + first = mbpp_manifest_sha256(rows, phase="development") + second = mbpp_manifest_sha256(reversed(rows), phase="development") + confirmation = mbpp_manifest_sha256(rows, phase="confirmation") + + assert first == second + assert first == "95ddcf8f5bc51b4445d9676bbf4c5644580f244d2de23b7abd5d12eb7820ee81" + assert first != confirmation + manifest = build_mbpp_manifest(rows, phase="development") + assert mbpp_manifest(rows, phase="development") == manifest + assert manifest["source_split"] == "validation" + assert manifest["selection_namespace"] is None + assert [item["task_id"] for item in manifest["rows"]] == [1, 2, 3] + + +def test_prompt_code_formatter_keeps_target_separate_and_normalizes_newlines() -> None: + example = row(9, newline="\r\n", code="def answer():\r\n return 9\r\n\r\n") + example["text"] = "Write task 9." + formatted = format_mbpp_prompt_code(example) + + assert formatted.prompt == ( + "You are an expert Python programmer, and here is your task: Write task 9.\n" + "Your code should pass these tests:\n\nassert task_9() == 9\n[BEGIN]\n" + ) + assert formatted.code == "def answer():\n return 9\n\n" + assert formatted.combined[formatted.code_start :] == formatted.code + assert format_mbpp_example(row(9)).prompt.startswith("You are an expert Python programmer") + + +def test_loader_is_pinned_and_calibration_is_selected_after_lazy_call() -> None: + calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + def fake_loader(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + calls.append((args, kwargs)) + return [row(task_id) for task_id in range(1, 7)] + + loaded = load_mbpp_phase( + "calibration", + calibration_size=2, + load_dataset_fn=fake_loader, + ) + + assert len(loaded) == 2 + assert calls == [ + ( + (MBPP_DATASET_ID, MBPP_CONFIG), + {"revision": MBPP_REVISION, "split": "train", "streaming": True}, + ) + ] + + +def test_development_maps_to_validation_without_confirmation_token() -> None: + seen_split: list[str] = [] + + def fake_loader(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + seen_split.append(kwargs["split"]) + return [row(5)] + + assert load_mbpp_phase("development", load_dataset_fn=fake_loader)[0]["task_id"] == 5 + assert seen_split == ["validation"] + + +def test_script_api_applies_limit_after_frozen_calibration_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + candidates = [row(task_id) for task_id in range(1, 150)] + + def fake_loader(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + assert kwargs["split"] == "train" + return candidates + + monkeypatch.setattr("recurquant.public_data._load_dataset_lazily", fake_loader) + full = load_mbpp_rows("calibration") + limited = load_mbpp_rows("calibration", limit=3) + + assert len(full) == 128 + assert limited == full[:3] + with pytest.raises(ValueError, match="limit"): + load_mbpp_rows("development", limit=0) + + +def test_confirmation_guard_blocks_before_loader_and_accepts_explicit_token() -> None: + calls = 0 + + def fake_loader(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + nonlocal calls + calls += 1 + assert kwargs["split"] == "test" + return [row(601)] + + with pytest.raises(ConfirmationLockedError, match="confirmation is locked"): + load_mbpp_phase("confirmation", load_dataset_fn=fake_loader) + with pytest.raises(ConfirmationLockedError): + load_mbpp_phase( + "confirmation", + confirmation_token="wrong-token", + load_dataset_fn=fake_loader, + ) + assert calls == 0 + + loaded = load_mbpp_phase( + "confirmation", + confirmation_token=MBPP_CONFIRMATION_TOKEN, + load_dataset_fn=fake_loader, + ) + assert calls == 1 + assert loaded[0]["task_id"] == 601 + + +def test_script_api_requires_explicit_confirmation_lock(monkeypatch: pytest.MonkeyPatch) -> None: + calls = 0 + + def fake_loader(*args: Any, **kwargs: Any) -> Iterable[Mapping[str, Any]]: + nonlocal calls + calls += 1 + return [row(601)] + + monkeypatch.setattr("recurquant.public_data._load_dataset_lazily", fake_loader) + with pytest.raises(ConfirmationLockedError): + load_mbpp_rows("confirmation") + assert calls == 0 + + with pytest.raises(ValueError, match="limit is forbidden"): + load_mbpp_rows( + "confirmation", + limit=1, + confirmation_lock=MBPP_CONFIRMATION_LOCK, + ) + assert calls == 0 + + loaded = load_mbpp_rows( + "confirmation", + confirmation_lock=MBPP_CONFIRMATION_LOCK, + ) + assert calls == 1 + assert loaded[0]["task_id"] == 601 + + +def test_unknown_phase_and_invalid_rows_fail_closed() -> None: + with pytest.raises(ValueError, match="unknown MBPP phase"): + mbpp_source_split("preview") + with pytest.raises(TypeError, match="task_id"): + mbpp_row_sha256({**row(1), "task_id": "1"}) + with pytest.raises(TypeError, match="test_list"): + mbpp_row_sha256({**row(1), "test_list": "assert True"}) diff --git a/tests/test_quantization.py b/tests/test_quantization.py index 36c5a97..d172c07 100644 --- a/tests/test_quantization.py +++ b/tests/test_quantization.py @@ -3,7 +3,7 @@ import pytest import torch -from recurquant.quantization import QuantizationSpec, quantize_dequantize +from recurquant.quantization import QuantizationSpec, quantize_dequantize, quantize_pack def test_zero_state_round_trips_exactly() -> None: @@ -36,6 +36,54 @@ def test_more_bits_reduce_error_on_same_groups() -> None: assert int8.max_absolute_error < int4.max_absolute_error +def test_fp16_scale_storage_changes_the_numerical_round_trip() -> None: + state = torch.tensor([0.0, 0.017, 0.061, 0.1234567], dtype=torch.float32) + + fp16_scales = quantize_dequantize( + state, + QuantizationSpec(bits=4, group_size=4, scale_bits=16, flatten_last_dims=1), + ) + fp32_scales = quantize_dequantize( + state, + QuantizationSpec(bits=4, group_size=4, scale_bits=32, flatten_last_dims=1), + ) + + assert not torch.equal(fp16_scales.tensor, fp32_scales.tensor) + assert fp16_scales.scale_bits == 16 + assert fp32_scales.scale_bits == 32 + + +def test_fp16_scale_storage_stays_finite_for_tiny_nonzero_values() -> None: + state = torch.full((8,), 1e-10, dtype=torch.float32) + spec = QuantizationSpec( + bits=4, + group_size=8, + scale_bits=16, + flatten_last_dims=1, + epsilon=1e-12, + ) + + qdq = quantize_dequantize(state, spec) + packed = quantize_pack(state, spec) + + assert torch.isfinite(qdq.tensor).all() + assert torch.isfinite(packed.dequantize()).all() + assert torch.equal(packed.dequantize(), qdq.tensor) + + +@pytest.mark.parametrize("bits", [4, 8]) +def test_physical_pack_matches_qdq_and_exact_storage(bits: int) -> None: + state = torch.linspace(-2.3, 1.7, 30, dtype=torch.float32).reshape(1, 2, 3, 5) + spec = QuantizationSpec(bits=bits, group_size=8) + + qdq = quantize_dequantize(state, spec) + packed = quantize_pack(state, spec) + + assert torch.equal(packed.dequantize(), qdq.tensor) + assert packed.storage_bytes == qdq.estimated_bytes + assert packed.payload.dtype == (torch.uint8 if bits == 4 else torch.int8) + + def test_stochastic_rounding_is_seeded() -> None: state = torch.linspace(-1.0, 1.0, 64, dtype=torch.float32).reshape(1, 1, 8, 8) spec = QuantizationSpec(bits=4, group_size=16, rounding="stochastic", seed=17) @@ -52,6 +100,7 @@ def test_stochastic_rounding_is_seeded() -> None: ({"bits": 1}, "bits"), ({"bits": 4, "group_size": 0}, "group_size"), ({"bits": 4, "scale_bits": 0}, "scale_bits"), + ({"bits": 4, "scale_bits": 8}, "scale_bits"), ({"bits": 4, "flatten_last_dims": 0}, "flatten_last_dims"), ], ) @@ -63,3 +112,19 @@ def test_invalid_specs_are_rejected(kwargs: dict[str, int], message: str) -> Non def test_integer_tensor_is_rejected() -> None: with pytest.raises(TypeError, match="floating-point"): quantize_dequantize(torch.ones(4, dtype=torch.int64), QuantizationSpec(bits=4)) + + +@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), float("-inf")]) +def test_nonfinite_tensor_is_rejected_consistently(invalid: float) -> None: + state = torch.tensor([1.0, invalid]) + spec = QuantizationSpec(bits=4, group_size=2, flatten_last_dims=1) + + with pytest.raises(ValueError, match="finite"): + quantize_dequantize(state, spec) + with pytest.raises(ValueError, match="finite"): + quantize_pack(state, spec) + + +def test_physical_pack_rejects_unimplemented_bit_width() -> None: + with pytest.raises(ValueError, match="INT4 and INT8"): + quantize_pack(torch.ones(8), QuantizationSpec(bits=6)) diff --git a/tests/test_qwen35_factory.py b/tests/test_qwen35_factory.py new file mode 100644 index 0000000..a0f4005 --- /dev/null +++ b/tests/test_qwen35_factory.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import pytest +import torch +from transformers import GPT2Config, Qwen3_5ForCausalLM + +import recurquant.qwen35 as qwen35_module +from recurquant import ( + PackedRecurrentStateCache, + create_qwen35_packed_cache, + create_qwen35_v02_mixed_cache, +) +from recurquant.cli import build_parser +from recurquant.packed_cache import PackedLinearAttentionLayer +from recurquant.qwen35_quickstart import ( + MIXED_POLICY, + UNIFORM_INT4_STRESS_POLICY, + _model_dtype, + run_qwen35_quickstart, +) +from tests.test_transformers_cache import tiny_config + + +def _tiny_model(layer_types: list[str] | None = None) -> Qwen3_5ForCausalLM: + return Qwen3_5ForCausalLM._from_config( + tiny_config(layer_types), + attn_implementation="eager", + ).eval() + + +def test_public_factory_runs_prefill_and_incremental_decode_on_cpu() -> None: + torch.manual_seed(41) + model = _tiny_model() + cache = create_qwen35_packed_cache(model, bits=4, group_size=16) + prompt = torch.tensor([[1, 2, 3, 4]]) + + with torch.inference_mode(): + prefill = model(prompt, past_key_values=cache, use_cache=True) + next_token = prefill.logits[:, -1, :].argmax(dim=-1, keepdim=True) + decode = model(next_token, past_key_values=cache, use_cache=True) + + assert isinstance(cache, PackedRecurrentStateCache) + assert decode.logits.shape == (1, 1, model.config.vocab_size) + assert cache.storage_summary() == { + "resident_bytes": 80, + "full_precision_equivalent_bytes": 512, + "largest_materialized_state_bytes": 512, + "resident_compression_ratio": 6.4, + "physical_reduction_realized": True, + } + + +def test_public_factory_runs_greedy_generation_on_cpu() -> None: + torch.manual_seed(43) + model = _tiny_model() + cache = create_qwen35_packed_cache(model.config, bits=4, group_size=16) + prompt = torch.tensor([[1, 2, 3, 4]]) + + with torch.inference_mode(): + generated = model.generate( + prompt, + past_key_values=cache, + use_cache=True, + max_new_tokens=3, + do_sample=False, + pad_token_id=0, + ) + + assert generated.shape == (1, 7) + assert cache.storage_summary()["physical_reduction_realized"] is True + + +def test_public_factory_runs_beam_generation_and_reorders_on_cpu( + monkeypatch: pytest.MonkeyPatch, +) -> None: + torch.manual_seed(47) + model = _tiny_model() + cache = create_qwen35_packed_cache(model, bits=4, group_size=16) + prompt = torch.tensor([[1, 2, 3, 4]]) + reorder_indices: list[torch.Tensor] = [] + original_reorder = PackedLinearAttentionLayer.reorder_cache + + def record_reorder( + layer: PackedLinearAttentionLayer, + beam_idx: torch.LongTensor, + ) -> None: + reorder_indices.append(beam_idx.detach().cpu().clone()) + original_reorder(layer, beam_idx) + + monkeypatch.setattr(PackedLinearAttentionLayer, "reorder_cache", record_reorder) + with torch.inference_mode(): + generated = model.generate( + prompt, + past_key_values=cache, + use_cache=True, + max_new_tokens=3, + do_sample=False, + num_beams=3, + pad_token_id=0, + ) + + assert generated.shape == (1, 7) + assert reorder_indices + assert all(indices.shape == (3,) for indices in reorder_indices) + assert cache.storage_summary()["resident_bytes"] == 240 + + +def test_public_factory_rejects_unsupported_inputs_with_actions() -> None: + with pytest.raises(TypeError, match="text-only Qwen3.5"): + create_qwen35_packed_cache(GPT2Config()) # type: ignore[arg-type] + + training_model = Qwen3_5ForCausalLM._from_config( + tiny_config(), + attn_implementation="eager", + ) + with pytest.raises(ValueError, match=r"Call model\.eval\(\).*torch\.inference_mode\(\)"): + create_qwen35_packed_cache(training_model) + + with pytest.raises(ValueError, match="no linear_attention layers"): + create_qwen35_packed_cache(tiny_config(["full_attention"])) + + with pytest.raises(ValueError, match="bits=4 or bits=8"): + create_qwen35_packed_cache(tiny_config(), bits=3) + + +def test_config_only_factory_validates_structure_not_runtime() -> None: + config = tiny_config() + config._attn_implementation = "sdpa" + + cache = create_qwen35_packed_cache(config, bits=4, group_size=16) + + assert isinstance(cache, PackedRecurrentStateCache) + + +def test_frozen_v02_helper_applies_the_exact_mixed_policy() -> None: + cache = create_qwen35_v02_mixed_cache( + tiny_config(["linear_attention", "linear_attention", "full_attention"]) + ) + + specs = {layer_index: layer.spec for layer_index, layer in cache.packed_layers()} + assert {layer_index: spec.bits for layer_index, spec in specs.items()} == {0: 8, 1: 4} + assert all(spec.group_size == 128 for spec in specs.values()) + assert all(spec.scale_bits == 16 for spec in specs.values()) + assert all(spec.rounding == "nearest" for spec in specs.values()) + assert all(spec.seed == 2339 for spec in specs.values()) + + +def test_model_factory_rejects_untested_attention_backend() -> None: + model = _tiny_model() + model.config._attn_implementation = "sdpa" + + with pytest.raises(ValueError, match="attn_implementation='eager'"): + create_qwen35_packed_cache(model) + + +def test_model_factory_rejects_sharded_device_map() -> None: + model = _tiny_model() + model.hf_device_map = { + "model.layers.0": "cpu", + "model.layers.1": "cuda:0", + } + + with pytest.raises(ValueError, match="sharded or multi-device"): + create_qwen35_packed_cache(model) + + +def test_model_factory_rejects_meta_parameters() -> None: + model = _tiny_model().to("meta") + + with pytest.raises(ValueError, match="containing meta tensors"): + create_qwen35_packed_cache(model) + + +def test_eval_model_forward_requires_disabled_autograd() -> None: + model = _tiny_model() + cache = create_qwen35_packed_cache(model, bits=4, group_size=16) + + with pytest.raises(RuntimeError, match=r"model\.eval\(\) alone does not disable autograd"): + model(torch.tensor([[1, 2, 3, 4]]), past_key_values=cache, use_cache=True) + + +@pytest.mark.parametrize( + "version", + ["5.14.0", "5.14.2", "5.14.2rc1", "5.14.2.dev1", "5.15.0"], +) +def test_public_factory_rejects_untested_or_prerelease_transformers( + version: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(qwen35_module.transformers, "__version__", version) + with pytest.raises(RuntimeError, match=r"transformers==5\.14\.1"): + create_qwen35_packed_cache(tiny_config()) + + +def test_public_factory_rejects_invalid_version_before_loading_qwen_classes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(qwen35_module.transformers, "__version__", "not-a-version") + + def classes_must_not_load() -> tuple[type[object], type[object]]: + pytest.fail("Qwen classes loaded before version validation") + + monkeypatch.setattr(qwen35_module, "_load_qwen_classes", classes_must_not_load) + with pytest.raises(RuntimeError, match="could not interpret Transformers version"): + create_qwen35_packed_cache(tiny_config()) + + +def test_quickstart_dtype_contract(monkeypatch: pytest.MonkeyPatch) -> None: + assert _model_dtype(torch.device("cpu")) is torch.float32 + + monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda: True) + assert _model_dtype(torch.device("cuda")) is torch.bfloat16 + + monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda: False) + with pytest.warns(RuntimeWarning, match="not been validated for FP16"): + assert _model_dtype(torch.device("cuda")) is torch.float16 + + +def test_installed_qwen35_command_uses_shared_mixed_policy_defaults() -> None: + parser = build_parser() + + default_args = parser.parse_args(["qwen35"]) + assert default_args.handler is run_qwen35_quickstart + assert default_args.policy == MIXED_POLICY + assert default_args.max_new_tokens == 32 + + stress_args = parser.parse_args( + ["qwen35", "--policy", UNIFORM_INT4_STRESS_POLICY, "--local-files-only"] + ) + assert stress_args.policy == UNIFORM_INT4_STRESS_POLICY + assert stress_args.local_files_only is True diff --git a/tests/test_qwen35_quickstart_output.py b/tests/test_qwen35_quickstart_output.py new file mode 100644 index 0000000..be41bd8 --- /dev/null +++ b/tests/test_qwen35_quickstart_output.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest +import torch + +import recurquant.qwen35_quickstart as quickstart + + +class _FakeEncoding(dict[str, torch.Tensor]): + def to(self, device: torch.device) -> _FakeEncoding: + del device + return self + + +class _FakeTokenizer: + eos_token_id = 2 + + def __call__(self, prompt: str, *, return_tensors: str) -> _FakeEncoding: + assert prompt == quickstart.DEFAULT_PROMPT + assert return_tensors == "pt" + return _FakeEncoding(input_ids=torch.tensor([[1, 2]])) + + def decode(self, token_ids: torch.Tensor, *, skip_special_tokens: bool) -> str: + assert token_ids.tolist() == [2] + assert skip_special_tokens is True + return "synthetic completion" + + +class _FakeModel: + def __init__(self) -> None: + self.training = True + + def to(self, device: torch.device) -> _FakeModel: + assert device == torch.device("cpu") + return self + + def eval(self) -> _FakeModel: + self.training = False + return self + + def __call__(self, **kwargs: object) -> SimpleNamespace: + assert kwargs["use_cache"] is True + assert kwargs["logits_to_keep"] == 1 + logits = torch.tensor([[[0.0, 0.0, 1.0]]]) + return SimpleNamespace(logits=logits) + + +class _FakeCache: + summary = { + "resident_bytes": 24, + "full_precision_equivalent_bytes": 192, + "largest_materialized_state_bytes": 96, + "resident_compression_ratio": 8.0, + "physical_reduction_realized": True, + } + + def storage_summary(self) -> dict[str, int | float | bool]: + return dict(self.summary) + + +@pytest.fixture +def fake_workflow(monkeypatch: pytest.MonkeyPatch) -> _FakeCache: + tokenizer = _FakeTokenizer() + model = _FakeModel() + cache = _FakeCache() + + def load_tokenizer(*args: object, **kwargs: object) -> _FakeTokenizer: + assert args == (quickstart.MODEL_ID,) + assert kwargs["revision"] == quickstart.MODEL_REVISION + assert kwargs["local_files_only"] is True + return tokenizer + + def load_model(*args: object, **kwargs: object) -> _FakeModel: + assert args == (quickstart.MODEL_ID,) + assert kwargs["revision"] == quickstart.MODEL_REVISION + assert kwargs["local_files_only"] is True + return model + + def create_cache(loaded_model: torch.nn.Module, policy: str) -> _FakeCache: + assert loaded_model is model + assert policy == quickstart.MIXED_POLICY + return cache + + monkeypatch.setattr(quickstart.AutoTokenizer, "from_pretrained", load_tokenizer) + monkeypatch.setattr(quickstart.AutoModelForCausalLM, "from_pretrained", load_model) + monkeypatch.setattr(quickstart, "_create_cache", create_cache) + return cache + + +def test_json_mode_prints_one_complete_document( + fake_workflow: _FakeCache, + capsys: pytest.CaptureFixture[str], +) -> None: + result = quickstart.main( + ["--device", "cpu", "--max-new-tokens", "1", "--local-files-only", "--json"] + ) + + captured = capsys.readouterr() + assert result == 0 + assert captured.err == "" + assert captured.out.count("\n") == 1 + assert json.loads(captured.out) == { + "generated_text": "synthetic completion", + "model_id": quickstart.MODEL_ID, + "model_revision": quickstart.MODEL_REVISION, + "policy": quickstart.MIXED_POLICY, + "storage_summary": fake_workflow.summary, + } + + +def test_human_mode_preserves_existing_output( + fake_workflow: _FakeCache, + capsys: pytest.CaptureFixture[str], +) -> None: + result = quickstart.main( + ["--device", "cpu", "--max-new-tokens", "1", "--local-files-only"] + ) + + captured = capsys.readouterr() + assert result == 0 + assert captured.err == "" + assert captured.out == ( + "policy=mixed-v02\n" + "synthetic completion\n" + "resident_recurrent_state_bytes=24\n" + "full_precision_equivalent_recurrent_state_bytes=192\n" + "largest_materialized_recurrent_state_bytes=96\n" + "resident_compression_ratio=8.000x\n" + "physical_reduction_realized=True\n" + ) diff --git a/tests/test_transformers_cache.py b/tests/test_transformers_cache.py index 148c5dc..8b0adf0 100644 --- a/tests/test_transformers_cache.py +++ b/tests/test_transformers_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pytest import torch from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig @@ -7,12 +8,14 @@ from recurquant.transformers_cache import RecurrentStateQDQCache -def tiny_config() -> Qwen3_5TextConfig: +def tiny_config(layer_types: list[str] | None = None) -> Qwen3_5TextConfig: + if layer_types is None: + layer_types = ["linear_attention", "full_attention"] return Qwen3_5TextConfig( vocab_size=128, hidden_size=64, intermediate_size=128, - num_hidden_layers=2, + num_hidden_layers=len(layer_types), num_attention_heads=4, num_key_value_heads=2, head_dim=16, @@ -23,7 +26,7 @@ def tiny_config() -> Qwen3_5TextConfig: linear_value_head_dim=8, linear_num_key_heads=2, linear_num_value_heads=2, - layer_types=["linear_attention", "full_attention"], + layer_types=layer_types, rope_parameters={ "rope_type": "default", "rope_theta": 10_000.0, @@ -90,3 +93,13 @@ def test_layer_specific_spec_overrides_default() -> None: cache.update_recurrent_state(state, layer_idx=0) assert cache.update_evidence[0].bits == 8 + + +def test_qdq_cache_rejects_non_linear_or_unknown_configuration() -> None: + config = tiny_config() + spec = QuantizationSpec(bits=4, group_size=16) + + with pytest.raises(ValueError, match="non-linear or unknown"): + RecurrentStateQDQCache(config, spec=spec, layer_specs={1: spec}) + with pytest.raises(ValueError, match="non-linear or unknown"): + RecurrentStateQDQCache(config, spec=spec, enabled_layers=[99])