From 33376129cbeb671916d706680688741e48032d05 Mon Sep 17 00:00:00 2001 From: zhouhy Date: Tue, 10 Mar 2026 19:42:06 +0800 Subject: [PATCH 1/4] feat: evaluation --- AGENTS.md | 29 + README.md | 4 +- README_ZH.md | 4 +- docs/BENCHMARK_INTEGRATION_GUIDE.md | 387 +++ docs/BENCHMARK_INTEGRATION_GUIDE_ZH.md | 382 +++ docs/index.md | 2 + playground/eval/benchmarks/AIME25/__init__.py | 3 + .../eval/benchmarks/AIME25/benchmark.py | 109 + .../eval/benchmarks/GPQADiamond/__init__.py | 3 + .../eval/benchmarks/GPQADiamond/benchmark.py | 48 + playground/eval/benchmarks/HMMT25/__init__.py | 3 + .../eval/benchmarks/HMMT25/benchmark.py | 6 + .../eval/benchmarks/IFBench/__init__.py | 3 + .../eval/benchmarks/IFBench/benchmark.py | 250 ++ .../benchmarks/IFBench/official/__init__.py | 1 + .../IFBench/official/evaluation_lib.py | 191 ++ .../IFBench/official/instructions.py | 2326 +++++++++++++++++ .../IFBench/official/instructions_registry.py | 78 + .../IFBench/official/instructions_util.py | 1631 ++++++++++++ .../IFBench/official/resource_config.py | 42 + .../eval/benchmarks/MMLUPro/__init__.py | 3 + .../eval/benchmarks/MMLUPro/benchmark.py | 17 + playground/eval/benchmarks/common.py | 275 ++ playground/eval/eval_sets/simple_eval.py | 494 ++++ .../qwen3_1p7b_eval_simple_benchmarks.py | 139 + .../step3p5/step3p5_eval_simple_benchmarks.py | 148 ++ playground/rlvr/qwen3_1p5b_rlvr_math.py | 6 +- playground/rlvr/simple_trainable.py | 23 +- .../step3p5_flash_sft_step3_data_muon.py | 1 + pyproject.toml | 12 +- steptronoss/exp/gen_eval.py | 29 + steptronoss/exp/inference.py | 60 +- steptronoss/generation/async_generation.py | 119 +- steptronoss/generation/base_benchmark.py | 369 +++ steptronoss/generation/base_generatable.py | 11 + steptronoss/generation/vllm/vllm_client.py | 17 +- steptronoss/generation/vllm/vllm_router.py | 16 +- steptronoss/utils/general.py | 29 + tests/benchmarks/test_common_metrics.py | 73 + tests/benchmarks/test_ifbench_benchmark.py | 177 ++ tests/benchmarks/test_math_benchmarks.py | 46 + tests/test_async_generation.py | 151 +- tests/test_simple_eval_cache.py | 412 +++ tests/test_vllm_router.py | 29 + 44 files changed, 8052 insertions(+), 106 deletions(-) create mode 100644 docs/BENCHMARK_INTEGRATION_GUIDE.md create mode 100644 docs/BENCHMARK_INTEGRATION_GUIDE_ZH.md create mode 100644 playground/eval/benchmarks/AIME25/__init__.py create mode 100644 playground/eval/benchmarks/AIME25/benchmark.py create mode 100644 playground/eval/benchmarks/GPQADiamond/__init__.py create mode 100644 playground/eval/benchmarks/GPQADiamond/benchmark.py create mode 100644 playground/eval/benchmarks/HMMT25/__init__.py create mode 100644 playground/eval/benchmarks/HMMT25/benchmark.py create mode 100644 playground/eval/benchmarks/IFBench/__init__.py create mode 100644 playground/eval/benchmarks/IFBench/benchmark.py create mode 100644 playground/eval/benchmarks/IFBench/official/__init__.py create mode 100644 playground/eval/benchmarks/IFBench/official/evaluation_lib.py create mode 100644 playground/eval/benchmarks/IFBench/official/instructions.py create mode 100644 playground/eval/benchmarks/IFBench/official/instructions_registry.py create mode 100644 playground/eval/benchmarks/IFBench/official/instructions_util.py create mode 100644 playground/eval/benchmarks/IFBench/official/resource_config.py create mode 100644 playground/eval/benchmarks/MMLUPro/__init__.py create mode 100644 playground/eval/benchmarks/MMLUPro/benchmark.py create mode 100644 playground/eval/benchmarks/common.py create mode 100644 playground/eval/eval_sets/simple_eval.py create mode 100644 playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py create mode 100644 playground/eval/step3p5/step3p5_eval_simple_benchmarks.py create mode 100644 steptronoss/exp/gen_eval.py create mode 100644 steptronoss/generation/base_benchmark.py create mode 100644 tests/benchmarks/test_common_metrics.py create mode 100644 tests/benchmarks/test_ifbench_benchmark.py create mode 100644 tests/benchmarks/test_math_benchmarks.py create mode 100644 tests/test_simple_eval_cache.py diff --git a/AGENTS.md b/AGENTS.md index e9102de5..0652eda2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,6 +171,35 @@ Improve pass: - `use_swiglu_limit` - `use_swiglu_limit_shared` +### Eval experiment pattern + +- `playground/eval/*` currently uses a thin `GenableEvalConfig` wrapper, not the trainer stack. +- The common eval skeleton is three roles in `resource_cfg.task_specs`: `router`, `vllm`, `evaluator`. +- Runtime flow is: `Exp.entrypoint()` dispatch by `ROLE` -> router publishes `VLLM_ROUTER_ADDR_PORT_` in exp Redis -> vLLM worker health-checks then registers -> evaluator waits via `vllm_cfg.build_cli().wait_for_server()` and runs `eval_cfg.eval()`. +- Sample execution path is `GenableEvalConfig.eval()` -> `GenerationController.generate()` -> `SimpleTrainable.generate()` -> router `/v1/completions`; task-specific metrics are computed in the concrete eval config, not in a shared benchmark harness. +- These eval jobs depend on `STEPTRON_MEET_DIR` because `get_exp_redis()` uses the shared filesystem there for Redis rendezvous. +- For a single-node eval, prefer `python tools/mp_run.py playground/eval/...py`; for multi-node/manual scheduling, generate per-role scripts with `python tools/build_scripts.py playground/eval/...py `. +- A fresh vLLM 0.17 eval startup can spend several minutes before `/health` opens: model inspection, distributed init, weight load, `torch.compile`, KV-cache sizing, and CUDA graph capture all happen before the controller can register success. Repeated controller-side `Connection refused` during that phase is expected if logs still show forward progress. +- Use `GenableItem` for generation-only eval items and reserve `TrainableItem` for objects that actually implement `generate_for_train()`. `GenerationController` now accepts `GenableItem` on the normal generate path and only requires `TrainableItem` when `for_train=True`. +- The chat eval wraps each `chat/completions` request with `retry_on(...)` for transient HTTP failures. Retry only transport errors and transient statuses (`408`, `425`, `429`, `5xx`); keep permanent `4xx` responses fail-fast so bad requests are not retried blindly. +- `steptronoss/generation/vllm/vllm_router.py` now aims to be timeout-transparent: its upstream `aiohttp` session disables `total/connect/sock_connect/sock_read` timeouts instead of imposing an extra router-side timeout layer. Client-side timeouts still exist, but they are not propagated over HTTP; the closest transparent behavior is for the router not to add its own. +- `GenerationController.set_tqdm(disabled, total, desc)` is the supported way to customize progress output. The callback thread owns the actual tqdm instance; callers should configure it from the main thread instead of constructing ad-hoc bars around controller callbacks. +- In `steptronoss/generation/async_generation.py`, global generation throttling must happen in the main `GenerationController` dispatch path, not by bounding the worker `mp.Queue` alone. Each worker immediately drains that queue into its own local asyncio queue, so a plain queue `maxsize` is not a real global concurrency cap. Use callback/result arrival as the ack that frees one in-flight slot and dispatches the next pending genable. +- In the eval, do not pass raw `max_decode_steps=max_seq_len` straight through to `chat/completions`. Cap each request by `max_model_len - len(prompt.tokens)`; otherwise vLLM rejects every call with `VLLMValidationError` because the prompt leaves zero completion budget. +- `vllm_gpu_memory_utilization=0.95` can make the full mixed-benchmark eval collapse with `EngineDeadError` / `Process EngineCore_DP* died` once generation starts. Lowering the vLLM flag to `0.85` stabilized the default `num_generation_workers=32` subset run (benchmark `down_sample_to=1`) and allowed the full run to start cleanly without immediate OOM spam. +- If you introduce benchmark abstractions on the OSS side, keep the base protocol under `steptronoss/generation/base_benchmark.py`, and put concrete benchmark implementations under `playground/eval/benchmarks//`. Avoid hiding benchmark selection behind a registry when the benchmark set is still evolving quickly; explicit construction in the eval exp is easier to audit and refactor. +- Shared simple-benchmark eval plumbing now lives in `playground/eval/eval_sets/simple_eval.py`. That module owns the simple-benchmark list itself; model-specific eval files should only bind model/resource/tokenizer config on top of `SimpleBenchmarksEvalConfig`. +- Some Step3/Step3.5 training exports under `/oss/checkpoints/.../hf` contain only safetensor shards plus `model.safetensors.index.json`, without `config.json` or tokenizer assets. Those raw dirs are not directly serveable by vLLM; prepare a wrapper HF dir (for example `hf_vllm`) that adds a compatible `config.json`, and point `tokenizer_path` at a separate mounted tokenizer. +- Sampling policy for the shared simple-benchmark eval should live in `SimpleBenchmarksEvalConfig.get_sampling_params(...)`, not be hardcoded inside `SimpleChatGeneratable`. Keep the generatable responsible only for per-request normalization such as context-budget clamping and filling a default seed when the config leaves it unset. +- Benchmark-focused tests under `tests/` should live in `tests/benchmarks/` instead of the top-level `tests/` directory, so benchmark wrappers and their fixtures stay grouped together. +- Benchmark-specific code should stay inside its own benchmark folder under `playground/eval/benchmarks//`; avoid spreading benchmark logic, helper modules, or downloaded benchmark assets into unrelated directories. +- Benchmark class initialization must stay lightweight. Prefer lazy import, lazy data parsing, and lazy verifier/resource setup; importing a benchmark module or constructing the benchmark object should not trigger heavyweight package imports, network access, or resource downloads. +- Benchmark resources should live under one explicit root path agreed for that benchmark, and the benchmark class should receive that path through initialization parameters or derive it from a caller-provided parent such as `datasets_dir`. Do not hide resource paths across multiple hardcoded locations. +- If a benchmark depends on external resources beyond Python packages, such as NLTK corpora/models, download them ahead of time into that benchmark resource root and have runtime code read from there. Do not rely on import-time auto-download behavior. +- `playground/eval/benchmarks/IFBench/benchmark.py` should lazy-import the official AllenAI verifier from `playground/eval/benchmarks/IFBench/official/` and derive its explicit resource root from the caller-provided simple-benchmark `datasets_dir`, using `/IFBENCH/` for `IFBench_test.jsonl` plus `nltk_data/`. `simple_eval` should pass that directory root directly, and the benchmark should only accept that directory-root form instead of carrying compatibility for explicit prompt-file paths. Do not keep a second hardcoded IFBench resource root in the benchmark or helper modules. Keep NLTK/resource setup lazy too; do not trigger imports, downloads, or directory creation at module import time. It defaults to official `loose` scoring and strips inline `...`-style reasoning before verification; rollout/sampling settings still come from `simple_eval`, so leaderboard parity still requires matching the official generation settings such as `temperature=0`. +- Keep IFBench benchmark-owned sampling overrides narrow. `playground/eval/benchmarks/IFBench/benchmark.py` should pin official settings like `temperature=0`, but should not hardcode `extra_body.chat_template_kwargs`; IFBench thinking/chat-template behavior should flow from `SimpleBenchmarksEvalConfig.chat_template_args`. +- For vendored official benchmark helpers such as `playground/eval/benchmarks/IFBench/official/`, keep only the runtime scoring path needed by the OSS benchmark wrapper. Script-style file I/O helpers, report printers, and other standalone-binary scaffolding from the upstream repo are dead weight unless the OSS call path actually invokes them. + ## 6. Parallelism and Checkpointing ### Parallel state diff --git a/README.md b/README.md index d4554b93..46d82050 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,6 @@ set_optimization( - [x] SFT exps - [x] Reference configs: Qwen3 8B `playground/pretrain/qwen3/qwen3_8.py`, Step3.5 Flash `playground/pretrain/step3p5/step3p5_flash.py` -- [ ] Eval +- [x] Eval - [ ] RLVR implementation -- [ ] Triton kernel implementation +- [x] Triton kernel implementation diff --git a/README_ZH.md b/README_ZH.md index 1e6d8424..2354e3af 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -175,6 +175,6 @@ set_optimization( - [x] SFT exps - [x] Reference configs: Qwen3 8B `playground/pretrain/qwen3/qwen3_8.py`, Step3.5 Flash `playground/pretrain/step3p5/step3p5_flash.py` -- [ ] Eval +- [x] Eval - [ ] RLVR 实现 -- [ ] Triton kernel 实现 +- [x] Triton kernel 实现 diff --git a/docs/BENCHMARK_INTEGRATION_GUIDE.md b/docs/BENCHMARK_INTEGRATION_GUIDE.md new file mode 100644 index 00000000..3c5d060a --- /dev/null +++ b/docs/BENCHMARK_INTEGRATION_GUIDE.md @@ -0,0 +1,387 @@ +# Benchmark Integration Guide + +This guide describes how to add a benchmark to the current OSS benchmark +stack. + +It reflects the code that exists today in: + +- `steptronoss/generation/base_benchmark.py` +- `playground/eval/benchmarks/common.py` +- `playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py` + +## Target Layout + +Add benchmark code under: + +```text +playground/eval/benchmarks// + __init__.py + benchmark.py +``` + +## Current Wiring + +Benchmark support is the explicit list returned by +`SimpleBenchmarksEvalConfig.get_benchmarks()` in +`playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py`. + +Each supported benchmark is imported and constructed there directly. +`selected_datasets` filters this explicit supported set. +Dataset files on disk become active only when they are constructed in +`get_benchmarks()`. + +## Choose the Base Class First + +There are two intended entry points. + +### `BaseBenchmark` + +Use `BaseBenchmark` when the benchmark does not naturally come from an exported +jsonl file. + +Typical cases: + +- a synthetic benchmark +- a benchmark generated in code +- a benchmark backed by a database or service +- a one-off debug benchmark + +`BaseBenchmark` only requires: + +- `name` +- `get_cases()` +- `evaluate(results)` + +It does not assume `data_path`. + +### `JsonlChatBenchmark` + +Use `JsonlChatBenchmark` when the benchmark is backed by exported chat-style +jsonl data. + +It already handles: + +- loading jsonl rows +- parsing exported `messages` +- parsing exported `source_item` +- chat-template tokenization +- prompt fan-out with `sample_per_prompt` +- default generation-first aggregation + +Its constructor currently takes: + +- `data_path` +- `tokenizer` +- `sample_per_prompt` +- `down_sample_to` +- `shuffle_prompts` +- `shuffle_seed` +- `chat_template_options` + +## Core Data Model + +`steptronoss/generation/base_benchmark.py` defines the benchmark-facing data +flow. + +### `Prompt` + +Client-side request payload for one generation. + +Fields: + +- `tokens` +- `messages` +- `prompt_token_count` +- `sampling_params` + +Important: + +- `Prompt` is the request-ready object from the benchmark/client view. +- exactly one of `tokens` or `messages` should be populated +- the choice depends on the generation API the benchmark runner calls +- `prompt_token_count` stores prompt length for budget checks when the request + payload uses `messages` +- `sampling_params` belongs here because the same logical prompt can be issued + as different concrete generation requests + +### `BenchmarkMeta` + +Benchmark-owned metadata. + +Fields: + +- `benchmark_name` +- `item_id` +- `context` + +Use `context` for benchmark-specific scoring data such as: + +- gold answers +- judge labels +- subcategories +- references +- checklist data +Benchmark-specific scoring data lives here. + +### `EvaluationMeta` + +Evaluator-owned runtime metadata. + +Fields: + +- `prompt_index` +- `run_index` + +This layer should stay benchmark-agnostic. + +### `EvaluationCase` + +The minimal unit sent through generation: + +```text +EvaluationCase = Prompt + BenchmarkMeta + EvaluationMeta +``` + +### `Generated` + +The final generation result. + +Fields: + +- `case` +- `choice` +- `response` +- `reasoning_content` +- `error` + +Scorers should read benchmark/evaluator metadata through explicit ownership +paths such as: + +- `result.case.benchmark.context` +- `result.case.benchmark.item_id` +- `result.case.evaluation.run_index` +`Generated` exposes `case` as the metadata entry point. + +## Minimal Non-jsonl Example + +If you want the smallest possible benchmark, inherit `BaseBenchmark` directly. + +This example asks: + +- prompt: "How many r are in strawberry?" +- correctness rule: the response contains `3` + +```python +from steptronoss.generation.base_benchmark import ( + BaseBenchmark, + BaseMetric, + BenchmarkMeta, + EvaluationCase, + EvaluationMeta, + Generated, + Prompt, + SamplingParams, +) + + +class StrawberryRBenchmark(BaseBenchmark): + name = "STRAWBERRY_R" + + def get_cases(self) -> list[EvaluationCase]: + return [ + EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "How many r are in strawberry?"}], + prompt_token_count=3, + sampling_params=SamplingParams(max_tokens=8, seed=0), + ), + benchmark=BenchmarkMeta( + benchmark_name=self.name, + item_id="strawberry_r_count_0", + context={"answer": "3"}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=0), + ) + ] + + def evaluate(self, results: list[Generated]) -> BaseMetric: + sample_values = [ + 1.0 + if result.error is None and "3" in result.response + else 0.0 + for result in results + ] + score_avg = sum(sample_values) / len(sample_values) + return BaseMetric(score_avg=score_avg, score_std=0.0, pass_at_k={1: score_avg}) +``` + +This is the correct starting point when there is no jsonl to read. +The `pass_at_k={1: score_avg}` shortcut is only correct here because this +example emits exactly one sample per prompt. + +## Jsonl-backed Benchmark Pattern + +For exported chat benchmarks, inherit `JsonlChatBenchmark`. + +Minimal shape: + +```python +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BaseMetric, Generated + + +class MyBenchmark(JsonlChatBenchmark): + dataset_name = "MY_DATASET" + + @classmethod + def _is_correct(cls, result: Generated) -> bool: + answer = result.case.benchmark.context.get("answer") + gold = answer.strip() if isinstance(answer, str) else str(answer).strip() + return result.error is None and result.response.strip() == gold + + def evaluate(self, results: list[Generated]) -> BaseMetric: + sample_values = [1.0 if self._is_correct(result) else 0.0 for result in results] + return self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=self._is_correct, + ) +``` + +Important: + +- exported `source_item` is parsed by `JsonlChatBenchmark` into + `BenchmarkMeta(item_id=..., context=...)` +- scorers should read `result.case.benchmark.context` +- scorer logic should depend on `BenchmarkMeta`, not on raw exported row shape + +## Metric Semantics + +For repeated-sampling benchmarks, `JsonlChatBenchmark._build_metric(...)` +computes two different aggregates: + +- `score_avg`: the mean per-sample score over all generated outputs +- `pass_at_k`: the standard unbiased HumanEval-style pass@k estimator, + computed per `item_id` from `n` sampled outputs and `c` successful outputs + +For one item: + +- `pass@k = 1 - C(n-c, k) / C(n, k)` +- if `n - c < k`, then `pass@k = 1` + +Important implications: + +- `pass_at_k` is order-invariant: it depends on how many samples succeeded, not + on which `run_index` succeeded first +- `pass_at_k` is not the old "did any of the first k samples succeed" prefix + metric +- when `sample_per_prompt == 1`, `pass_at_k[1] == score_avg` + +## Export the Symbol + +Create `__init__.py`: + +```python +from .benchmark import MyBenchmark + +__all__ = ["MyBenchmark"] +``` + +## Wire the Benchmark into the Eval Exp + +Open: + +- `playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py` + +Then: + +1. import the benchmark inside `get_benchmarks()` +2. append an explicit constructor call to the `benchmarks` list + +Follow the existing style: + +- keep benchmark construction explicit +- do not hide selection behind a registry +- validate `selected_datasets` against the explicit supported set + +## Handle Unsupported Datasets Explicitly + +If a dataset should not be supported: + +- do not add it to `get_benchmarks()` +- keep `selected_datasets` failing fast when explicitly requested +- document the omission if the exported dataset exists on disk + +## Validation + +Use at least three checks. + +### 1. Static syntax check + +```bash +python3 -m py_compile \ + steptronoss/generation/base_benchmark.py \ + playground/eval/benchmarks//benchmark.py \ + playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py +``` + +### 2. Synthetic scorer check + +Construct a tiny `Generated` sample directly and verify: + +- extraction +- normalization +- fallback behavior +- obvious negative cases + +### 3. Fresh eval wiring check + +Make sure the benchmark can be constructed from +`SimpleBenchmarksEvalConfig.get_benchmarks()` and survives +`selected_datasets` filtering. + +## Metric Semantics + +Be explicit about what the score means. + +Valid meanings: + +- exact correctness +- heuristic quality +- generation-first completion + +Do not claim correctness when the exported data only supports a heuristic. + +## Recommended Checklist + +Before considering a benchmark integrated, confirm: + +- the class exists under `playground/eval/benchmarks//` +- `__init__.py` exports the symbol +- the eval exp constructs it explicitly in `get_benchmarks()` +- the scorer reads benchmark metadata from `result.case.benchmark.context` +- the benchmark works on a synthetic sample +- static syntax checking passes +- `selected_datasets` accepts the benchmark name and rejects unsupported names + +## Common Pitfalls + +- Inheriting `JsonlChatBenchmark` for a benchmark that does not read jsonl +- Treating exported `source_item` as a stable base-layer type +- Putting benchmark-specific scoring fields into `EvaluationMeta` +- Hiding data ownership with convenience properties instead of using + `result.case...` +- Treating a judge question as if it were the original model prompt +- Treating a reference answer as a gold answer +- Forcing a heuristic onto rows with insufficient signal +- Using `Any` or `dict[str, Any]` in new benchmark-facing type hints + +## Rule of Thumb + +If exported data contains: + +- a gold answer: write a real scorer +- a structured target: write a parser-based scorer +- only weak metadata: write a heuristic scorer +- almost no scoring signal: keep it generation-first or exclude it diff --git a/docs/BENCHMARK_INTEGRATION_GUIDE_ZH.md b/docs/BENCHMARK_INTEGRATION_GUIDE_ZH.md new file mode 100644 index 00000000..a9b3a4ef --- /dev/null +++ b/docs/BENCHMARK_INTEGRATION_GUIDE_ZH.md @@ -0,0 +1,382 @@ +# Benchmark 接入指南（中文） + +本文说明如何在当前 OSS benchmark 栈中接入新的 benchmark。 + +本文内容以当前代码为准，主要对应： + +- `steptronoss/generation/base_benchmark.py` +- `playground/eval/benchmarks/common.py` +- `playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py` + +英文版见： + +- `docs/BENCHMARK_INTEGRATION_GUIDE.md` + +## 目标目录结构 + +新增 benchmark 时，代码放在： + +```text +playground/eval/benchmarks// + __init__.py + benchmark.py +``` + +## 当前接线方式 + +当前真正的支持集，等于 +`playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py` 中 +`SimpleBenchmarksEvalConfig.get_benchmarks()` 返回的显式 benchmark 列表。 + +每个被支持的 benchmark 都在这里被直接 import 并直接构造。 +`selected_datasets` 只在这个显式支持集上做过滤。 +磁盘上的数据文件，只有在 `get_benchmarks()` 中被构造后，才会进入 harness。 + +## 先选对基类 + +这里有两个预期入口。 + +### `BaseBenchmark` + +如果 benchmark 并不是天然来自导出的 jsonl，就应该直接继承 +`BaseBenchmark`。 + +典型场景： + +- synthetic benchmark +- 代码里直接生成的数据 +- 来自数据库或服务的 benchmark +- 一次性的 debug benchmark + +`BaseBenchmark` 只要求： + +- `name` +- `get_cases()` +- `evaluate(results)` + +它不假设存在 `data_path`。 + +### `JsonlChatBenchmark` + +如果 benchmark 来自导出的 chat-style jsonl，则继承 +`JsonlChatBenchmark`。 + +它已经处理好了： + +- jsonl 行加载 +- 导出 `messages` 解析 +- 导出 `source_item` 解析 +- chat template tokenization +- 基于 `sample_per_prompt` 的 prompt fan-out +- 默认 generation-first 聚合逻辑 + +它当前的构造参数是： + +- `data_path` +- `tokenizer` +- `sample_per_prompt` +- `down_sample_to` +- `shuffle_prompts` +- `shuffle_seed` +- `chat_template_options` + +## 核心数据模型 + +`steptronoss/generation/base_benchmark.py` 定义了 benchmark 侧的基础数据流。 + +### `Prompt` + +`Prompt` 表示一条面向 client 的生成请求。 + +字段： + +- `tokens` +- `messages` +- `prompt_token_count` +- `sampling_params` + +重要约束： + +- `Prompt` 是 request-ready 的对象。 +- `tokens` 和 `messages` 二选一 +- 具体使用哪一个，取决于 benchmark runner 调用的推理 API +- 当请求载荷使用 `messages` 时，`prompt_token_count` 用来保存 prompt 长度，供 budget 检查使用 +- `sampling_params` 放在 `Prompt` 上，因为同一个逻辑 prompt 可以对应多次不同的具体生成请求 + +### `BenchmarkMeta` + +`BenchmarkMeta` 是 benchmark 自己拥有的元数据。 + +字段： + +- `benchmark_name` +- `item_id` +- `context` + +`context` 用来承载 benchmark-specific 的评分上下文，例如： + +- gold answer +- judge label +- subcategory +- references +- checklist +benchmark-specific 的评分数据就放在这里。 + +### `EvaluationMeta` + +`EvaluationMeta` 是 evaluator 拥有的、与 benchmark 无关的运行时元数据。 + +字段： + +- `prompt_index` +- `run_index` + +这一层应保持 benchmark-agnostic。 + +### `EvaluationCase` + +`EvaluationCase` 是送进 generation 流水线的最小单元： + +```text +EvaluationCase = Prompt + BenchmarkMeta + EvaluationMeta +``` + +### `Generated` + +`Generated` 是最终的生成结果。 + +字段： + +- `case` +- `choice` +- `response` +- `reasoning_content` +- `error` + +scorer 应通过明确的 ownership 路径访问元数据，例如： + +- `result.case.benchmark.context` +- `result.case.benchmark.item_id` +- `result.case.evaluation.run_index` +`Generated` 通过 `case` 暴露元数据入口。 + +## 最小非 jsonl 示例 + +如果你要写的是最小 benchmark，且根本不需要读 jsonl，就直接继承 +`BaseBenchmark`。 + +下面这个例子对应最简单的 case： + +- prompt: “How many r are in strawberry?” +- 判定规则：回答中出现 `3` 即视为正确 + +```python +from steptronoss.generation.base_benchmark import ( + BaseBenchmark, + BaseMetric, + BenchmarkMeta, + EvaluationCase, + EvaluationMeta, + Generated, + Prompt, + SamplingParams, +) + + +class StrawberryRBenchmark(BaseBenchmark): + name = "STRAWBERRY_R" + + def get_cases(self) -> list[EvaluationCase]: + return [ + EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "How many r are in strawberry?"}], + prompt_token_count=3, + sampling_params=SamplingParams(max_tokens=8, seed=0), + ), + benchmark=BenchmarkMeta( + benchmark_name=self.name, + item_id="strawberry_r_count_0", + context={"answer": "3"}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=0), + ) + ] + + def evaluate(self, results: list[Generated]) -> BaseMetric: + sample_values = [ + 1.0 + if result.error is None and "3" in result.response + else 0.0 + for result in results + ] + score_avg = sum(sample_values) / len(sample_values) + return BaseMetric(score_avg=score_avg, score_std=0.0, pass_at_k={1: score_avg}) +``` + +这就是“没有 jsonl 时的正确起点”。 +这里把 `pass_at_k` 写成 `{1: score_avg}` 只在这个最小示例里成立， +因为它对每个 prompt 只生成 1 个样本。 + +## 基于 jsonl 的 benchmark 模式 + +如果 benchmark 来自导出的 chat jsonl，继承 `JsonlChatBenchmark`。 + +最小形式： + +```python +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BaseMetric, Generated + + +class MyBenchmark(JsonlChatBenchmark): + dataset_name = "MY_DATASET" + + @classmethod + def _is_correct(cls, result: Generated) -> bool: + answer = result.case.benchmark.context.get("answer") + gold = answer.strip() if isinstance(answer, str) else str(answer).strip() + return result.error is None and result.response.strip() == gold + + def evaluate(self, results: list[Generated]) -> BaseMetric: + sample_values = [1.0 if self._is_correct(result) else 0.0 for result in results] + return self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=self._is_correct, + ) +``` + +这里要注意： + +- 导出的 `source_item` 会被 `JsonlChatBenchmark` 解析成 + `BenchmarkMeta(item_id=..., context=...)` +- scorer 读的是 `result.case.benchmark.context` +- scorer 逻辑应依赖 `BenchmarkMeta`，而不是依赖导出行的原始结构 + +## Metric 语义 + +对于会重复采样的 benchmark，`JsonlChatBenchmark._build_metric(...)` +会计算两个不同的聚合量： + +- `score_avg`：把所有生成样本摊平后的逐样本平均分 +- `pass_at_k`：标准的 HumanEval 风格无偏 `pass@k` 估计量， + 先按 `item_id` 分组，再根据每题的 `n` 个样本里有 `c` 个成功样本来计算 + +对单个题目： + +- `pass@k = 1 - C(n-c, k) / C(n, k)` +- 如果 `n - c < k`，则 `pass@k = 1` + +几个要点： + +- `pass_at_k` 与样本顺序无关；只看成功样本个数，不看哪个 `run_index` + 先成功 +- `pass_at_k` 不再是旧的“前 k 个样本里是否命中”的 prefix 指标 +- 当 `sample_per_prompt == 1` 时，`pass_at_k[1] == score_avg` + +## 导出符号 + +创建 `__init__.py`： + +```python +from .benchmark import MyBenchmark + +__all__ = ["MyBenchmark"] +``` + +## 接入 eval exp + +打开： + +- `playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py` + +然后： + +1. 在 `get_benchmarks()` 内 import 新 benchmark +2. 在 `benchmarks` 列表中显式追加构造调用 + +遵循当前风格： + +- benchmark 构造必须显式 +- 不要用 registry 隐藏选择逻辑 +- `selected_datasets` 只对显式支持集做校验 + +## 显式处理不支持的数据集 + +如果某个数据集不应被支持： + +- 不要把它加入 `get_benchmarks()` +- 当用户在 `selected_datasets` 里显式请求它时，保持 fail fast +- 如果磁盘上已有导出数据，最好在文档或注释里明确说明为何未接入 + +## 验证方式 + +至少做三层验证。 + +### 1. 静态语法检查 + +```bash +python3 -m py_compile \ + steptronoss/generation/base_benchmark.py \ + playground/eval/benchmarks//benchmark.py \ + playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py +``` + +### 2. Synthetic scorer 检查 + +手动构造一个很小的 `Generated` 样本，验证： + +- 提取逻辑 +- 归一化逻辑 +- fallback 行为 +- 明显负例 + +### 3. Fresh eval wiring 检查 + +确认 benchmark 能通过 +`SimpleBenchmarksEvalConfig.get_benchmarks()` 被构造出来，并且能通过 +`selected_datasets` 过滤。 + +## Metric 语义 + +分数语义必须明确属于下面之一： + +- 精确 correctness +- heuristic quality +- generation-first completion + +如果导出数据只能支持 heuristic，就不要把它包装成 correctness。 + +## 推荐检查清单 + +在认为 benchmark 已经接入完成之前，确认： + +- 类存在于 `playground/eval/benchmarks//` +- `__init__.py` 正确导出符号 +- eval exp 在 `get_benchmarks()` 里显式构造了它 +- scorer 从 `result.case.benchmark.context` 读取 benchmark 元数据 +- synthetic sample 能跑通 +- 静态语法检查通过 +- `selected_datasets` 能接受它的 benchmark 名称，并拒绝不支持的名称 + +## 常见坑 + +- 明明不读 jsonl，却硬继承 `JsonlChatBenchmark` +- 把导出的 `source_item` 当成稳定的 base-layer 类型 +- 把 benchmark-specific 的评分字段塞进 `EvaluationMeta` +- 用 convenience property 隐藏 ownership，而不是显式写 `result.case...` +- 把 judge question 当成模型原始 prompt +- 把 reference answer 当成 gold answer +- 对信号不足的样本硬上 heuristic +- 在新类型标注里使用 `Any` 或 `dict[str, Any]` + +## 经验法则 + +如果导出数据里有： + +- gold answer：写真实 scorer +- 结构化目标：写 parser-based scorer +- 只有弱 metadata：写 heuristic scorer +- 几乎没有评分信号：保持 generation-first，或者直接排除 diff --git a/docs/index.md b/docs/index.md index d9ae14b0..820eb238 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,8 @@ cfshow playground/rlvr/qwen3_1p5b_rlvr_math.py ## Documentation - Start here: [LAUNCH_EXPERIMENTS](LAUNCH_EXPERIMENTS.md) +- Benchmark integration guide (EN): [BENCHMARK_INTEGRATION_GUIDE](BENCHMARK_INTEGRATION_GUIDE.md) +- Benchmark 接入指南 (ZH): [BENCHMARK_INTEGRATION_GUIDE_ZH](BENCHMARK_INTEGRATION_GUIDE_ZH.md) - Launch guide (EN): [LAUNCH_EXPERIMENTS](LAUNCH_EXPERIMENTS.md) - Launch guide (ZH): [LAUNCH_EXPERIMENTS_ZH](LAUNCH_EXPERIMENTS_ZH.md) - SFT 数据准备: [SFT_DATA_PREPARATION](SFT_DATA_PREPARATION.md) diff --git a/playground/eval/benchmarks/AIME25/__init__.py b/playground/eval/benchmarks/AIME25/__init__.py new file mode 100644 index 00000000..9f09e5d7 --- /dev/null +++ b/playground/eval/benchmarks/AIME25/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import AIME25Benchmark + +__all__ = ["AIME25Benchmark"] diff --git a/playground/eval/benchmarks/AIME25/benchmark.py b/playground/eval/benchmarks/AIME25/benchmark.py new file mode 100644 index 00000000..31248bc7 --- /dev/null +++ b/playground/eval/benchmarks/AIME25/benchmark.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from functools import lru_cache + +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BaseMetric, Generated + + +class AIME25Benchmark(JsonlChatBenchmark): + dataset_name = "AIME2025" + + @staticmethod + def _extract_boxed_contents(response: str) -> list[str]: + contents: list[str] = [] + marker = r"\boxed{" + start = 0 + while True: + boxed_index = response.find(marker, start) + if boxed_index == -1: + break + + index = boxed_index + len(marker) + depth = 1 + while index < len(response) and depth > 0: + char = response[index] + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + index += 1 + + if depth == 0: + contents.append(response[boxed_index + len(marker) : index - 1]) + start = index + else: + break + return contents + + @staticmethod + def _normalize_answer_text(text: str) -> str: + return ( + text + .strip() + .replace(r"\dfrac", r"\frac") + .replace(r"\tfrac", r"\frac") + .replace(r"\left", "") + .replace(r"\right", "") + .replace(" ", "") + .replace("\n", "") + ) + + @staticmethod + def _wrap_boxed(text: str) -> str: + return rf"\boxed{{{text}}}" + + @staticmethod + @lru_cache(maxsize=1) + def _math_verify_ops(): + try: + from math_verify import parse, verify + except ImportError: + return None, None + return parse, verify + + @classmethod + def _math_verify_equal(cls, predicted: str, answer: str) -> bool: + parse, verify = cls._math_verify_ops() + if parse is None or verify is None: + return False + + try: + parsed_predicted = parse(cls._wrap_boxed(predicted), parsing_timeout=30) + parsed_answer = parse(cls._wrap_boxed(answer), parsing_timeout=30) + return bool(verify(parsed_predicted, parsed_answer, timeout_seconds=30)) + except Exception: + return False + + @classmethod + def _extract_answer(cls, response: str) -> str: + if not response: + return "" + boxed_matches = cls._extract_boxed_contents(response) + if boxed_matches: + return boxed_matches[-1].strip() + return response.strip() + + @staticmethod + def _is_correct(result: Generated, answer: str) -> bool: + if result.error: + return False + predicted_raw = AIME25Benchmark._extract_answer(result.response) + predicted = AIME25Benchmark._normalize_answer_text(predicted_raw) + normalized_answer = AIME25Benchmark._normalize_answer_text(answer) + if predicted == normalized_answer: + return True + return AIME25Benchmark._math_verify_equal(predicted_raw, answer) + + def evaluate(self, results: list[Generated]) -> BaseMetric: + def _gold_answer(result: Generated) -> str: + answer = result.case.benchmark.context.get("answer") + return answer.strip() if isinstance(answer, str) else str(answer).strip() + + sample_values = [1.0 if self._is_correct(result, _gold_answer(result)) else 0.0 for result in results] + return self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=lambda result: self._is_correct(result, _gold_answer(result)), + ) diff --git a/playground/eval/benchmarks/GPQADiamond/__init__.py b/playground/eval/benchmarks/GPQADiamond/__init__.py new file mode 100644 index 00000000..ee4018b6 --- /dev/null +++ b/playground/eval/benchmarks/GPQADiamond/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import GPQADiamondBenchmark + +__all__ = ["GPQADiamondBenchmark"] diff --git a/playground/eval/benchmarks/GPQADiamond/benchmark.py b/playground/eval/benchmarks/GPQADiamond/benchmark.py new file mode 100644 index 00000000..87f2fb91 --- /dev/null +++ b/playground/eval/benchmarks/GPQADiamond/benchmark.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import re + +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BaseMetric, Generated + + +class GPQADiamondBenchmark(JsonlChatBenchmark): + dataset_name = "GPQA_DIAMOND" + _OPTION_PATTERN = re.compile(r"(? str: + if not response: + return "" + + boxed_matches = cls._BOXED_PATTERN.findall(response) + for candidate in reversed(boxed_matches): + matches = cls._OPTION_PATTERN.findall(candidate.upper()) + if matches: + return matches[-1] + + matches = cls._OPTION_PATTERN.findall(response.upper()) + if matches: + return matches[-1] + return "" + + @staticmethod + def _is_correct(result: Generated, answer: str) -> bool: + if result.error: + return False + predicted = GPQADiamondBenchmark._extract_choice(result.response) + return predicted == answer.strip().upper() + + def evaluate(self, results: list[Generated]) -> BaseMetric: + def _gold_answer(result: Generated) -> str: + answer = result.case.benchmark.context.get("answer") + return answer.strip() if isinstance(answer, str) else str(answer).strip() + + sample_values = [1.0 if self._is_correct(result, _gold_answer(result)) else 0.0 for result in results] + return self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=lambda result: self._is_correct(result, _gold_answer(result)), + ) diff --git a/playground/eval/benchmarks/HMMT25/__init__.py b/playground/eval/benchmarks/HMMT25/__init__.py new file mode 100644 index 00000000..04d23bb2 --- /dev/null +++ b/playground/eval/benchmarks/HMMT25/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import HMMT25Benchmark + +__all__ = ["HMMT25Benchmark"] diff --git a/playground/eval/benchmarks/HMMT25/benchmark.py b/playground/eval/benchmarks/HMMT25/benchmark.py new file mode 100644 index 00000000..6f85ab10 --- /dev/null +++ b/playground/eval/benchmarks/HMMT25/benchmark.py @@ -0,0 +1,6 @@ +from playground.eval.benchmarks.AIME25 import AIME25Benchmark + + +class HMMT25Benchmark(AIME25Benchmark): + dataset_name = "HMMT25" + pass diff --git a/playground/eval/benchmarks/IFBench/__init__.py b/playground/eval/benchmarks/IFBench/__init__.py new file mode 100644 index 00000000..4fadf3ff --- /dev/null +++ b/playground/eval/benchmarks/IFBench/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import IFBenchBenchmark + +__all__ = ["IFBenchBenchmark"] diff --git a/playground/eval/benchmarks/IFBench/benchmark.py b/playground/eval/benchmarks/IFBench/benchmark.py new file mode 100644 index 00000000..da69ce5f --- /dev/null +++ b/playground/eval/benchmarks/IFBench/benchmark.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import copy +import json +import random +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + +from playground.eval.benchmarks.common import ChatTokenizer, JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BaseMetric, Generated, JsonObject, SamplingParams + + +@dataclass +class IFBenchMetric(BaseMetric): + evaluation_mode: str = "loose" + official_metrics: JsonObject = field(default_factory=dict) + + def to_dict(self) -> JsonObject: + payload = super().to_dict() + payload["evaluation_mode"] = self.evaluation_mode + payload["official_metrics"] = copy.deepcopy(self.official_metrics) + return payload + + +class IFBenchBenchmark(JsonlChatBenchmark): + """IFBench adapter backed by AllenAI's official prompt set and verifier.""" + + dataset_name = "IFBENCH" + _PROMPT_FILENAME = "IFBench_test.jsonl" + _INLINE_REASONING_PATTERNS = ( + re.compile(r"^\s*.*?\s*", re.DOTALL), + re.compile(r"^\s*.*?\s*", re.DOTALL), + re.compile(r"^\s*.*?\s*", re.DOTALL), + re.compile(r"^\s*.*?\s*", re.DOTALL), + re.compile(r"^\s*<\|begin_of_thought\|>.*?<\|end_of_thought\|>\s*", re.DOTALL), + ) + _OFFICIAL_SAMPLING_PARAMS = SamplingParams(temperature=0.0) + + def __init__( + self, + data_path: str, + tokenizer: ChatTokenizer, + sample_per_prompt: int, + down_sample_to: int | None = None, + shuffle_prompts: bool = False, + chat_template_options: JsonObject | None = None, + evaluation_mode: Literal["loose", "strict"] = "loose", + strip_reasoning: bool = True, + sampling_params: SamplingParams | None = None, + ): + resolved_resource_root = Path(data_path) + if resolved_resource_root.name == self._PROMPT_FILENAME or resolved_resource_root.suffix == ".jsonl": + raise ValueError( + "IFBenchBenchmark.data_path must point to the IFBENCH resource directory, " + f"not the prompt file itself: {resolved_resource_root}" + ) + resolved_data_path = str(resolved_resource_root / self._PROMPT_FILENAME) + self.resource_root = str(resolved_resource_root) + self._validate_official_data_path(resolved_data_path) + self.evaluation_mode = evaluation_mode + self.strip_reasoning = strip_reasoning + self.sampling_params = ( + self._OFFICIAL_SAMPLING_PARAMS if sampling_params is None else copy.deepcopy(sampling_params) + ) + self._input_examples_cache: list[JsonObject] | None = None + super().__init__( + data_path=resolved_data_path, + tokenizer=tokenizer, + sample_per_prompt=sample_per_prompt, + down_sample_to=down_sample_to, + shuffle_prompts=shuffle_prompts, + chat_template_options=chat_template_options, + ) + + @staticmethod + def _resource_config(): + from playground.eval.benchmarks.IFBench.official import resource_config + + return resource_config + + def _evaluation_lib(self): + self._resource_config().set_resource_root(self.resource_root) + from playground.eval.benchmarks.IFBench.official import evaluation_lib + + return evaluation_lib + + @staticmethod + def _validate_official_data_path(data_path: str) -> None: + if not Path(data_path).is_file(): + raise FileNotFoundError( + "IFBench prompt file is missing. " + f"Expected {data_path}. " + "Stage IFBench resources under the caller-provided IFBENCH resource root " + "(for simple_eval this is typically /IFBENCH/)." + ) + with open(data_path, encoding="utf-8") as fin: + first_line = fin.readline() + if not first_line: + raise ValueError(f"IFBench data file is empty: {data_path}") + first_record = json.loads(first_line) + required_keys = {"key", "prompt", "instruction_id_list", "kwargs"} + if not required_keys.issubset(first_record): + raise ValueError( + "IFBenchBenchmark requires the official IFBench_test.jsonl prompt file. " + f"Expected keys {sorted(required_keys)} in {data_path}, got {sorted(first_record)}." + ) + + def _load_input_examples(self) -> list[JsonObject]: + if self._input_examples_cache is None: + input_examples: list[JsonObject] = [] + with open(self.data_path, encoding="utf-8") as fin: + for line in fin: + if not line.strip(): + continue + raw_record = json.loads(line) + prompt = raw_record.get("prompt") + instruction_id_list = raw_record.get("instruction_id_list") + kwargs = raw_record.get("kwargs") + key = raw_record.get("key") + if not isinstance(prompt, str): + raise TypeError("IFBench prompt record.prompt must be str") + if not isinstance(instruction_id_list, list) or not all( + isinstance(item, str) for item in instruction_id_list + ): + raise TypeError("IFBench prompt record.instruction_id_list must be list[str]") + if not isinstance(kwargs, list): + raise TypeError("IFBench prompt record.kwargs must be list") + normalized_kwargs: list[dict[str, object]] = [] + for prompt_kwargs in kwargs: + if not isinstance(prompt_kwargs, dict): + raise TypeError("IFBench prompt record.kwargs entries must be dict") + normalized_kwargs.append({ + field: value for field, value in prompt_kwargs.items() if value is not None + }) + input_examples.append({ + "key": str(key), + "prompt": prompt, + "instruction_id_list": list(instruction_id_list), + "kwargs": normalized_kwargs, + }) + self._input_examples_cache = input_examples + return list(self._input_examples_cache) + + def _load_records(self) -> list[tuple[str, list[dict[str, str]], str, JsonObject]]: + if self._records_cache is None: + records: list[tuple[str, list[dict[str, str]], str, JsonObject]] = [] + for example in self._load_input_examples(): + context: JsonObject = { + "key": str(example["key"]), + "prompt": str(example["prompt"]), + "instruction_id_list": copy.deepcopy(example["instruction_id_list"]), + "kwargs": copy.deepcopy(example["kwargs"]), + } + records.append(( + self.dataset_name, + [{"role": "user", "content": str(example["prompt"])}], + str(example["key"]), + context, + )) + self._records_cache = records + records = list(self._records_cache) + if self.shuffle_prompts: + rng = random.Random(1234) + rng.shuffle(records) + if self.down_sample_to is not None: + records = records[: self.down_sample_to] + return records + + def get_cases(self): + cases = super().get_cases() + if self.sampling_params is None: + return cases + return [case.with_sampling_params(copy.deepcopy(self.sampling_params)) for case in cases] + + def _build_input_example(self, context: JsonObject) -> object: + key = context.get("key") + prompt = context.get("prompt") + instruction_id_list = context.get("instruction_id_list") + kwargs = context.get("kwargs") + if not isinstance(key, str): + raise TypeError(f"IFBench context.key must be str, got {type(key).__name__}") + if not isinstance(prompt, str): + raise TypeError(f"IFBench context.prompt must be str, got {type(prompt).__name__}") + if not isinstance(instruction_id_list, list) or not all(isinstance(item, str) for item in instruction_id_list): + raise TypeError("IFBench context.instruction_id_list must be list[str]") + if not isinstance(kwargs, list): + raise TypeError(f"IFBench context.kwargs must be list, got {type(kwargs).__name__}") + return self._evaluation_lib().InputExample( + key=int(key), + instruction_id_list=list(instruction_id_list), + prompt=prompt, + kwargs=copy.deepcopy(kwargs), + ) + + @classmethod + def _strip_inline_reasoning(cls, response: str) -> str: + stripped = response + for pattern in cls._INLINE_REASONING_PATTERNS: + stripped = pattern.sub("", stripped, count=1) + return stripped.strip() + + def _response_for_official_eval(self, result: Generated) -> str | None: + if result.error: + return None + response = result.response or "" + if self.strip_reasoning and response: + response = self._strip_inline_reasoning(response) + return response + + def _input_example_for_result(self, result: Generated) -> object: + return self._build_input_example(result.case.benchmark.context) + + def _evaluate_one(self, result: Generated, *, mode: Literal["loose", "strict"]) -> object: + example = self._input_example_for_result(result) + prompt_to_response = {example.prompt: self._response_for_official_eval(result)} + evaluation_lib = self._evaluation_lib() + if mode == "strict": + return evaluation_lib.test_instruction_following_strict(example, prompt_to_response) + return evaluation_lib.test_instruction_following_loose(example, prompt_to_response) + + def evaluate(self, results: list[Generated]) -> BaseMetric: + evaluation_lib = self._evaluation_lib() + strict_outputs = [self._evaluate_one(result, mode="strict") for result in results] + loose_outputs = [self._evaluate_one(result, mode="loose") for result in results] + primary_outputs = strict_outputs if self.evaluation_mode == "strict" else loose_outputs + + sample_values = [1.0 if output.follow_all_instructions else 0.0 for output in primary_outputs] + success_by_result = { + id(result): output.follow_all_instructions for result, output in zip(results, primary_outputs, strict=True) + } + base_metric = self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=lambda result: success_by_result[id(result)], + ) + strict_report = evaluation_lib.build_accuracy_report(strict_outputs) + loose_report = evaluation_lib.build_accuracy_report(loose_outputs) + return IFBenchMetric( + score_avg=base_metric.score_avg, + score_std=base_metric.score_std, + pass_at_k=dict(base_metric.pass_at_k), + evaluation_mode=self.evaluation_mode, + official_metrics={ + "strict": strict_report.to_dict(), + "loose": loose_report.to_dict(), + }, + ) diff --git a/playground/eval/benchmarks/IFBench/official/__init__.py b/playground/eval/benchmarks/IFBench/official/__init__.py new file mode 100644 index 00000000..c9c2ef67 --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/playground/eval/benchmarks/IFBench/official/evaluation_lib.py b/playground/eval/benchmarks/IFBench/official/evaluation_lib.py new file mode 100644 index 00000000..a2440f2d --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/evaluation_lib.py @@ -0,0 +1,191 @@ +# Copyright 2025 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation helpers used by the OSS IFBench benchmark wrapper.""" + +import collections +import dataclasses + +from . import instructions_registry + + +@dataclasses.dataclass +class InputExample: + key: int + instruction_id_list: list[str] + prompt: str + kwargs: list[dict[str, str | int | None]] + + +@dataclasses.dataclass +class OutputExample: + instruction_id_list: list[str] + prompt: str + response: str + follow_all_instructions: bool + follow_instruction_list: list[bool] + + +@dataclasses.dataclass +class AccuracyReport: + prompt_level_accuracy: float + instruction_level_accuracy: float + tier0_accuracy: dict[str, float] + tier1_accuracy: dict[str, float] + + def to_dict(self) -> dict[str, float | dict[str, float]]: + return { + "prompt_level_accuracy": self.prompt_level_accuracy, + "instruction_level_accuracy": self.instruction_level_accuracy, + "tier0_accuracy": dict(self.tier0_accuracy), + "tier1_accuracy": dict(self.tier1_accuracy), + } + + +def test_instruction_following_strict( + inp, + prompt_to_response, +): + """Tests response to see if instrutions are followed.""" + response = prompt_to_response[inp.prompt] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + inp.kwargs[index] = {key: value for key, value in inp.kwargs[index].items() if value is not None} + instruction.build_description(**inp.kwargs[index]) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + if response and response.strip() and instruction.check_following(response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def test_instruction_following_loose( + inp, + prompt_to_response, +): + """Tests response for an upper bound for following instructions.""" + response = prompt_to_response[inp.prompt] + if response is None: + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response="", + follow_all_instructions=False, + follow_instruction_list=[False] * len(inp.instruction_id_list), + ) + + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + instruction.build_description(**inp.kwargs[index]) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + is_following = False + for r in all_responses: + if r.strip() and instruction.check_following(r): + is_following = True + break + + is_following_list.append(is_following) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def build_accuracy_report(outputs: list[OutputExample]) -> AccuracyReport: + prompt_total = 0 + prompt_correct = 0 + instruction_total = 0 + instruction_correct = 0 + + tier0_total: dict[str, int] = collections.defaultdict(int) + tier0_correct: dict[str, int] = collections.defaultdict(int) + tier1_total: dict[str, int] = collections.defaultdict(int) + tier1_correct: dict[str, int] = collections.defaultdict(int) + + for example in outputs: + follow_instruction_list = example.follow_instruction_list + instruction_id_list = example.instruction_id_list + + prompt_total += 1 + if all(follow_instruction_list): + prompt_correct += 1 + + instruction_total += len(instruction_id_list) + instruction_correct += sum(follow_instruction_list) + + for instruction_id, followed_or_not in zip(instruction_id_list, follow_instruction_list, strict=True): + tier0_id = instruction_id.split(":")[0] + tier0_total[tier0_id] += 1 + tier1_total[instruction_id] += 1 + if followed_or_not: + tier0_correct[tier0_id] += 1 + tier1_correct[instruction_id] += 1 + + return AccuracyReport( + prompt_level_accuracy=prompt_correct / max(prompt_total, 1), + instruction_level_accuracy=instruction_correct / max(instruction_total, 1), + tier0_accuracy={ + instruction_id: tier0_correct[instruction_id] / tier0_total[instruction_id] + for instruction_id in sorted(tier0_total) + }, + tier1_accuracy={ + instruction_id: tier1_correct[instruction_id] / tier1_total[instruction_id] + for instruction_id in sorted(tier1_total) + }, + ) diff --git a/playground/eval/benchmarks/IFBench/official/instructions.py b/playground/eval/benchmarks/IFBench/official/instructions.py new file mode 100644 index 00000000..7dbe4ffe --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/instructions.py @@ -0,0 +1,2326 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of instructions.""" + +import csv +import io +import logging +import random +import re +import string +import unicodedata +from collections import Counter +from collections.abc import Sequence + +from . import instructions_util + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = dict[str, int | str | Sequence[str]] | None + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + +# The number of numbers. +_NUM_NUMBERS = 6 + +# Period length for periodic words. +_NUM_WORD_CYCLE = 30 + +# Maximum number of times a word can be repeated. +_MAX_REPEATS = 5 + +# Which sentence must contain a keyword. +_NUM_KEYWORD_SENTENCE = 20 + +# Minimum number of pronouns. +_NUM_PRONOUNS = 25 + +# The size of increment for lengths. +_NUM_INCREMENT = 5 + +# The number of coordinating conjunctions. +_NUM_CONJUNCTIONS = 6 + + +def _emoji_module(): + import emoji + + return emoji + + +def _syllapy_module(): + import syllapy + + return syllapy + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +# Everything as follows is part of OOD IFEval + + +class WordCountRangeChecker(Instruction): + """Word Count Range: The response must contain between X and Y words.""" + + def build_description(self, *, min_words=None, max_words=None): + """Build the instruction description. + + Args: + min_words: An integer specifying the minimum number of words contained in the response. + max_words: An integer specifying the maximum number of words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._min_words = min_words + self._max_words = max_words + + if self._min_words is None or self._min_words < 0: + self._min_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) + + # Make the range small + if self._max_words is None or self._max_words < 0: + self._max_words = self._min_words + random.randint(int(self._min_words * 0.05), int(self._min_words * 0.1)) + + self._description_pattern = "The response must contain between {min_words} and {max_words} words." + + return self._description_pattern.format(min_words=self._min_words, max_words=self._max_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"min_words": self._min_words, "max_words": self._max_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["min_words", "max_words"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + return self._min_words <= num_words <= self._max_words + + +class UniqueWordCountChecker(Instruction): + """Unique Word Count: The response must contain X unique words.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + n: An integer specifying the number of unique words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_unique_words = N + + if self._num_unique_words is None or self._num_unique_words < 0: + self._num_unique_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) + + self._description_pattern = "Use at least {N} unique words in the response." + + return self._description_pattern.format(N=self._num_unique_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_unique_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains the expected number of unique words.""" + words = value.lower().split() + unique_words = set() + for word in words: + unique_words.add(word.strip("".join(string.punctuation) + " ")) + # Convert to set to get unique words + return len(unique_words) >= self._num_unique_words + + +class StopWordPercentageChecker(Instruction): + """Ensure that stop words constitute no more than {percentage}% of the total words in your response.""" + + def build_description(self, *, percentage=None): + """Build the instruction description. + + Args: + percentage: An integer specifying the percentage of stop words that are allowed in the response. + + Returns: + A string representing the instruction description. + """ + self._percentage = percentage + + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = ( + "Ensure that stop words constitute no more than {percentage}% of the total words in your response." + ) + + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["percentage"] + + def check_following(self, value): + """Checks if the response contains the expected percentage of stop words.""" + num_words = instructions_util.count_words(value) + if num_words == 0: + return False + num_stopwords = instructions_util.count_stopwords(value) + stopword_percentage = (num_stopwords / num_words) * 100 + return stopword_percentage <= self._percentage + + +class SentTypeRatioChecker(Instruction): + """Maintain a 2:1 ratio of declarative to interrogative sentences.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Maintain a 2:1 ratio of declarative to interrogative sentences." + + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected ratio of declarative to interrogative sentences.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of declarative and interrogative sentences + declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) + # Check if the ratio is 2:1 + return declarative_count == 2 * interrogative_count + + +class SentBalanceChecker(Instruction): + """Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced.""" + + def build_description(self): + """Build the instruction description.""" + + self._description_pattern = ( + "Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a balanced ratio of sentence types.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of each sentence type + declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) + exclamatory_count = sum(1 for sentence in sentences if sentence.endswith("!")) + # Check if the ratio of sentence types is balanced + return declarative_count == interrogative_count == exclamatory_count + + +class ConjunctionCountChecker(Instruction): + """Use at least {small_n} different coordinating conjunctions in the response.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the number of different coordinating conjunctions contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_conjunctions = small_n + + if self._num_conjunctions is None or self._num_conjunctions < 0: + self._num_conjunctions = random.randint(2, _NUM_CONJUNCTIONS) + + self._description_pattern = "Use at least {small_n} different coordinating conjunctions in the response." + + return self._description_pattern.format(small_n=self._num_conjunctions) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_conjunctions} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response contains the expected number of different coordinating conjunctions.""" + # Split the text into words + words = value.split() + # Count the number of coordinating conjunctions + conjunctions = [ + word + for word in words + if word.strip("".join(string.punctuation) + " ").lower() in ["and", "but", "for", "nor", "or", "so", "yet"] + ] + unique_conjunctions = set(conjunctions) + return len(unique_conjunctions) >= self._num_conjunctions + + +class PersonNameCountChecker(Instruction): + """Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia...""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of unique person names contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_person_names = N + + if self._num_person_names is None or self._num_person_names < 0: + self._num_person_names = random.randint(1, 50) + + self._description_pattern = "Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia, Jackson, Olivia, Noah, Ava, Lucas, Isabella, Mason, Mia, Ethan, Charlotte, Alexander, Amelia, Benjamin, Harper, Leo, Zoe, Daniel, Chloe, Samuel, Lily, Matthew, Grace, Owen, Abigail, Gabriel, Ella, Jacob, Scarlett, Nathan, Victoria, Elijah, Layla, Nicholas, Audrey, David, Hannah, Christopher, Penelope, Thomas, Nora, Andrew, Aria, Joseph, Claire, Ryan, Stella, Jonathan ." + return self._description_pattern.format(N=self._num_person_names) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_person_names} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains at least the expected number of unique person names.""" + person_name_list = [ + "Emma", + "Liam", + "Sophia", + "Jackson", + "Olivia", + "Noah", + "Ava", + "Lucas", + "Isabella", + "Mason", + "Mia", + "Ethan", + "Charlotte", + "Alexander", + "Amelia", + "Benjamin", + "Harper", + "Leo", + "Zoe", + "Daniel", + "Chloe", + "Samuel", + "Lily", + "Matthew", + "Grace", + "Owen", + "Abigail", + "Gabriel", + "Ella", + "Jacob", + "Scarlett", + "Nathan", + "Victoria", + "Elijah", + "Layla", + "Nicholas", + "Audrey", + "David", + "Hannah", + "Christopher", + "Penelope", + "Thomas", + "Nora", + "Andrew", + "Aria", + "Joseph", + "Claire", + "Ryan", + "Stella", + "Jonathan", + ] + # Extract the named entities + person_names = [] + for name in person_name_list: + # Use regex with word boundaries + pattern = rf"\b{re.escape(name)}\b" + if re.search(pattern, value): + person_names.append(name) + unique_person_names = set(person_names) + + return len(unique_person_names) >= self._num_person_names + + +class NGramOverlapChecker(Instruction): + """Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text.""" + + def build_description(self, *, reference_text=None, percentage=None): + """Build the instruction description. + + Args: + reference_text: A string representing the reference text. + percentage: An integer specifying the percent trigram overlap + to maintain in the response. + + Returns: + A string representing the instruction description. + """ + self._reference_text = reference_text + self._percentage = percentage + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = ( + "Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text." + ) + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"reference_text": self._reference_text, "percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["reference_text", "percentage"] + + def check_following(self, value): + """Checks if the response maintains a trigram overlap with the reference text within 2% of {percent}.""" + n = 3 + nltk_module = instructions_util.get_nltk() + ngrams = set(nltk_module.ngrams(value, n)) + ref_ngrams = set(nltk_module.ngrams(self._reference_text, n)) + if not ngrams: + return False + overlap = len(ngrams.intersection(ref_ngrams)) / len(ngrams) + return self._percentage - 2 <= overlap * 100 <= self._percentage + 2 + + +class NumbersCountChecker(Instruction): + """Include exactly {N} numbers in the response.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the exact number of numbers + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._count_numbers = N + if self._count_numbers is None or self._count_numbers < 0: + self._count_numbers = random.randint(1, _NUM_NUMBERS) + + self._description_pattern = "Include exactly {N} numbers in the response." + return self._description_pattern.format(N=self._count_numbers) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._count_numbers} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes exactly {N} numbers.""" + # Strip punctuation to handle decimals and commas in numbers correctly + value = value.translate(str.maketrans("", "", string.punctuation)) + numbers = re.findall(r"\d+", value) + return len(numbers) == self._count_numbers + + +class AlphabetLoopChecker(Instruction): + """Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word of the response starts with the next letter of the alphabet.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.strip("".join(string.punctuation) + " ").split() + if not words: + return False + alphabet = string.ascii_lowercase + correct_letter = words[0][0].lower() + if correct_letter not in alphabet: # numbers are fails + return False + for word in words[1:]: + word = word.strip("".join(string.punctuation) + " ").lower() + if not word: + continue + correct_letter = alphabet[(alphabet.index(correct_letter) + 1) % 26] + if word[0] != correct_letter: + return False + return True + + +class SingleVowelParagraphChecker(Instruction): + """Write a paragraph using words that contain only three type of vowels.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write a paragraph using words that contain only three types of vowels." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no more than three types of vowels are used in the response and the response is only 1 paragraph.""" + paragraphs = value.strip().split("\n") + if len(paragraphs) != 1: + return False + paragraph = paragraphs[0].lower() + + vowels = set("aeiou") + paragraph_vowels = {char for char in paragraph if char in vowels} + return len(paragraph_vowels) <= 3 + + +class ConsonantClusterChecker(Instruction): + """Ensure each word in your response has at least one consonant cluster (two or more consonants together).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Ensure each word in your response has at least one consonant cluster (two or more consonants together)." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word in the response includes at least one consonant cluster.""" + words = value.lower().strip().split() + consonants = set("bcdfghjklmnpqrstvwxyz") + for word in words: + cluster = False + for i in range(len(word) - 1): + if word[i] in consonants and word[i + 1] in consonants: + cluster = True + break + if not cluster: + return False + return True + + +class IncrementingAlliterationChecker(Instruction): + """Each sentence must have a longer sequence of consecutive alliterative words than the previous one.""" + + def build_description(self): + """Build the instruction description.""" + + self._description_pattern = ( + "Each sentence must have a longer sequence of consecutive alliterative words than the previous one." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each sentence in the response has more alliterative words (determined by common first letter) than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + prev_alliteration = -1 + for sentence in sentences: + words = sentence.lower().split() + alliteration = 0 + prev_alliterative = False + new_words = [] + for word in words: + clean = word.lstrip("".join(string.punctuation) + " ") + if clean: + new_words.append(clean) + for i in range(len(new_words) - 1): + if new_words[i][0] == new_words[i + 1][0]: + if prev_alliterative: + alliteration += 1 + else: + alliteration += 2 + prev_alliterative = True + else: + prev_alliterative = False + if alliteration <= prev_alliteration: + return False + prev_alliteration = alliteration + return True + + +class PalindromeChecker(Instruction): + """Include at least 10 single-word palindromes, each at least 5 characters long.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Include at least 10 single-word palindromes, each at least 5 characters long." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least 10 single-word palindromes of length at least 5.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.lower().split() + palindromes = [word for word in words if word == word[::-1] and len(word) >= 5] + return len(palindromes) >= 10 + + +class PunctuationCoverChecker(Instruction): + """Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!)." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes every standard punctuation mark at least once, including the interrobang (?!).""" + punctuation = {".", ",", "!", "?", ";", ":"} + if not ("!?" in value or "?!" in value or "‽" in value): + return False + new_value = value.replace("?!", "", 1) + if len(new_value) == len(value): + new_value = value.replace("!?", "", 1) + for char in new_value: + if char in punctuation: + punctuation.remove(char) + return not punctuation + + +class NestedParenthesesChecker(Instruction): + """Nest parentheses (and [brackets {and braces}]) at least 5 levels deep.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Nest parentheses (and [brackets {and braces}]) at least 5 levels deep." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes a correctly closed set of at least 5 nested brackets.""" + levels = [] + min_levels = 5 + max_depth = 0 + for char in value: + if char in "([{": + levels.append(char) + if len(levels) > max_depth: + max_depth = len(levels) + elif char in ")]}": + if levels and ( + (levels[-1] == "(" and char == ")") + or (levels[-1] == "[" and char == "]") + or (levels[-1] == "{" and char == "}") + ): + levels.pop() + # Check if we just closed a group that reached 5+ depth + if max_depth >= min_levels and len(levels) < max_depth: + return True + else: + # Mismatch — reset + levels = [] + max_depth = 0 + + return False + + +class NestedQuotesChecker(Instruction): + """Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes nested quotes to at least 3 levels + alternating between " and ' starting with either character.""" + levels = [] + min_levels = 3 + reached_depth = 0 + current_depth = 0 + for char in value: + if len(levels) != 0 and char == levels[-1]: + levels.pop() + current_depth -= 1 + if reached_depth - current_depth >= min_levels: + return True + elif char == '"' or char == "'": + levels.append(char) + current_depth += 1 + if current_depth > reached_depth: + reached_depth = current_depth + return False + + +class PrimeLengthsChecker(Instruction): + """Use only words with lengths that are prime numbers.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use only words with lengths that are prime numbers." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response only includes words with prime length.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + words = value.split() + primes = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97} + return all(len(word) in primes for word in words) + + +class OptionsResponseChecker(Instruction): + """Answer with one of the following options: {options}. Do not give any explanation.""" + + def build_description(self, *, options=None): + """Build the instruction description. + + Args: + options: A string specifying the permitted options for + the response. + + Returns: + A string representing the instruction description. + """ + # Options string may be: yes/no/maybe, I know or I don't know, a), b), c), d) + # Can be separated by "/", "or", "," + options_bank = ["yes/no/maybe", "I know or I don't know", "a), b), c), d)"] + if options is None: + options = random.choice(options_bank) + + # Be more strict about format for multiple choice letters than for text options + self._strict = False + if re.match(r"\W*[aA]\W*[bB]\W*[cC]\W*", options) is not None: + self._strict = True + if "/" in options: + separator = "/" + elif "or" in options: + separator = "or" + else: + separator = "," + self._options = [option.strip() for option in options.split(separator)] + self._options_text = options # in text, shouldn't be formatted as a list + self._description_pattern = "Answer with one of the following options: {options}. Do not give any explanation." + return self._description_pattern.format(options=self._options_text) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"options": self._options_text} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["options"] + + def check_following(self, value): + """Checks if the response is exactly one of {options}.""" + if self._strict: + return value in self._options + value = value.strip("".join(string.punctuation) + " ").lower() + return any(option.strip("".join(string.punctuation) + " ").lower() == value for option in self._options) + + +class NewLineWordsChecker(Instruction): + """Write each word on a new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write each word on a new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has each word on a new line.""" + value = value.translate(str.maketrans("", "", string.punctuation)) + lines = value.strip().split("\n") + while "" in lines: + lines.remove("") + return len(lines) == len(value.strip().split()) + + +class EmojiSentenceChecker(Instruction): + """Please use an emoji at the end of every sentence.""" + + def build_description(self): + """Build the instruction description.""" + + self._description_pattern = "Please use an emoji at the end of every sentence." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes an emoji at the end of every sentence.""" + + sentences = instructions_util.split_into_sentences(value) + emoji_module = _emoji_module() + for i, sentence in enumerate(sentences): + stripped = sentence.translate(str.maketrans("", "", string.punctuation)).strip() + # check for empty string + if not stripped: + return False + last_char = stripped[-1] + # because blank spaces are treated oddly + second_last_char = stripped[-2] if len(stripped) > 1 else stripped[-1] + if not emoji_module.is_emoji(last_char) and not emoji_module.is_emoji(second_last_char): + if i < len(sentences) - 1: + stripped = sentences[i + 1].translate(str.maketrans("", "", string.punctuation)).strip() + # fixed empty string + if not stripped: + return False + first_char = stripped[0] + if not emoji_module.is_emoji(first_char): + return False + else: + return False + return True + + +class CharacterCountUniqueWordsChecker(Instruction): + """Respond with three sentences, all containing the same number of characters but using all different words.""" + + def build_description(self): + """Build the instruction description.""" + + self._description_pattern = ( + "Respond with three sentences, all containing the same number of characters but using all different words." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has exactly 3 sentences containing the same number of characters but different words.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 3: + return False + char_count = len(sentences[0].strip()) + return all(len(sentence.strip()) == char_count for sentence in sentences) + + +class NthWordJapaneseChecker(Instruction): + """Every {N}th word of your response must be in Japanese.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the cycle length for + Japanese words to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._japanese_position = N + if self._japanese_position is None or self._japanese_position < 0: + self._japanese_position = random.randint(1, _NUM_WORD_CYCLE) + + self._description_pattern = "Every {N}th word of your response must be in Japanese." + if N % 10 == 1: + self._description_pattern = "Every {N}st of your response must be in Japanese." + if N % 10 == 2: + self._description_pattern = "Every {N}nd of your response must be in Japanese." + elif N % 10 == 3: + self._description_pattern = "Every {N}rd of your response must be in Japanese." + return self._description_pattern.format(N=self._japanese_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._japanese_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if every {N}th word of the response is in Japanese.""" + + def is_japanese(text): + """ + Checks if a string contains Japanese characters (Hiragana, Katakana, or Kanji). + + Args: + text: The string to check. + + Returns: + True if the string contains Japanese characters, False otherwise. + """ + japanese_pattern = re.compile(r"[\u3040-\u30ff\u4e00-\u9fff]") + return bool(japanese_pattern.search(text)) + + words = value.split() + for i, word in enumerate(words): + word = word.strip("".join(string.punctuation) + " ") + if (i + 1) % self._japanese_position == 0 and word and not word.isdigit(): + if not is_japanese(word): + return False + return True + + +class StartWithVerbChecker(Instruction): + """The response must start with a verb.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The response must start with a verb." + + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response starts with a verb.""" + text = instructions_util.word_tokenize(value) + tagged = instructions_util.pos_tag(text) if text else [] + return len(text) > 0 and len(tagged) > 0 and "VB" in tagged[0][1] + + +class LimitedWordRepeatChecker(Instruction): + """The response should not repeat any word more than {small_n} times.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the maximum number of times + that a word can be repeated in the response. + + Returns: + A string representing the instruction description. + """ + self._max_repeats = small_n + if self._max_repeats is None or self._max_repeats < 0: + self._max_repeats = random.randint(1, _MAX_REPEATS) + + self._description_pattern = "The response should not repeat any word more than {small_n} times." + return self._description_pattern.format(small_n=self._max_repeats) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._max_repeats} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response repeats any word more than {small_n} times.""" + words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() + word_count = Counter(words) + return all(count <= self._max_repeats for count in word_count.values()) + + +class IncludeKeywordChecker(Instruction): + """The response must include keyword {word} in the {N}-th sentence.""" + + def build_description(self, *, word=None, N=None): + """Build the instruction description. + + Args: + word: A string specifying the keyword that is + required to appear in the response. + N: An integer specifying which sentence of the + response is required to have the keyword. + + Returns: + A string representing the instruction description. + """ + + if not word: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = word + self._keyword_position = N + if self._keyword_position is None or self._keyword_position < 0: + self._keyword_position = random.randint(1, _NUM_KEYWORD_SENTENCE) + + self._description_pattern = 'The response must include keyword "{word}" in the {N}-th sentence.' + return self._description_pattern.format(word=self._keyword, N=self._keyword_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"word": self._keyword, "N": self._keyword_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["word", "N"] + + def check_following(self, value): + """Checks if the {N}th sentence of the response includes keyword {word}.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._keyword_position: + return False + # Use regex with word boundaries for robust matching + pattern = rf"\b{re.escape(self._keyword)}\b" + return bool(re.search(pattern, sentences[int(self._keyword_position - 1)], re.IGNORECASE)) + + +class PronounCountChecker(Instruction): + """The response should include at least {N} pronouns.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of pronouns + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_pronouns = N + if self._num_pronouns is None or self._num_pronouns < 0: + self._num_pronouns = random.randint(1, _NUM_PRONOUNS) + + self._description_pattern = "The response should include at least {N} pronouns." + return self._description_pattern.format(N=self._num_pronouns) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_pronouns} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes at least {N} pronouns.""" + pronouns = { + "i", + "me", + "my", + "mine", + "myself", + "we", + "us", + "our", + "ours", + "ourselves", + "you", + "your", + "yours", + "yourself", + "yourselves", + "he", + "him", + "his", + "himself", + "she", + "her", + "hers", + "herself", + "it", + "its", + "itself", + "they", + "them", + "their", + "theirs", + "themselves", + } + value = value.replace( + "/", " " + ) # to correctly count pronoun sets like she/her/hers, a common use case of pronouns + # Use NLTK word_tokenize for better tokenization + words = instructions_util.word_tokenize(value.lower()) + pronoun_count = sum(1 for word in words if word in pronouns) + return pronoun_count >= self._num_pronouns + + +class AlternateParitySyllablesChecker(Instruction): + """Alternate between words with odd and even numbers of syllables.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Alternate between words with odd and even numbers of syllables." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response alternates between words with odd and even numbers of syllables.""" + words = value.translate(str.maketrans("", "", string.punctuation)).lower().split() + syllapy_module = _syllapy_module() + syllables = [syllapy_module.count(word) % 2 for word in words if word.strip()] + return all(syllables[i] != syllables[i + 1] for i in range(len(syllables) - 1)) + + +class LastWordFirstNextChecker(Instruction): + """The last word of each sentence must become the first word of the next sentence.""" + + def build_description(self): + """Build the instruction description.""" + + self._description_pattern = "The last word of each sentence must become the first word of the next sentence." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the last word of each sentence in the response is the first word of the next sentence.""" + sentences = instructions_util.split_into_sentences(value) + for i in range(len(sentences) - 1): + last_words = sentences[i].rstrip("".join(string.punctuation) + " ").split() + first_words = sentences[i + 1].lstrip("".join(string.punctuation) + " ").split() + if not last_words or not first_words: + return False + if last_words[-1].lower() != first_words[0].lower(): + return False + return True + + +class ParagraphLastFirstWordMatchChecker(Instruction): + """Each paragraph must end with the same word it started with, separate paragraphs with a newline.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Each paragraph must end with the same word it started with, separate paragraphs with a newline." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each paragraph of the response ends with the same word it started with.""" + paragraphs = value.split("\n") + for paragraph in paragraphs: + paragraph = paragraph.strip().lower() + if not paragraph: + continue + words = paragraph.strip("".join(string.punctuation) + " ").split() + if not words: + continue + if words[0] != words[-1]: + return False + return True + + +class IncrementingWordCountChecker(Instruction): + """Each sentence must contain exactly {small_n} more words than the previous one.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the exact increment for + the number of words in each sentence of the response. + + Returns: + A string representing the instruction description. + """ + self._num_increment = small_n + if self._num_increment is None or self._num_increment < 0: + self._num_increment = random.randint(1, _NUM_INCREMENT) + + self._description_pattern = "Each sentence must contain exactly {small_n} more words than the previous one." + return self._description_pattern.format(small_n=self._num_increment) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_increment} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if each sentence of the response uses exactly {small_n} more words than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + words = sentences[0].translate(str.maketrans("", "", string.punctuation)).strip().split() + while "" in words: + words.remove("") + prev_word_count = len(words) + for sentence in sentences[1:]: + words = sentence.translate(str.maketrans("", "", string.punctuation)).strip().split() + while "" in words: + words.remove("") + if len(words) != prev_word_count + self._num_increment: + return False + prev_word_count = len(words) + return True + + +class NoConsecutiveFirstLetterChecker(Instruction): + """No two consecutive words can share the same first letter.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "No two consecutive words can share the same first letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no two consecutive words in the response share the same first letter.""" + words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() + while "" in words: + words.remove("") + return all(words[i][0] != words[i + 1][0] for i in range(len(words) - 1)) + + +class IndentStairsChecker(Instruction): + """Create stairs by incrementally indenting each new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Create stairs by incrementally indenting each new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response incrementally indents each new line.""" + lines = value.split("\n") + for line in lines: + if not line.strip(): + lines.remove(line) + for i in range(len(lines) - 1): + if len(lines[i + 1]) - len(lines[i + 1].lstrip(" ")) <= len(lines[i]) - len(lines[i].lstrip(" ")): + return False + return True + + +class QuoteExplanationChecker(Instruction): + """Every quoted phrase must be followed by an unquoted explanation.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Every quoted phrase must be followed by an unquoted explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there are no quotes next to each other + and the passage does not end with a quote.""" + value = value.replace('"', '"').replace('"', '"') + value = value.replace("'\"'", "") # remove references to the character '"' + value = "".join(value.split()) # remove all whitespace + if '""' in value: + return False + stripped = value.strip(string.digits + string.punctuation.replace('"', "")) + return not (stripped and stripped[-1] == '"') + + +class SpecialBulletPointsChecker(Instruction): + """Answer with a list of items, instead of bullet points use {sep}.""" + + def build_description(self, *, sep=None): + """Build the instruction description. + + Args: + sep: A string specifying the bullet point marker for + the list in the response. + + Returns: + A string representing the instruction description. + """ + self._bullet_marker = sep + if sep is None: + self._bullet_marker = random.choice(["...", "SEPARATOR", "!?!?", "-"]) + self._description_pattern = "Answer with a list of items, instead of bullet points use {sep}." + return self._description_pattern.format(sep=self._bullet_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"sep": self._bullet_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["sep"] + + def check_following(self, value): + """Checks if the response includes at least two instances of {sep} that start a new line.""" + return len(re.findall(re.escape(self._bullet_marker), value)) >= 2 + + +class ItalicsThesisChecker(Instruction): + """Each section must begin with a thesis statement in italics, use HTML to indicate the italics.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Each section must begin with a thesis statement in italics, use HTML to indicate the italics." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there is at least one line in italics as indicated + by HTML that is followed by unitalicized text.""" + index = value.find("") + if index == -1: + index = value.find("") + if index == -1: + return False + value = value[index:] + end_thesis = value.find("") + if end_thesis == -1: + end_thesis = value.find("") + if end_thesis == -1: + return False + thesis = value[3:end_thesis] + if thesis.strip() == "": + return False + text = value[end_thesis + 4 :] + return text.strip() != "" + + +class SubBulletPointsChecker(Instruction): + """Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that there is at least one * that starts a line and each * that starts a line + is followed by at least one line starting with -.""" + bullets = value.split("*") + return all("-" in bullet for bullet in bullets[1:]) + + +class SomeBulletPointsChecker(Instruction): + """Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least two sentences + followed by at least two lines that start with *.""" + lines = value.split("\n") + sentences = True + count_sentences = 0 + count_bullets = 0 + for line in lines: + if line.strip().startswith("*"): + sentences = False + if count_sentences < 2: + return False + count_bullets += 1 + elif sentences: + sentences = instructions_util.split_into_sentences(line.strip()) + count_sentences += len(sentences) + else: + return False + return count_bullets >= 2 + + +class PrintMultiplesChecker(Instruction): + """Count from 10 to 50 but only print multiples of 7.""" + + def build_description(self, **kwargs): + self._description_pattern = "Count from 10 to 50 but only print multiples of 7." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response prints multiples of 7 from 10 to 50.""" + value = value.replace(",", ", ") + numbers = re.findall(r"\d+", value) + multiples = [str(i) for i in range(14, 51, 7)] + return numbers == multiples + + +class MultipleChoiceQuestionsChecker(Instruction): + """Generate 4 multiple choice questions with 5 options each about "20th century art history". Each question should start with the label "Question". The questions should get progressively longer. Do not provide an explanation.""" + + def build_description(self, **kwargs): + self._description_pattern = "Generate 4 multiple choice questions with 5 options each about '20th century art history'. Each question should start with the label \"Question\". The questions should get progressively longer. Do not provide an explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response generates 4 multiple choice questions with 5 options.""" + # Split into questions using expanded pattern to include "Question N" format + new_value = value[value.find("Question") :] + if new_value != value: + return False # failed no explanation + value = new_value + questions = re.split(r"\n*(?:Question \d+[\.|\):;]?\s*)", value) + if questions[0] == "": + questions = questions[1:] + questions = [q.strip() for q in questions if q.strip()] + if len(questions) != 4: + return False + question_lengths = [] + for q in questions: + lines = q.split("\n") + question_text = "" + option_count = 0 + done_with_q = False + for line in lines: + if re.match(r"^[A-Ea-e][\.|\)]\s*\w+", line.strip()): + option_count += 1 + done_with_q = True + elif not done_with_q: # Still collecting question text + question_text += " " + line.strip() + if option_count != 5: + return False + question_lengths.append(len(question_text.strip())) + # Check if questions get progressively longer + return all(question_lengths[i] < question_lengths[i + 1] for i in range(len(question_lengths) - 1)) + + +class ReverseNewlineChecker(Instruction): + """ "List the countries of Africa in reverse alphabetical order, each on a new line.""" + + def build_description(self, **kwargs): + self._description_pattern = "List the countries of Africa in reverse alphabetical order, each on a new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ + Checks if text satisfies the following constraints: + 1. Contains at least 53 newlines with text + 2. Lines are in reverse alphabetical order + 3. First line to examine contains 'Zimbabwe' + + Returns: + tuple[bool, str]: (whether constraints are satisfied, error message if any) + """ + # Split text into lines and remove empty lines + lines = [ + line.strip("".join(string.punctuation) + " ") + for line in value.split("\n") + if line.strip("".join(string.punctuation) + " ") + ] + + try: + start_index = next(i for i, line in enumerate(lines) if "Zimbabwe" in line) + except StopIteration: + return False + + # Extract the 53 lines starting from Zimbabwe line + target_lines = lines[start_index:] + + # Check if we have at least 53 lines + if len(target_lines) < 52: + return False + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize("NFKD", text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") + return ascii_text + + # Create normalized versions for comparison while keeping originals for error messages + normalized_lines = [normalize_text(line) for line in target_lines] + sorted_normalized = sorted(normalized_lines, reverse=True) + return normalized_lines == sorted_normalized + + +class WordReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word.""" + + def build_description(self, **kwargs): + + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the reverse of the sentence is a valid English sentence.""" + value = value.lower().strip().translate(str.maketrans("", "", string.punctuation)) + value = " ".join(value.split()[::-1]) + if "bald eagle" not in value: + return False + return value in instructions_util.split_into_sentences(value) + + +class CharacterReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter.""" + + def build_description(self, **kwargs): + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = value.lower() + return "elgae dlab" in value + + +class SentenceAlphabetChecker(Instruction): + """Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order.""" + + def build_description(self, **kwargs): + + self._description_pattern = "Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 26: + return False + for i, sentence in enumerate(sentences): + words = sentence.lstrip().split() + if not words or not words[0]: + return False + if words[0].lower()[0] != chr(97 + i): + return False + return True + + +class EuropeanCapitalsSortChecker(Instruction): + """Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = "Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response lists the relevant capitals of Europe in correct order.""" + order = [ + "Reykjavik", + "Helsinki", + "Oslo", + "Tallinn", + "Stockholm", + "Riga", + "Moscow", + "Copenhagen", + "Vilnius", + "Minsk", + "Dublin", + "Berlin", + "Amsterdam", + "Warsaw", + "London", + "Brussels", + "Prague", + "Luxembourg", + "Paris", + "Vienna", + "Bratislava", + "Budapest", + "Vaduz", + "Chisinau", + "Bern", + "Ljubljana", + "Zagreb", + ] + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize("NFKD", text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") + return ascii_text + + value = normalize_text(value) + + capitals = value.split(",") + capitals = [cap for cap in capitals if cap.strip()] + if len(capitals) != len(order): + return False + return all(capitals[i].strip() == order[i] for i in range(len(capitals))) + + +class CityCSVChecker(Instruction): + """Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is valid csv data with column names + ["ID", "Country", "City", "Year", "Count"] and 7 rows.""" + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 8: + return False + header = data[0] + if header != ["ID", "Country", "City", "Year", "Count"]: + return False + return all(len(row) == 5 for row in data[1:]) + + +class SpecialCharacterCSVChecker(Instruction): + """Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is valid csv data with column names + ["ProductID", "Category", "Brand", "Price", "Stock"] and 14 rows. + Also checks if one field contains a special character enclosed in double quotes.""" + header = value.split("\n")[0].strip() + if not re.match( + r'^(ProductID|"ProductID"),[ \t]*(Category|"Category"),[ \t]*(Brand|"Brand"),[ \t]*(Price|"Price"),[ \t]*(Stock|"Stock")$', + header, + ): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 15: + return False + for row in data[1:]: + if len(row) != 5: + return False + if any(re.match(r'".*[^\d\w\s].*"', field) for field in row): + return True + return False + + +class QuotesCSVChecker(Instruction): + """Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is valid csv data with column names + ["StudentID", "Subject", "Grade", "Semester", "Score"] and 3 rows. + Also checks if each field is enclosed in double quotes.""" + header = value.split("\n")[0].strip() + if not re.match( + r'^(StudentID|"StudentID")\t *(Subject|"Subject")\t *(Grade|"Grade")\t *(Semester|"Semester")\t *(Score|"Score")$', + header, + ): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io, delimiter="\t") + data = list(reader) + if len(data) != 4: + return False + for row in data: + if len(row) != 5: + return False + if not all(field.strip()[0] == '"' and field.strip()[-1] == '"' for field in row): + return False + return True + + +class DateFormatListChecker(Instruction): + """List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = "List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ "Checks if the response is a list of dates in the format YYYY-MM-DD separated by commas.""" + value = value.strip() + dates = value.split(",") + for date in dates: + date = date.strip() + if not re.match(r"^\d{4}-\d{2}-\d{2}$", date): + return False + date = date.split("-") + if int(date[0]) < 1769 or int(date[0]) > 1821: + return False + if int(date[1]) > 12: + return False + if int(date[1]) in [1, 3, 5, 7, 8, 10, 12] and int(date[2]) > 31: + return False + if int(date[1]) in [4, 6, 9, 11] and int(date[2]) > 30: + return False + if int(date[1]) == 2 and int(date[2]) > 29: + return False + return True + + +class KeywordsMultipleChecker(Instruction): + """Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response.""" + + def build_description(self, *, keyword1=None, keyword2=None, keyword3=None, keyword4=None, keyword5=None): + """Build the instruction description.""" + if keyword1 is None: + self._keyword1 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword1 = keyword1.strip() + if keyword2 is None: + self._keyword2 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword2 = keyword2.strip() + if keyword3 is None: + self._keyword3 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword3 = keyword3.strip() + if keyword4 is None: + self._keyword4 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword4 = keyword4.strip() + if keyword5 is None: + self._keyword5 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword5 = keyword5.strip() + self._description_pattern = "Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response." + return self._description_pattern.format( + keyword1=self._keyword1, + keyword2=self._keyword2, + keyword3=self._keyword3, + keyword4=self._keyword4, + keyword5=self._keyword5, + ) + + def get_instruction_args(self): + return { + "keyword1": self._keyword1, + "keyword2": self._keyword2, + "keyword3": self._keyword3, + "keyword4": self._keyword4, + "keyword5": self._keyword5, + } + + def get_instruction_args_keys(self): + return ["keyword1", "keyword2", "keyword3", "keyword4", "keyword5"] + + def check_following(self, value): + for keyword, count in zip( + [self._keyword1, self._keyword2, self._keyword3, self._keyword4, self._keyword5], [1, 2, 3, 5, 7] + ): + if value.lower().count(keyword.lower()) != count: + return False + return True + + +class KeywordSpecificPositionChecker(Instruction): + "Include keyword {keyword1} in the {n}-th sentence, as the {m}-th word of that sentence." + + def build_description(self, keyword=None, n=None, m=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + n: An integer representing the sentence number. + m: An integer representing the word number. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + if not n: + self._n = random.randint(20, 30) + else: + self._n = n + if not m: + self._m = random.randint(30, 40) + else: + self._m = m + + self._description_pattern = ( + "Include keyword {keyword} in the {n}-th sentence, as the {m}-th word of that sentence." + ) + + return self._description_pattern.format(keyword=self._keyword, n=self._n, m=self._m) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"keyword": self._keyword, "n": self._n, "m": self._m} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "n", "m"] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._n: + return False + words = instructions_util.word_tokenize(sentences[self._n - 1]) + if len(words) < self._m: + return False + return words[self._m - 1].lower() == self._keyword.lower() + + +class WordsPositionChecker(Instruction): + "The second word in your response and the second to last word in your response should be the word {keyword}." + + def build_description(self, *, keyword=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if keyword is None: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + self._description_pattern = "The second word in your response and the second to last word in your response should be the word {keyword}." + return self._description_pattern.format(keyword=self._keyword) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keyword": self._keyword} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword"] + + def check_following(self, value): + """Checks if the second word and the second to last word in the response are the same. + + Args: + value: A string representing the response. + + Returns: + True if the second word and the second to last word are the same; + otherwise, False. + """ + words = instructions_util.word_tokenize(value) + if len(words) < 2: + return False + return words[1].lower() == words[-2].lower() == self._keyword.lower() + + +class RepeatChangeChecker(Instruction): + "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + + self._description_pattern = "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request! Request: {prompt_to_repeat}" + return self._description_pattern.format(prompt_to_repeat=self._prompt_to_repeat) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the repeated request. + + Args: + value: A string representing the response. + + Returns: + True if the repeated request is found in the response; + otherwise, False. + """ + if self._prompt_to_repeat == value: + return False + return " ".join(self._prompt_to_repeat.split()[1:]) == " ".join(value.split()[1:]) + + +class RepeatSimpleChecker(Instruction): + "Only output this sentence here, ignore all other requests." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Only output this sentence here, ignore all other requests." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + return value.strip().lower() == self._description_pattern.strip().lower() + + +class RepeatSpanChecker(Instruction): + "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" + + def build_description(self, prompt_to_repeat=None, n_start=None, n_end=None): + """Build the instruction description. + + Args: + n_start: An integer representing the start index of the span. + n_end: An integer representing the end index of the span. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + if not n_start: + self._n_start = random.randint(0, len(self._prompt_to_repeat.split()) - 2) + else: + self._n_start = n_start + if not n_end: + self._n_end = random.randint(self._n_start + 1, len(self._prompt_to_repeat.split()) - 1) + else: + self._n_end = n_end + self._description_pattern = "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" + return self._description_pattern.format( + n_start=self._n_start, n_end=self._n_end, prompt_to_repeat=self._prompt_to_repeat + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"n_start": self._n_start, "n_end": self._n_end, "prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["n_start", "n_end", "prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the expected number of phrases with the correct modifications.""" + return ( + value.strip().lower().split() == self._prompt_to_repeat.strip().lower().split()[self._n_start : self._n_end] + ) + + +class TitleCaseChecker(Instruction): + "Write the entire response in title case (capitalize the first letter of every major word)." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Write the entire response in title case (capitalize the first letter of every major word)." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is in title case. + + Args: + value: A string representing the response. + + Returns: + True if the response is in title case; + otherwise, False. + """ + words = instructions_util.word_tokenize(value) + for word in words: + if not word or not word[0].isalpha(): + continue + if len(word) == 1: + if word[0].islower(): + return False + continue + if word[0].isupper() and word[1:].islower(): + continue + elif (word[0].islower() and word[1:].isupper()) or (word[0].islower() and word[1:].islower()): + return False + return True + + +class OutputTemplateChecker(Instruction): + "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response follows the specified template. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the specified template; + otherwise, False. + """ + return "My Answer:" in value and "My Conclusion:" in value and "Future Outlook:" in value + + +class NoWhitespaceChecker(Instruction): + "The output should not contain any whitespace." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The output should not contain any whitespace." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains any whitespace. + + Args: + value: A string representing the response. + + Returns: + True if the response contains no whitespace; + otherwise, False. + """ + return not any(char.isspace() for char in value) diff --git a/playground/eval/benchmarks/IFBench/official/instructions_registry.py b/playground/eval/benchmarks/IFBench/official/instructions_registry.py new file mode 100644 index 00000000..2d27a499 --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/instructions_registry.py @@ -0,0 +1,78 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +from . import instructions + +INSTRUCTION_DICT = { + "count:word_count_range": instructions.WordCountRangeChecker, + "count:unique_word_count": instructions.UniqueWordCountChecker, + "ratio:stop_words": instructions.StopWordPercentageChecker, + "ratio:sentence_type": instructions.SentTypeRatioChecker, + "ratio:sentence_balance": instructions.SentBalanceChecker, + "count:conjunctions": instructions.ConjunctionCountChecker, + "count:person_names": instructions.PersonNameCountChecker, + "ratio:overlap": instructions.NGramOverlapChecker, + "count:numbers": instructions.NumbersCountChecker, + "words:alphabet": instructions.AlphabetLoopChecker, + "words:vowel": instructions.SingleVowelParagraphChecker, + "words:consonants": instructions.ConsonantClusterChecker, + "sentence:alliteration_increment": instructions.IncrementingAlliterationChecker, + "words:palindrome": instructions.PalindromeChecker, + "count:punctuation": instructions.PunctuationCoverChecker, + "format:parentheses": instructions.NestedParenthesesChecker, + "format:quotes": instructions.NestedQuotesChecker, + "words:prime_lengths": instructions.PrimeLengthsChecker, + "format:options": instructions.OptionsResponseChecker, + "format:newline": instructions.NewLineWordsChecker, + "format:emoji": instructions.EmojiSentenceChecker, + "ratio:sentence_words": instructions.CharacterCountUniqueWordsChecker, + "count:words_japanese": instructions.NthWordJapaneseChecker, + "words:start_verb": instructions.StartWithVerbChecker, + "words:repeats": instructions.LimitedWordRepeatChecker, + "sentence:keyword": instructions.IncludeKeywordChecker, + "count:pronouns": instructions.PronounCountChecker, + "words:odd_even_syllables": instructions.AlternateParitySyllablesChecker, + "words:last_first": instructions.LastWordFirstNextChecker, + "words:paragraph_last_first": instructions.ParagraphLastFirstWordMatchChecker, + "sentence:increment": instructions.IncrementingWordCountChecker, + "words:no_consecutive": instructions.NoConsecutiveFirstLetterChecker, + "format:line_indent": instructions.IndentStairsChecker, + "format:quote_unquote": instructions.QuoteExplanationChecker, + "format:list": instructions.SpecialBulletPointsChecker, + "format:thesis": instructions.ItalicsThesisChecker, + "format:sub-bullets": instructions.SubBulletPointsChecker, + "format:no_bullets_bullets": instructions.SomeBulletPointsChecker, + "custom:multiples": instructions.PrintMultiplesChecker, + "custom:mcq_count_length": instructions.MultipleChoiceQuestionsChecker, + "custom:reverse_newline": instructions.ReverseNewlineChecker, + "custom:word_reverse": instructions.WordReverseOrderChecker, + "custom:character_reverse": instructions.CharacterReverseOrderChecker, + "custom:sentence_alphabet": instructions.SentenceAlphabetChecker, + "custom:european_capitals_sort": instructions.EuropeanCapitalsSortChecker, + "custom:csv_city": instructions.CityCSVChecker, + "custom:csv_special_character": instructions.SpecialCharacterCSVChecker, + "custom:csv_quotes": instructions.QuotesCSVChecker, + "custom:date_format_list": instructions.DateFormatListChecker, + "count:keywords_multiple": instructions.KeywordsMultipleChecker, + "words:keywords_specific_position": instructions.KeywordSpecificPositionChecker, + "words:words_position": instructions.WordsPositionChecker, + "repeat:repeat_change": instructions.RepeatChangeChecker, + "repeat:repeat_simple": instructions.RepeatSimpleChecker, + "repeat:repeat_span": instructions.RepeatSpanChecker, + "format:title_case": instructions.TitleCaseChecker, + "format:output_template": instructions.OutputTemplateChecker, + "format:no_whitespace": instructions.NoWhitespaceChecker, +} diff --git a/playground/eval/benchmarks/IFBench/official/instructions_util.py b/playground/eval/benchmarks/IFBench/official/instructions_util.py new file mode 100644 index 00000000..25a53dc5 --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/instructions_util.py @@ -0,0 +1,1631 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility library of instructions.""" + +import functools +import random +import re + +from . import resource_config + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + + +@functools.lru_cache(maxsize=1) +def get_nltk(): + import nltk + + nltk_data_dir = resource_config.nltk_data_path() + nltk_data_dir_str = str(nltk_data_dir) + if nltk_data_dir_str not in nltk.data.path: + nltk.data.path.insert(0, nltk_data_dir_str) + return nltk + + +@functools.lru_cache(maxsize=1) +def ensure_nltk_resources(): + nltk = get_nltk() + missing: list[str] = [] + for resource_name in ( + "tokenizers/punkt", + "tokenizers/punkt_tab", + "corpora/stopwords", + "taggers/averaged_perceptron_tagger_eng", + ): + try: + nltk.data.find(resource_name) + except LookupError: + missing.append(resource_name) + if missing: + missing_desc = ", ".join(missing) + raise FileNotFoundError( + "IFBench NLTK resources are missing. " + f"Expected them under {resource_config.nltk_data_path()}. " + f"Missing: {missing_desc}." + ) + return True + + +def word_tokenize(text): + ensure_nltk_resources() + return get_nltk().word_tokenize(text) + + +def pos_tag(tokens): + ensure_nltk_resources() + return get_nltk().pos_tag(tokens) + + +def split_into_sentences(text): + """Split the text into sentences using NLTK. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + ensure_nltk_resources() + return get_nltk().sent_tokenize(text) + + +def count_words(text): + """Counts the number of words.""" + tokenizer = get_nltk().tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + + +def count_stopwords(text): + """Counts the number of stopwords.""" + """Counts the number of stopwords.""" + ensure_nltk_resources() + stopwords = get_nltk().corpus.stopwords.words("english") + tokenizer = get_nltk().tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_stopwords = len([t for t in tokens if t.lower() in stopwords]) + return num_stopwords + + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) diff --git a/playground/eval/benchmarks/IFBench/official/resource_config.py b/playground/eval/benchmarks/IFBench/official/resource_config.py new file mode 100644 index 00000000..880b3664 --- /dev/null +++ b/playground/eval/benchmarks/IFBench/official/resource_config.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_resource_root: Path | None = None + + +def _clear_dependent_caches() -> None: + instructions_util = sys.modules.get("playground.eval.benchmarks.IFBench.official.instructions_util") + if instructions_util is None: + return + for attr_name in ("get_nltk", "ensure_nltk_resources"): + cached_fn = getattr(instructions_util, attr_name, None) + if cached_fn is not None and hasattr(cached_fn, "cache_clear"): + cached_fn.cache_clear() + + +def set_resource_root(path: str | Path) -> None: + global _resource_root + next_root = Path(path) + if _resource_root == next_root: + return + _resource_root = next_root + _clear_dependent_caches() + + +def get_resource_root() -> Path: + if _resource_root is None: + raise RuntimeError( + "IFBench resource root is not configured. " + "Call set_resource_root(...) before using the official IFBench helpers." + ) + return _resource_root + + +def prompt_file_path() -> Path: + return get_resource_root() / "IFBench_test.jsonl" + + +def nltk_data_path() -> Path: + return get_resource_root() / "nltk_data" diff --git a/playground/eval/benchmarks/MMLUPro/__init__.py b/playground/eval/benchmarks/MMLUPro/__init__.py new file mode 100644 index 00000000..601efc23 --- /dev/null +++ b/playground/eval/benchmarks/MMLUPro/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import MMLUProBenchmark + +__all__ = ["MMLUProBenchmark"] diff --git a/playground/eval/benchmarks/MMLUPro/benchmark.py b/playground/eval/benchmarks/MMLUPro/benchmark.py new file mode 100644 index 00000000..c64a70e2 --- /dev/null +++ b/playground/eval/benchmarks/MMLUPro/benchmark.py @@ -0,0 +1,17 @@ +import re + +from playground.eval.benchmarks.GPQADiamond import GPQADiamondBenchmark +from steptronoss.generation.base_benchmark import Generated + + +class MMLUProBenchmark(GPQADiamondBenchmark): + dataset_name = "MMLU_PRO" + + _OPTION_PATTERN = re.compile(r"(? bool: + if result.error: + return False + predicted = MMLUProBenchmark._extract_choice(result.response) + return predicted == answer.strip().upper() diff --git a/playground/eval/benchmarks/common.py b/playground/eval/benchmarks/common.py new file mode 100644 index 00000000..24200cf7 --- /dev/null +++ b/playground/eval/benchmarks/common.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import copy +import json +import math +import random +from collections.abc import Callable +from typing import Protocol + +from steptronoss.generation.base_benchmark import ( + BaseBenchmark, + BaseMetric, + BenchmarkMeta, + ChatMessage, + EvaluationCase, + EvaluationMeta, + Generated, + GroundTruth, + GroundTruthValue, + JsonObject, + JsonValue, + Messages, + Prompt, +) + + +class ChatTokenizer(Protocol): + """Minimal tokenizer contract needed by jsonl-backed chat benchmarks.""" + + def apply_chat_template( + self, + messages: Messages, + *, + tokenize: bool, + add_generation_prompt: bool, + **kwargs: JsonValue, + ) -> list[int]: ... + + def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]: ... + + +def _estimate_pass_at_k(num_samples: int, num_successes: int, k: int) -> float: + """Estimate pass@k with the standard unbiased HumanEval-style estimator. + + For one benchmark item with `n` sampled completions and `c` successful ones: + + - `pass@k = 1 - C(n-c, k) / C(n, k)` + - if `n - c < k`, then `pass@k = 1` + + This estimator is order-invariant: only the number of successful samples + matters, not which `run_index` first succeeded. When fewer than `k` + samples are available, clip `k` to the available sample count. + """ + + if num_samples <= 0: + raise ValueError("num_samples must be positive") + if not 0 <= num_successes <= num_samples: + raise ValueError("num_successes must be within [0, num_samples]") + if k <= 0: + raise ValueError("k must be positive") + + effective_k = min(k, num_samples) + if num_successes == 0: + return 0.0 + if num_samples - num_successes < effective_k: + return 1.0 + + return 1.0 - math.prod( + 1.0 - effective_k / denominator for denominator in range(num_samples - num_successes + 1, num_samples + 1) + ) + + +def _parse_json_value(raw_value: object) -> JsonValue: + if raw_value is None or isinstance(raw_value, (bool, int, float, str)): + return raw_value + if isinstance(raw_value, list): + return [_parse_json_value(value) for value in raw_value] + if isinstance(raw_value, dict): + parsed_object: JsonObject = {} + for key, value in raw_value.items(): + if not isinstance(key, str): + raise TypeError(f"JSON object key must be str, got {type(key).__name__}") + parsed_object[key] = _parse_json_value(value) + return parsed_object + raise TypeError(f"Unsupported JSON value type: {type(raw_value).__name__}") + + +def _parse_ground_truth(raw_ground_truth: object) -> GroundTruth: + if not isinstance(raw_ground_truth, dict): + raise TypeError("ground_truth must be an object") + value = raw_ground_truth.get("value") + if not isinstance(value, dict): + raise TypeError("ground_truth.value must be an object") + item_id = value.get("item_id") + dataset = value.get("dataset") + if not isinstance(item_id, str) or not isinstance(dataset, str): + raise TypeError("ground_truth.value.item_id and dataset must be strings") + ground_truth_value: GroundTruthValue = {"item_id": item_id, "dataset": dataset} + return {"value": ground_truth_value} + + +def _parse_message(raw_message: object) -> ChatMessage: + if not isinstance(raw_message, dict): + raise TypeError("message must be an object") + role = raw_message.get("role") + content = raw_message.get("content") + if not isinstance(role, str) or not isinstance(content, str): + raise TypeError("message.role and message.content must be strings") + message: ChatMessage = {"role": role, "content": content} + if "ground_truth" in raw_message: + message["ground_truth"] = _parse_ground_truth(raw_message["ground_truth"]) + return message + + +def _parse_messages(raw_messages: object) -> Messages: + if not isinstance(raw_messages, list): + raise TypeError("messages must be a list") + return [_parse_message(raw_message) for raw_message in raw_messages] + + +def _parse_source_item(raw_source_item: object) -> tuple[str, JsonObject]: + if not isinstance(raw_source_item, dict): + raise TypeError("source_item must be an object") + + item_id = raw_source_item.get("item_id") + if not isinstance(item_id, str): + raise TypeError("source_item.item_id must be a string") + context: JsonObject = {} + for key, value in raw_source_item.items(): + if not isinstance(key, str): + raise TypeError(f"source_item key must be str, got {type(key).__name__}") + if key == "item_id": + continue + context[key] = _parse_json_value(value) + return item_id, context + + +def _parse_record(raw_record: object) -> tuple[str, Messages, str, JsonObject]: + if not isinstance(raw_record, dict): + raise TypeError("benchmark record must be an object") + dataset = raw_record.get("dataset") + if not isinstance(dataset, str): + raise TypeError("benchmark record.dataset must be a string") + return ( + dataset, + _parse_messages(raw_record.get("messages")), + *_parse_source_item(raw_record.get("source_item")), + ) + + +class JsonlChatBenchmark(BaseBenchmark): + dataset_name: str | None = None + + def __init__( + self, + data_path: str, + tokenizer: ChatTokenizer, + sample_per_prompt: int, + down_sample_to: int | None = None, + shuffle_prompts: bool = False, + chat_template_options: JsonObject | None = None, + ): + if not self.dataset_name: + raise ValueError(f"{type(self).__name__} must define dataset_name") + self.name = self.dataset_name + self.data_path = data_path + self.tokenizer = tokenizer + self.sample_per_prompt = sample_per_prompt + self.down_sample_to = down_sample_to + self.shuffle_prompts = shuffle_prompts + self.chat_template_options = chat_template_options + self._records_cache: list[tuple[str, Messages, str, JsonObject]] | None = None + + @staticmethod + def _normalize_messages(messages: Messages) -> Messages: + normalized = _parse_messages(json.loads(json.dumps(messages))) + if normalized and normalized[-1].get("role") == "assistant" and normalized[-1].get("content", "") == "": + return normalized[:-1] + return normalized + + def _load_records(self) -> list[tuple[str, Messages, str, JsonObject]]: + if self._records_cache is None: + with open(self.data_path, encoding="utf-8") as fin: + self._records_cache = [_parse_record(json.loads(line)) for line in fin] + records = list(self._records_cache) + if self.shuffle_prompts: + rng = random.Random(1234) + rng.shuffle(records) + if self.down_sample_to is not None: + records = records[: self.down_sample_to] + return records + + def get_cases(self) -> list[EvaluationCase]: + records = self._load_records() + cases: list[EvaluationCase] = [] + tokenization_options = {} if self.chat_template_options is None else dict(self.chat_template_options) + + for prompt_index, (_dataset, messages, item_id, context) in enumerate(records): + normalized_messages = self._normalize_messages(messages) + tokens = self.tokenizer.apply_chat_template( + normalized_messages, + tokenize=True, + add_generation_prompt=True, + **tokenization_options, + ) + for run_index in range(self.sample_per_prompt): + prompt = Prompt( + messages=copy.deepcopy(normalized_messages), + prompt_token_count=len(tokens), + ) + benchmark = BenchmarkMeta( + benchmark_name=self.name, + item_id=item_id, + context=copy.deepcopy(context), + ) + evaluation = EvaluationMeta( + prompt_index=prompt_index, + run_index=run_index, + ) + cases.append(EvaluationCase(prompt=prompt, benchmark=benchmark, evaluation=evaluation)) + + return cases + + def count_response_tokens(self, response: str) -> int: + return len(self.tokenizer.encode(response, add_special_tokens=False)) + + @staticmethod + def _is_success(result: Generated) -> bool: + return result.error is None and result.finish_reason == "stop" + + @staticmethod + def _build_metric( + results: list[Generated], + sample_values: list[float], + sample_per_prompt: int, + is_success_fn: Callable[[Generated], bool], + ) -> BaseMetric: + if not results: + return BaseMetric(score_avg=math.nan, score_std=math.nan, pass_at_k={}) + + score_avg = sum(sample_values) / len(sample_values) + variance = sum((value - score_avg) ** 2 for value in sample_values) / len(sample_values) + score_std = math.sqrt(variance) + grouped: dict[str, list[Generated]] = {} + for result in results: + grouped.setdefault(result.case.benchmark.item_id, []).append(result) + + pass_at_k: dict[int, float] = {} + candidate_ks = {1, sample_per_prompt} + k = 2 + while k < sample_per_prompt: + candidate_ks.add(k) + k *= 2 + for candidate_k in sorted(candidate_ks): + pass_probability_sum = 0.0 + for item_results in grouped.values(): + ordered = sorted(item_results, key=lambda item: item.case.evaluation.run_index) + num_successes = sum(1 for item in ordered if is_success_fn(item)) + pass_probability_sum += _estimate_pass_at_k( + num_samples=len(ordered), + num_successes=num_successes, + k=candidate_k, + ) + pass_at_k[candidate_k] = pass_probability_sum / max(len(grouped), 1) + + return BaseMetric(score_avg=score_avg, score_std=score_std, pass_at_k=pass_at_k) + + def evaluate(self, results: list[Generated]) -> BaseMetric: + sample_values = [1.0 if self._is_success(result) else 0.0 for result in results] + return self._build_metric( + results=results, + sample_values=sample_values, + sample_per_prompt=self.sample_per_prompt, + is_success_fn=self._is_success, + ) diff --git a/playground/eval/eval_sets/simple_eval.py b/playground/eval/eval_sets/simple_eval.py new file mode 100644 index 00000000..6f234855 --- /dev/null +++ b/playground/eval/eval_sets/simple_eval.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +import asyncio +import copy +import hashlib +import json +import os +import random +from datetime import datetime, timezone +from pprint import pformat +from typing import Literal, TypeAlias + +import aiohttp +from configurize import Ref, writable_property +from diskcache import Cache +from loguru import logger + +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.exp.base_exp import TokenizerConfig +from steptronoss.exp.gen_eval import GenableEvalConfig +from steptronoss.generation.async_generation import GenerationController +from steptronoss.generation.base_benchmark import ( + CompletionChoice, + CompletionMessage, + EvaluationCase, + Generated, + JsonObject, + SamplingParams, +) +from steptronoss.generation.base_generatable import EndpointGetter, GenableItem, ModelNameGetter +from steptronoss.utils.general import GroupedProgressBar, retry_on + +ChatTemplateArgValue: TypeAlias = str | bool | int +ChatTemplateArgs: TypeAlias = dict[str, ChatTemplateArgValue] + + +class RetriableChatCompletionError(RuntimeError): + """Transient chat/completions failure that should be retried.""" + + +class SimpleChatGeneratable(GenableItem): + def __init__( + self, + case: EvaluationCase, + endpoint_getter: EndpointGetter, + model_name_getter: ModelNameGetter, + max_model_len: int, + sampling_params: SamplingParams, + ): + super().__init__() + self.case = case + self.endpoint_getter = endpoint_getter + self.model_name_getter = model_name_getter + + prompt = self.case.prompt + if prompt.messages is None: + raise TypeError("SimpleAirChatGeneratable requires Prompt.messages for chat/completions requests.") + if prompt.prompt_token_count is None: + raise ValueError("SimpleAirChatGeneratable requires Prompt.prompt_token_count for context budgeting.") + + remaining_context = max_model_len - prompt.prompt_token_count + if remaining_context < 1: + raise ValueError( + "Prompt for " + f"{self.case.benchmark.benchmark_name}:{self.case.benchmark.item_id} already uses " + f"{prompt.prompt_token_count} tokens, " + f"which leaves no room under max_model_len={max_model_len}." + ) + resolved_max_tokens = remaining_context + if sampling_params.max_tokens is not None: + resolved_max_tokens = min(sampling_params.max_tokens, remaining_context) + resolved_sampling_params = SamplingParams( + temperature=sampling_params.temperature, + top_p=sampling_params.top_p, + top_k=sampling_params.top_k, + max_tokens=resolved_max_tokens, + seed=self.case.evaluation.run_index if sampling_params.seed is None else sampling_params.seed, + extra_body=copy.deepcopy(sampling_params.extra_body), + ) + self.case = self.case.with_sampling_params(resolved_sampling_params) + + def fingerprint(self) -> str: + payload = { + "genable_type": type(self).__name__, + "prompt": self.case.prompt.to_dict(), + } + serialized_payload = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + digest = hashlib.sha256(serialized_payload.encode("utf-8")).hexdigest() + return f"{type(self).__name__}:v1:{digest}" + + @staticmethod + def _parse_choice(raw_choice: object) -> tuple[CompletionChoice, str, str | None]: + if not isinstance(raw_choice, dict): + raise TypeError(f"Unexpected vLLM choice payload: {raw_choice}") + + choice_raw: JsonObject = raw_choice + choice: CompletionChoice = {"raw": choice_raw} + + finish_reason = choice_raw.get("finish_reason") + if isinstance(finish_reason, str): + choice["finish_reason"] = finish_reason + + response_text = "" + reasoning_content: str | None = None + + message_value = choice_raw.get("message") + if isinstance(message_value, dict): + completion_message: CompletionMessage = {} + message_content = message_value.get("content") + if isinstance(message_content, str): + completion_message["content"] = message_content + response_text = message_content + raw_reasoning = message_value.get("reasoning_content") + if isinstance(raw_reasoning, str): + completion_message["reasoning_content"] = raw_reasoning + reasoning_content = raw_reasoning + if completion_message: + choice["message"] = completion_message + + text_value = choice_raw.get("text") + if isinstance(text_value, str): + choice["text"] = text_value + if not response_text: + response_text = text_value + + return choice, response_text, reasoning_content + + @retry_on( + (aiohttp.ClientError, asyncio.TimeoutError, RetriableChatCompletionError), + for_times=3, + delay=1.0, + max_delay=10.0, + backoff=2.0, + jitter=0.1, + ) + async def _post_chat_completion(self, payload: JsonObject) -> str: + timeout = aiohttp.ClientTimeout(total=7200.0) + async with ( + aiohttp.ClientSession(timeout=timeout, trust_env=False) as session, + session.post( + url=f"{self.endpoint_getter()}/v1/chat/completions", + json=payload, + ) as response, + ): + response_text = await response.text() + response_status = response.status + response_content_type = response.headers.get("content-type", "") + + if response_status != 200: + body_preview = response_text[:4000] + message = ( + f"vLLM chat/completions failed with HTTP {response_status} ({response_content_type}): {body_preview}" + ) + if response_status in {408, 425, 429} or 500 <= response_status < 600: + raise RetriableChatCompletionError(message) + raise RuntimeError(message) + return response_text + + async def generate(self) -> Generated: + prompt = self.case.prompt + if prompt.messages is None: + raise TypeError("SimpleAirChatGeneratable requires Prompt.messages for chat/completions requests.") + payload: JsonObject = { + "model": self.model_name_getter(), + "messages": prompt.messages, + } + if prompt.sampling_params is not None: + payload.update(prompt.sampling_params.to_dict()) + + response_text = await self._post_chat_completion(payload) + + try: + response_json = json.loads(response_text) + except json.JSONDecodeError as exc: + body_preview = response_text[:4000] + raise RuntimeError(f"vLLM chat/completions returned a non-JSON body {body_preview}") from exc + + if not isinstance(response_json, dict): + raise TypeError(f"Unexpected vLLM response: {response_json}") + if "choices" not in response_json: + raise ValueError(f"Unexpected vLLM response: {response_json}") + if not isinstance(response_json["choices"], list) or not response_json["choices"]: + raise ValueError(f"Unexpected vLLM response: {response_json}") + + choice, response_text, reasoning_content = self._parse_choice(response_json["choices"][0]) + return Generated( + case=self.case, + response=response_text, + choice=choice, + reasoning_content=reasoning_content, + ) + + +class SimpleBenchmarksEvalConfig(GenableEvalConfig): + selected_datasets: str | None = None + """Optional comma-separated dataset allowlist, e.g. "AIME2025,GPQA_DIAMOND".""" + + chat_template_args: ChatTemplateArgs = {"enable_thinking": True} + """Chat template kwargs used for prompt rendering and request.chat_template_kwargs.""" + + shuffle_prompts: bool = True + """If True, shuffle merged prompts from all benchmarks with seed 1234 to balance engine load.""" + + datasets_dir: str = "/oss/benchmarks/simple_benchmarks/datasets" + """Directory containing the simple benchmark datasets; IFBench expects `IFBENCH/` resources under this root.""" + + save_dir: str = Ref("..log_path") + """Directory used to save prediction dumps and summaries.""" + + predictions_cache_size_limit_bytes: int = 1 << 40 + """Diskcache size limit for prediction artifacts. Keep this large enough for full multi-benchmark runs.""" + + predictions_cache_eviction_policy: str = "none" + """Eviction policy for prediction cache. Use 'none' so completed generations are never silently culled.""" + + max_decode_steps: int = 128 * 1024 + """Maximum generated tokens per request before the request-level context cap is applied.""" + + num_concurrent_requests: int = 4096 + """Maximum number of in-flight genables allowed across GenerationController.""" + + tokenizer_cfg: TokenizerConfig + """Tokenizer config used to render prompts and count response tokens.""" + + router_addr_key: str = Ref("..vllm_cfg.router_addr_key") + """Key used to resolve the router address from Redis.""" + + model_name_template: str = Ref("..vllm_cfg.model_name_template") + """Template for the served model name in vLLM.""" + + max_model_len: int = Ref("..vllm_cfg.max_seq_len") + """Maximum total context length accepted by the backing vLLM server.""" + + rerun_level: Literal["error", "all"] | None = None + """Cache policy: None reuses all hits, 'error' reruns cached errors, 'all' reruns everything.""" + + def get_sampling_params(self, benchmark_sampling_params: SamplingParams | None) -> SamplingParams: + """Build request sampling params for one prompt, optionally merging benchmark overrides.""" + + if benchmark_sampling_params is None: + benchmark_sampling_params = SamplingParams() + + extra_body: JsonObject = {} + if self.chat_template_args is not None: + extra_body["chat_template_kwargs"] = copy.deepcopy(self.chat_template_args) + extra_body.update(copy.deepcopy(benchmark_sampling_params.extra_body)) + + return SamplingParams( + temperature=1.0 if benchmark_sampling_params.temperature is None else benchmark_sampling_params.temperature, + top_p=1.0 if benchmark_sampling_params.top_p is None else benchmark_sampling_params.top_p, + top_k=-1 if benchmark_sampling_params.top_k is None else benchmark_sampling_params.top_k, + max_tokens=self.max_decode_steps + if benchmark_sampling_params.max_tokens is None + else benchmark_sampling_params.max_tokens, + seed=benchmark_sampling_params.seed, + extra_body=extra_body, + ) + + @property + def predictions_path(self) -> str: + return os.path.join(self.save_dir, self.run_tag, "predictions") + + @property + def summary_path(self) -> str: + return os.path.join(self.save_dir, self.run_tag, "summary.json") + + @writable_property + def run_tag(self) -> str: + """Generation-cache namespace. Defaults to a UTC timestamp unless overridden.""" + + cache_tag = self.__dict__.get("_cache_tag_default") + if cache_tag is None: + cache_tag = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + self.__dict__["_cache_tag_default"] = cache_tag + return cache_tag + + def _select_benchmarks_by_name(self, benchmarks: list[JsonlChatBenchmark]) -> list[JsonlChatBenchmark]: + if self.selected_datasets is None: + return benchmarks + selected_text = self.selected_datasets.strip() + selected_names = [item.strip() for item in selected_text.split(",") if item.strip()] + if len(selected_names) == 0: + return benchmarks + + benchmark_map = {benchmark.name: benchmark for benchmark in benchmarks} + missing = [name for name in selected_names if name not in benchmark_map] + if missing: + raise ValueError(f"Unsupported selected_datasets={missing}. Supported datasets: {sorted(benchmark_map)}") + return [benchmark_map[name] for name in selected_names] + + def get_benchmarks(self) -> list[JsonlChatBenchmark]: + from playground.eval.benchmarks.AIME25 import AIME25Benchmark + from playground.eval.benchmarks.GPQADiamond import GPQADiamondBenchmark + from playground.eval.benchmarks.HMMT25 import HMMT25Benchmark + from playground.eval.benchmarks.IFBench import IFBenchBenchmark + from playground.eval.benchmarks.MMLUPro import MMLUProBenchmark + + chat_template_options = self.chat_template_args + tokenizer = self.tokenizer_cfg.build_tokenizer() + + benchmarks = [ + AIME25Benchmark( + data_path=os.path.join(self.datasets_dir, "AIME2025.jsonl"), + tokenizer=tokenizer, + sample_per_prompt=64, + chat_template_options=chat_template_options, + ), + GPQADiamondBenchmark( + data_path=os.path.join(self.datasets_dir, "GPQA_DIAMOND.jsonl"), + tokenizer=tokenizer, + sample_per_prompt=16, + chat_template_options=chat_template_options, + ), + HMMT25Benchmark( + data_path=os.path.join(self.datasets_dir, "HMMT25.jsonl"), + tokenizer=tokenizer, + sample_per_prompt=64, + chat_template_options=chat_template_options, + ), + IFBenchBenchmark( + data_path=os.path.join(self.datasets_dir, "IFBENCH"), + tokenizer=tokenizer, + sample_per_prompt=1, + ), + MMLUProBenchmark( + data_path=os.path.join(self.datasets_dir, "MMLU_PRO.jsonl"), + tokenizer=tokenizer, + sample_per_prompt=1, + chat_template_options=chat_template_options, + ), + ] + benchmarks = self._select_benchmarks_by_name(benchmarks) + return benchmarks + + def get_prompts(self) -> list[SimpleChatGeneratable]: + endpoint_getter = EndpointGetter(self.router_addr_key) + model_name_getter = ModelNameGetter(self.model_name_template) + + trainables: list[SimpleChatGeneratable] = [] + for benchmark in self.get_benchmarks(): + for case in benchmark.get_cases(): + sampling_params = self.get_sampling_params(case.prompt.sampling_params) + trainables.append( + SimpleChatGeneratable( + case=case, + endpoint_getter=endpoint_getter, + model_name_getter=model_name_getter, + max_model_len=self.max_model_len, + sampling_params=sampling_params, + ) + ) + + if self.shuffle_prompts: + random.Random(1234).shuffle(trainables) + + logger.info(f"Loaded {len(trainables)} generation requests from {self.datasets_dir}") + return trainables + + @staticmethod + def _rebind_generated(generated: Generated, genable: SimpleChatGeneratable) -> Generated: + """Reuse cached output with the current run's case metadata.""" + return Generated( + case=genable.case, + choice=None if generated.choice is None else copy.deepcopy(generated.choice), + response=generated.response, + reasoning_content=generated.reasoning_content, + error=generated.error, + ) + + def _summarize_results(self, results: list[Generated]) -> JsonObject: + benchmark_map = {benchmark.name: benchmark for benchmark in self.get_benchmarks()} + by_benchmark: dict[str, list[Generated]] = {name: [] for name in benchmark_map} + total_errors = 0 + + for result in results: + by_benchmark.setdefault(result.case.benchmark.benchmark_name, []).append(result) + if result.error: + total_errors += 1 + + summary: JsonObject = { + "total_requests": len(results), + "total_errors": total_errors, + "by_benchmark": {}, + } + + for benchmark_name, benchmark_results in by_benchmark.items(): + benchmark = benchmark_map[benchmark_name] + metric = benchmark.evaluate(benchmark_results) + finish_reason_counts: dict[str, int] = {} + total_chars = 0 + total_tokens = 0 + valid_count = 0 + error_count = 0 + for result in benchmark_results: + finish_reason_counts[result.finish_reason] = finish_reason_counts.get(result.finish_reason, 0) + 1 + if result.error: + error_count += 1 + continue + valid_count += 1 + total_chars += len(result.response) + total_tokens += benchmark.count_response_tokens(result.response) + + summary["by_benchmark"][benchmark_name] = { + "count": len(benchmark_results), + "errors": error_count, + "finish_reason_counts": finish_reason_counts, + "avg_response_chars": total_chars / max(valid_count, 1), + "avg_response_tokens": total_tokens / max(valid_count, 1), + "metric": metric.to_dict(), + } + + return summary + + def _generate(self, genables: list[SimpleChatGeneratable]) -> list[Generated]: + genables_by_fingerprint: dict[str, SimpleChatGeneratable] = {} + for raw_genable in genables: + fingerprint = raw_genable.fingerprint() + if fingerprint in genables_by_fingerprint: + raise ValueError(f"Duplicate generation fingerprint detected: {fingerprint}") + genables_by_fingerprint[fingerprint] = raw_genable + + group_totals = {benchmark.name: 0 for benchmark in self.get_benchmarks()} + for genable in genables: + group_name = genable.case.benchmark.benchmark_name + group_totals[group_name] = group_totals.get(group_name, 0) + 1 + progress_bar = GroupedProgressBar(group_totals) + controller: GenerationController | None = None + try: + pending_genables: list[SimpleChatGeneratable] = [] + reused_count = 0 + with Cache( + directory=self.predictions_path, + size_limit=self.predictions_cache_size_limit_bytes, + eviction_policy=self.predictions_cache_eviction_policy, + ) as generation_cache: + for fingerprint, genable in genables_by_fingerprint.items(): + should_generate = ( + self.rerun_level == "all" + or fingerprint not in generation_cache + or (self.rerun_level == "error" and generation_cache[fingerprint].error is not None) + ) + if not should_generate: + generation_cache[fingerprint] = self._rebind_generated(generation_cache[fingerprint], genable) + reused_count += 1 + progress_bar.update(genable.case.benchmark.benchmark_name) + continue + pending_genables.append(genable) + + logger.info( + f"Generation cache tag={self.run_tag} dir={self.predictions_path} " + f"reused={reused_count} missing={len(pending_genables)} " + f"total_unique={len(genables_by_fingerprint)} rerun_level={self.rerun_level}" + ) + + if pending_genables: + controller = GenerationController( + max_concurrent_genables=self.num_concurrent_requests, + ) + controller.set_tqdm(disabled=True, total=len(pending_genables), desc="Evaluation Requests") + for raw_genable, result in controller.generate(pending_genables): + genable: SimpleChatGeneratable = raw_genable + fingerprint = genable.fingerprint() + if isinstance(result, Exception): + generated = Generated(case=genable.case, error=repr(result)) + else: + generated = result + generation_cache[fingerprint] = generated + progress_bar.update(genable.case.benchmark.benchmark_name) + + results = [generation_cache[genable.fingerprint()] for genable in genables] + finally: + if controller is not None: + controller.shutdown() + progress_bar.close() + + return results + + def eval(self) -> JsonObject: + os.makedirs(self.save_dir, exist_ok=True) + genables = self.get_prompts() + results = self._generate(genables) + summary = self._summarize_results(results) + with open(self.summary_path, "w", encoding="utf-8") as fout: + json.dump(summary, fout, ensure_ascii=False, indent=2) + + logger.info(f"Saved predictions to {self.predictions_path}") + logger.info(f"Saved summary to {self.summary_path}") + logger.info(f"Eval summary: {pformat(summary)}") + return summary diff --git a/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py b/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py new file mode 100644 index 00000000..3684c5e5 --- /dev/null +++ b/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import os + +from loguru import logger + +from playground.eval.benchmarks.common import ChatTokenizer +from playground.eval.eval_sets.simple_eval import SimpleBenchmarksEvalConfig +from steptronoss.exp.base_exp import BaseExp, TokenizerConfig +from steptronoss.exp.inference import VLLMDeployConfig +from steptronoss.exp.resources import ResourceConfig, TaskSpec +from steptronoss.generation.vllm.vllm_router import VLLMRouterConfig + + +class Qwen3TokenizerConfig(TokenizerConfig): + tokenizer_path: str = "/oss/opensources_model/Qwen3-1.7B-Base/" + """Tokenizer directory for Qwen3-1.7B.""" + + def build_tokenizer(self) -> ChatTokenizer: + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(self.tokenizer_path, trust_remote_code=True) + + +class Qwen3SimpleEvalResourceConfig(ResourceConfig): + vllm_replica: int = 2 + """Number of vLLM worker tasks to launch for this eval.""" + + def __init__(self): + super().__init__() + self.command = "python {COMMAND}" + self.replica = 1 + self.gpu = 8 + self.node_type = "gpu" + self.vllm_replica = 2 + self._sync_task_specs() + + def _sync_task_specs(self) -> None: + self.task_specs = { + "evaluator": TaskSpec( + gpu=0, + node_type="cpu", + envs={"ROLE": "evaluator"}, + is_critical=True, + ), + "vllm": TaskSpec( + replica=self.vllm_replica, + envs={ + "ROLE": "vllm", + }, + ), + "router": TaskSpec( + gpu=0, + node_type="cpu", + envs={"ROLE": "router"}, + ), + } + + def find_leaf_task_specs(self): + self._sync_task_specs() + return super().find_leaf_task_specs() + + +class Qwen3_1p7BEvalVLLMDeployConfig(VLLMDeployConfig): + def __init__(self): + super().__init__() + self.model_config_path = "/oss/opensources_model/Qwen3-1.7B-Base/" + self.max_seq_len = 65536 + self.vllm_gpu_memory_utilization = 0.9 + + self.vllm_tp = 1 + self.vllm_dp = 8 + + self.vllm_enable_chunked_prefill = True + self.vllm_enable_prefix_caching = True + self.max_cache_size = 256 + + +class Qwen3_8BEvalVLLMDeployConfig(VLLMDeployConfig): + def __init__(self): + super().__init__() + self.model_config_path = "/oss/opensources_model/Qwen3-8B-Base/" + self.max_seq_len = 131072 + self.vllm_gpu_memory_utilization = 0.9 + + self.vllm_tp = 1 + self.vllm_dp = 8 + + self.vllm_enable_chunked_prefill = True + self.vllm_enable_prefix_caching = True + self.max_cache_size = 256 + + +class Qwen3SimpleEvalVLLMRouterConfig(VLLMRouterConfig): + routed_methods = { + "completions": ["POST"], + "chat/completions": ["POST"], + } + + +class Qwen3SimpleBenchmarksEvalConfig(SimpleBenchmarksEvalConfig): + tokenizer_cfg: Qwen3TokenizerConfig = Qwen3TokenizerConfig + """Tokenizer config kept for future prompt debugging and parity checks.""" + num_concurrent_requests = 4096 + + +class Exp(BaseExp): + vllm_cfg: VLLMDeployConfig = Qwen3_8BEvalVLLMDeployConfig + + resource_cfg: Qwen3SimpleEvalResourceConfig = Qwen3SimpleEvalResourceConfig + vllm_router_cfg: Qwen3SimpleEvalVLLMRouterConfig = Qwen3SimpleEvalVLLMRouterConfig + eval_cfg: Qwen3SimpleBenchmarksEvalConfig = Qwen3SimpleBenchmarksEvalConfig + + log_dir = "/oss/logs/" + + def entrypoint(self) -> None: + self.update_from_args() + role = os.environ.get("ROLE", "evaluator") + if role == "router": + logger.info("Starting vLLM router...") + self.vllm_router_cfg.run() + return + if role == "vllm": + logger.info("Starting vLLM worker...") + self.vllm_cfg.run_as_worker() + return + if role == "evaluator": + self.sanity_check() + logger.info("Waiting for vLLM servers to register...") + + self.vllm_cfg.build_cli().wait_for_server() + summary = self.eval_cfg.eval() + logger.info(f"Eval results: {summary}") + return + raise ValueError(f"Unknown ROLE: {role}") + + +if __name__ == "__main__": + Exp().entrypoint() diff --git a/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py b/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py new file mode 100644 index 00000000..1e57a543 --- /dev/null +++ b/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py @@ -0,0 +1,148 @@ +""" +SFT model trained use `playground/sft/step3/step3p5_flash_sft_step3_data_muon.py`: +- Benchmarks + | Benchmark | Requests | Trunc@128k | Avg chars | Avg toks | score_avg | score_std | pass@1 | + |--------------|------------|--------------|-------------|------------|-------------|-------------|----------| + | AIME2025 | 1920 | 0.73% | 539.76 | 219.71 | 0.94 | 0.23 | 0.94 | + | GPQA_DIAMOND | 3168 | 0.00% | 671.70 | 209.06 | 0.80 | 0.40 | 0.80 | + | HMMT25 | 1920 | 0.31% | 323.21 | 124.54 | 0.94 | 0.23 | 0.94 | + | IFBENCH | 294 | 9.86% | 1380.72 | 304.23 | 0.61 | 0.49 | 0.61 | + | MMLU_PRO | 12032 | 0.00% | 323.72 | 100.02 | 0.77 | 0.42 | 0.77 | + +- IFBENCH + | IFBENCH | per-prompt | count | custom | format | ratio | repeat | sentence | words | + |-----------|------------|---------|----------|----------|-------|----------|------------|---------| + | loose | 0.61 | 0.76 | 0.50 | 0.77 | 0.45 | 0.33 | 0.71 | 0.46 | + | strict | 0.57 | 0.76 | 0.40 | 0.70 | 0.45 | 0.33 | 0.64 | 0.44 | +""" + +from __future__ import annotations + +import os + +from configurize import Ref +from loguru import logger + +from playground.eval.benchmarks.common import ChatTokenizer +from playground.eval.eval_sets.simple_eval import SimpleBenchmarksEvalConfig +from steptronoss.exp.base_exp import BaseExp, TokenizerConfig +from steptronoss.exp.inference import VLLMDeployConfig +from steptronoss.exp.resources import ResourceConfig, TaskSpec +from steptronoss.generation.vllm.vllm_router import VLLMRouterConfig + + +class Step3p5TokenizerConfig(TokenizerConfig): + tokenizer_path: str = Ref("...vllm_cfg.tokenizer_path") + """Tokenizer directory for the target Step3.5 model family.""" + + def build_tokenizer(self) -> ChatTokenizer: + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(self.tokenizer_path, trust_remote_code=True) + + +class Step3p5SimpleEvalResourceConfig(ResourceConfig): + vllm_replica: int = 1 + """Number of Step3.5 vLLM worker tasks to launch.""" + + def __init__(self): + super().__init__() + self.command = ".venv/bin/python {COMMAND}" + workspace_venv_bin = os.path.join(os.getcwd(), ".venv", "bin") + current_path = os.environ.get("PATH", "") + self.envs["PATH"] = f"{workspace_venv_bin}:{current_path}" if current_path else workspace_venv_bin + self.replica = 1 + self.gpu = 8 + self.node_type = "gpu" + self.vllm_replica = 1 + self._sync_task_specs() + + def _sync_task_specs(self) -> None: + self.task_specs = { + "evaluator": TaskSpec( + gpu=0, + node_type="cpu", + envs={"ROLE": "evaluator"}, + is_critical=True, + ), + "vllm": TaskSpec( + replica=self.vllm_replica, + envs={"ROLE": "vllm"}, + ), + "router": TaskSpec( + gpu=0, + node_type="cpu", + envs={"ROLE": "router"}, + ), + } + + def find_leaf_task_specs(self): + self._sync_task_specs() + return super().find_leaf_task_specs() + + +class Step3p5SimpleEvalVLLMDeployConfig(VLLMDeployConfig): + def __init__(self): + super().__init__() + self.model_config_path = "/oss/checkpoints/step3_flash_sft_step3_data_muon/it4716/hf_vllm/" + self.tokenizer_path = "/oss/tokenizers/step3p5_flash_sft/" + self.reasoning_parser = "step3p5" + self.max_seq_len = 128 * 1024 + self.vllm_gpu_memory_utilization = 0.9 + + self.vllm_tp = 8 + self.vllm_dp = 1 + + self.vllm_enable_chunked_prefill = True + self.vllm_enable_prefix_caching = True + self.max_cache_size = 256 + + +class Step3p5SimpleEvalVLLMRouterConfig(VLLMRouterConfig): + routed_methods = { + "completions": ["POST"], + "chat/completions": ["POST"], + } + + +class Step3p5SimpleBenchmarksEvalConfig(SimpleBenchmarksEvalConfig): + tokenizer_cfg: Step3p5TokenizerConfig = Step3p5TokenizerConfig + """Tokenizer config for Step3.5 prompt rendering and token counting.""" + + num_concurrent_requests = 4096 + max_decode_steps = 128 * 1024 + + +class Exp(BaseExp): + vllm_cfg: VLLMDeployConfig = Step3p5SimpleEvalVLLMDeployConfig + + resource_cfg: Step3p5SimpleEvalResourceConfig = Step3p5SimpleEvalResourceConfig + vllm_router_cfg: Step3p5SimpleEvalVLLMRouterConfig = Step3p5SimpleEvalVLLMRouterConfig + eval_cfg: Step3p5SimpleBenchmarksEvalConfig = Step3p5SimpleBenchmarksEvalConfig + + log_dir = "/oss/logs/" + + def entrypoint(self) -> None: + self.update_from_args() + role = os.environ.get("ROLE", "evaluator") + if role == "router": + logger.info("Starting vLLM router...") + self.vllm_router_cfg.run() + return + if role == "vllm": + logger.info("Starting vLLM worker...") + self.vllm_cfg.run_as_worker() + return + if role == "evaluator": + self.sanity_check() + logger.info("Waiting for vLLM servers to register...") + + self.vllm_cfg.build_cli().wait_for_server() + summary = self.eval_cfg.eval() + logger.info(f"Eval results: {summary}") + return + raise ValueError(f"Unknown ROLE: {role}") + + +if __name__ == "__main__": + Exp().entrypoint() diff --git a/playground/rlvr/qwen3_1p5b_rlvr_math.py b/playground/rlvr/qwen3_1p5b_rlvr_math.py index 0057ac46..405b26f3 100644 --- a/playground/rlvr/qwen3_1p5b_rlvr_math.py +++ b/playground/rlvr/qwen3_1p5b_rlvr_math.py @@ -46,7 +46,6 @@ def __init__( self, endpoint_getter, model_name_getter, - sampling_params: dict[str, Any], prompt_text: str, gt: str, max_tokens: int, @@ -54,7 +53,6 @@ def __init__( super().__init__() self.endpoint_getter = endpoint_getter self.model_name_getter = model_name_getter - self.sampling_params = sampling_params self.prompt_text = prompt_text self.gt = gt self.max_tokens = max_tokens @@ -63,7 +61,7 @@ def __next__(self) -> dict[str, Any]: return SimpleTrainable( endpoint_getter=self.endpoint_getter, model_name_getter=self.model_name_getter, - sampling_params=self.sampling_params, + sampling_params={}, prompt_text=self.prompt_text, gt=self.gt, max_tokens=self.max_tokens, @@ -88,7 +86,6 @@ def build_dataloader(self, dp_rank=0, dp_size=1): endpoint_getter = EndpointGetter(vllm_cfg.router_addr_key) model_name_getter = ModelNameGetter(vllm_cfg.model_name_template) max_tokens = 1024 - sampling_params = vllm_cfg.get_sampling_params({"max_tokens": max_tokens}) tokenizer = self.build_tokenizer() messages = [ { @@ -100,7 +97,6 @@ def build_dataloader(self, dp_rank=0, dp_size=1): return FakeGenableGenerator( endpoint_getter=endpoint_getter, model_name_getter=model_name_getter, - sampling_params=sampling_params, prompt_text=prompt_text, gt="3", max_tokens=max_tokens, diff --git a/playground/rlvr/simple_trainable.py b/playground/rlvr/simple_trainable.py index 6c39e342..7eed6337 100644 --- a/playground/rlvr/simple_trainable.py +++ b/playground/rlvr/simple_trainable.py @@ -1,3 +1,5 @@ +import hashlib +import json import re from collections.abc import Callable from typing import Any @@ -42,8 +44,14 @@ def __init__( self.gt = gt self.endpoint_getter = endpoint_getter self.model_name_getter = model_name_getter - self.sampling_params = sampling_params self.max_tokens = max_tokens + self.sampling_params = { + "temperature": 1.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": max_tokens, + } + self.sampling_params.update(sampling_params) @staticmethod def _extract_boxed(answer: str) -> str: @@ -69,13 +77,24 @@ async def generate(self) -> dict[str, Any]: timeout=aiohttp.ClientTimeout(total=7200.0), ) as response: response = await response.json() - + if "choices" not in response: + raise ValueError(f"Unexpected vLLM response: {response}") choice = response["choices"][0] return { "choice": choice, "prompt": self.prompt_text, } + def fingerprint(self) -> str: + payload = { + "genable_type": type(self).__name__, + "prompt": self.prompt_text, + "sampling_params": self.sampling_params, + } + serialized_payload = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(serialized_payload.encode("utf-8")).hexdigest() + return f"{type(self).__name__}:v1:{digest}" + async def generate_for_train(self): generated = await self.generate() choice = generated["choice"] diff --git a/playground/sft/step3/step3p5_flash_sft_step3_data_muon.py b/playground/sft/step3/step3p5_flash_sft_step3_data_muon.py index 911058f3..0bc3da06 100644 --- a/playground/sft/step3/step3p5_flash_sft_step3_data_muon.py +++ b/playground/sft/step3/step3p5_flash_sft_step3_data_muon.py @@ -219,6 +219,7 @@ def configure_optimizable(self): moe_weighted_gather="triton", TokenDispatcher="deep_ep", grouped_gemm="nv_grouped_gemm", + # grouped_gemm="function_imple", # slower fallback AttentionCore="flash-attn-3", ) diff --git a/pyproject.toml b/pyproject.toml index 6e40e138..cf529269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,10 @@ classifiers = [ dependencies = [ "aiohttp", "aioredis", + "emoji", "boto3", "configurize==0.2.0", + "diskcache>=5.6.3", "einops", "fastapi", "hjson", @@ -29,22 +31,24 @@ dependencies = [ "loguru", "megfile", "msgpack", + "nltk", "numpy", "packaging", "pyyaml", "redis", "safetensors", "setuptools", + "syllapy", "tabulate", - "torch==2.9.0", + "torch>=2.9.0", "tqdm", - "transformers<5.0", - "triton==3.5.0", + "transformers", + "triton>=3.5.0", "uvicorn", "wandb", "httpx", "pip>=25.3", - "vllm>=0.11", + "vllm>=0.16", "tensorboard>=2.20.0", "asciichartpy>=1.5.25", ] diff --git a/steptronoss/exp/gen_eval.py b/steptronoss/exp/gen_eval.py new file mode 100644 index 00000000..7e4fdad2 --- /dev/null +++ b/steptronoss/exp/gen_eval.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from configurize import Config + +if TYPE_CHECKING: + from steptronoss.generation.base_generatable import GenableItem + + +class GenableEvalConfig(Config): + """使用Genable Eval流程非常简单，无需和trainer对接，直接使用GenerationController生成并计算metric即可。""" + + record_eval_rollout: bool = False + """If True, trainer may pass a prefix to dump eval rollouts.""" + + def get_prompts(self) -> list[GenableItem]: + raise NotImplementedError + + def eval(self) -> dict: + from steptronoss.generation.async_generation import GenerationController + + controller = GenerationController() + + prompts = self.get_prompts() + results = [] + for _genable, result in controller.generate(prompts): + results.append(result) + return results diff --git a/steptronoss/exp/inference.py b/steptronoss/exp/inference.py index 17bb455e..9b7a9364 100644 --- a/steptronoss/exp/inference.py +++ b/steptronoss/exp/inference.py @@ -2,6 +2,7 @@ import torch from configurize import Config, Ref +from loguru import logger class BaseInferenceConfig(Config): @@ -12,15 +13,6 @@ class BaseInferenceConfig(Config): max_cache_size = 64 """Inference Decode BatchSize, max_num_seqs for paged attention""" - temperature: float = 1.0 - top_p: float = 1.0 - top_k: int = 0 - - eos_ids: list[int] = [] - """stop if tokens[-1] == eos_id. (work for generate only)""" - stop_strings: str | list[str] = [] - """stop if trajectory.endswith(any_of(stop_string))""" - shuffle_all_samples = True """shuffle all generated samples after finish generation.""" @@ -78,26 +70,17 @@ class VLLMDeployConfig(BaseInferenceConfig): vllm_hf_overrides: dict = {} """The hf_overrides to use to override the model config in huggingface for vLLM.""" - vllm_sampling_params: dict = {} - """The sampling params to use for vLLM.""" - vllm_mtp_num_tokens: int = 0 vllm_mtp_method: str = "" """The method to use for vLLM MTP.""" def sanity_check(self): - """Validate configuration parameters. - - Validates that resource_cfg.gpu is evenly divisible by inference_tp. - This ensures mp_run can create equal-sized processes. - - Examples: - - ✓ gpu=8, inference_tp=2: 8 % 2 = 0 (4 processes) - - ✓ gpu=8, inference_tp=4: 8 % 4 = 0 (2 processes) - - ✓ gpu=8, inference_tp=8: 8 % 8 = 0 (1 process, standard deployment) - - ✗ gpu=8, inference_tp=3: 8 % 3 ≠ 0 (invalid) - """ super().sanity_check() + if torch.cuda.is_available(): + if self.vllm_dp * self.vllm_tp != (gpus := torch.cuda.device_count()): + logger.warning( + f"Using DP={self.vllm_dp} & TP={self.vllm_tp} for vLLM, but detected {gpus} GPU on node!" + ) if self.enable_auto_tool_choice: assert self.toolcall_parser @@ -186,37 +169,6 @@ def get_entrypoint_command_and_envs(self): return cmd, envs - def get_sampling_params(self, override_params: dict = {}): - - # Handle top_k - self.top_k = self.top_k - if self.top_k <= 0: - self.top_k = -1 - - # Handle stop strings - stop_strings = self.stop_strings - if stop_strings is not None: - if isinstance(stop_strings, str): - stop_strings = [stop_strings] - if isinstance(stop_strings, list) and len(stop_strings) == 0: - stop_strings = None - - # Handle stop token ids - eos_ids = self.eos_ids - if len(eos_ids) == 0: - eos_ids = None - - sampling_params = dict( - temperature=self.temperature, - top_p=self.top_p, - top_k=self.top_k, - stop=stop_strings, - stop_token_ids=eos_ids, - ) - sampling_params.update(self.vllm_sampling_params) - sampling_params.update(override_params) - return sampling_params - def deploy_training_model(self, models: list[torch.nn.Module]): from steptronoss.checkpointing.hf_checkpoint import dump_safetensors diff --git a/steptronoss/generation/async_generation.py b/steptronoss/generation/async_generation.py index ad25db52..11261e15 100644 --- a/steptronoss/generation/async_generation.py +++ b/steptronoss/generation/async_generation.py @@ -1,17 +1,17 @@ import asyncio import multiprocessing as mp import threading -import time +from collections import deque from collections.abc import Callable, Iterable from queue import Empty, Queue -from typing import Any, NoReturn, Optional +from typing import Any, NoReturn from uuid import uuid4 from loguru import logger from tqdm import tqdm from steptronoss.exp.rl import EnvTrajectory -from steptronoss.generation.base_generatable import TrainableItem +from steptronoss.generation.base_generatable import GenableItem, TrainableItem from steptronoss.utils import run_async @@ -27,9 +27,11 @@ def run_gen(): self._worker.start() @staticmethod - async def work_on_item(item: TrainableItem, callback, for_train) -> NoReturn: + async def work_on_item(item: GenableItem, callback, for_train) -> NoReturn: try: if for_train: + if not isinstance(item, TrainableItem): + raise TypeError(f"Expected TrainableItem for training generation, got {type(item).__name__}") result = await item.generate_for_train() else: result = await item.generate() @@ -37,13 +39,15 @@ async def work_on_item(item: TrainableItem, callback, for_train) -> NoReturn: import traceback logger.error("\n".join(traceback.format_exception(e))) - result = e + # Plain RuntimeError is safer to pass between worker processes than + # arbitrary exception subclasses with custom state. + result = RuntimeError(f"{type(e).__name__}: {e}") callback(item, result) def generate( - self, gen_items: list[TrainableItem], for_train: bool = False - ) -> Iterable[tuple[TrainableItem, list[EnvTrajectory] | Any]]: + self, gen_items: list[GenableItem], for_train: bool = False + ) -> Iterable[tuple[GenableItem, list[EnvTrajectory] | Any]]: out_queue = Queue() for item in gen_items: self.input_queue.put((item, lambda a, b: out_queue.put((a, b)), for_train)) @@ -54,9 +58,9 @@ def generate( def submit_with_callback( self, - genable: TrainableItem, + genable: GenableItem, for_train=False, - callback: Callable[[tuple[TrainableItem, Any]], NoReturn] | None = None, + callback: Callable[[tuple[GenableItem, Any]], NoReturn] | None = None, task_meta: dict | None = None, ) -> NoReturn: if callback is None: @@ -113,11 +117,27 @@ def _generation_worker_process(input_queue: mp.Queue, result_queue: mp.Queue) -> class GenerationController: - def __init__(self, num_workers: int | None = None): + def __init__( + self, + num_workers: int | None = None, + max_concurrent_genables: int | None = None, + ): self.num_workers = num_workers or mp.cpu_count() + if max_concurrent_genables is not None and max_concurrent_genables < 1: + raise ValueError("max_concurrent_genables must be >= 1") + self.max_concurrent_genables = max_concurrent_genables self.input_queue = mp.Queue() self.result_queue = mp.Queue() self.callback_map = {} + self._pending_submissions: deque[tuple[GenableItem, str, bool]] = deque() + self._inflight_task_ids: set[str] = set() + self._state_lock = threading.Lock() + self._tqdm_disabled = False + self._tqdm_total: int | None = None + self._tqdm_desc = "Cumulative Completed" + self._tqdm_customized = False + self._progress_bar = None + self._completed_count = 0 self._alive = True @@ -136,20 +156,65 @@ def __init__(self, num_workers: int | None = None): self.callback_thread = threading.Thread(target=self._callback_loop, daemon=True) self.callback_thread.start() + def set_tqdm(self, disabled: bool, total: int, desc: str) -> None: + if total < 0: + raise ValueError(f"Expected total >= 0, got {total}") + with self._state_lock: + self._tqdm_disabled = disabled + self._tqdm_total = total + self._tqdm_desc = desc + self._tqdm_customized = True + self._close_progress_bar_locked() + def _callback_loop(self): """在主进程的主线程中运行的回调处理循环""" - pbar = tqdm(desc="Cumulative Completed") - while self._alive: task_id, item, result = self.result_queue.get() - assert task_id in self.callback_map - callback = self.callback_map.pop(task_id) + with self._state_lock: + assert task_id in self.callback_map + callback = self.callback_map.pop(task_id) + self._inflight_task_ids.discard(task_id) + self._dispatch_pending_locked() callback(item, result) - pbar.update() + with self._state_lock: + pbar = self._get_or_create_progress_bar_locked() + self._completed_count += 1 + if pbar is not None: + pbar.update() + + def _close_progress_bar_locked(self) -> None: + if self._progress_bar is not None: + self._progress_bar.close() + self._progress_bar = None + + def _get_or_create_progress_bar_locked(self): + if self._tqdm_disabled: + self._close_progress_bar_locked() + return None + if self._progress_bar is None: + self._progress_bar = tqdm( + total=self._tqdm_total, + desc=self._tqdm_desc, + initial=self._completed_count, + disable=self._tqdm_disabled, + ) + return self._progress_bar + + def _can_dispatch_locked(self) -> bool: + return self.max_concurrent_genables is None or len(self._inflight_task_ids) < self.max_concurrent_genables + + def _dispatch_pending_locked(self) -> None: + # Limit how many genables are handed to worker processes at once. This + # enforces a controller-wide in-flight cap even though each worker keeps + # its own local asyncio queue. + while self._pending_submissions and self._can_dispatch_locked(): + genable, task_id, for_train = self._pending_submissions.popleft() + self._inflight_task_ids.add(task_id) + self.input_queue.put((genable, task_id, for_train)) def generate( - self, gen_items: list[TrainableItem], for_train: bool = False - ) -> Iterable[tuple[TrainableItem, list[EnvTrajectory] | Any]]: + self, gen_items: list[GenableItem], for_train: bool = False + ) -> Iterable[tuple[GenableItem, list[EnvTrajectory] | Any]]: """批量生成方法（保持原有接口）""" out_queue = Queue() task_ids = list(range(len(gen_items))) @@ -169,22 +234,24 @@ def generate( def submit_with_callback( self, - genable: TrainableItem, + genable: GenableItem, for_train: bool = False, - callback: Callable[[tuple[TrainableItem, Any]], NoReturn] | None = None, + callback: Callable[[tuple[GenableItem, Any]], NoReturn] | None = None, + task_id: str | None = None, ) -> NoReturn: """提交单个任务（支持自定义回调）""" if callback is None: callback = print # 生成唯一任务ID - task_id = str(uuid4()) - - # 保存回调函数（将在主线程执行） - self.callback_map[task_id] = callback + if task_id is None: + task_id = str(uuid4()) - # 将任务发送给工作进程 - self.input_queue.put((genable, task_id, for_train)) + with self._state_lock: + # 保存回调函数（将在主线程执行） + self.callback_map[task_id] = callback + self._pending_submissions.append((genable, task_id, for_train)) + self._dispatch_pending_locked() def shutdown(self): """关闭控制器""" @@ -202,3 +269,5 @@ def shutdown(self): # 等待回调线程结束 self.callback_thread.join(timeout=0.5) + with self._state_lock: + self._close_progress_bar_locked() diff --git a/steptronoss/generation/base_benchmark.py b/steptronoss/generation/base_benchmark.py new file mode 100644 index 00000000..31d8615a --- /dev/null +++ b/steptronoss/generation/base_benchmark.py @@ -0,0 +1,369 @@ +"""Minimal benchmark data model shared by generation and offline scoring. + +The intended ownership split is: + +- `Prompt`: request-ready model input owned by the benchmark implementation. +- `BenchmarkMeta`: benchmark-specific scoring context owned by the benchmark + implementation and never sent to the model. +- `EvaluationMeta`: runtime metadata owned by the evaluator/generation runner. +- `Generated`: final model output plus the metadata required for offline + analysis. +""" + +from __future__ import annotations + +import math +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, replace +from typing import TypeAlias, TypedDict + +JsonScalar: TypeAlias = None | bool | int | float | str +"""Primitive JSON scalar values used in exported benchmark artifacts.""" + +JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] +"""Recursive JSON value used to describe stored benchmark metadata.""" + +JsonObject: TypeAlias = dict[str, JsonValue] +"""JSON object with string keys and recursive JSON values.""" + + +class GroundTruthValue(TypedDict): + """Minimal ground-truth pointer stored on some exported assistant turns. + + Fields: + item_id: Stable benchmark item identifier referenced by this turn. + dataset: Dataset or benchmark family that owns the referenced item. + """ + + item_id: str + dataset: str + + +class GroundTruth(TypedDict): + """Ground-truth wrapper used by exported benchmark prompts. + + Fields: + value: Nested ground-truth pointer payload. + """ + + value: GroundTruthValue + + +class ChatMessage(TypedDict, total=False): + """Structured chat message used by tokenizer rendering and generation. + + Fields: + role: Chat role such as `system`, `user`, or `assistant`. + content: Plain-text content rendered into the model prompt. + ground_truth: Optional benchmark-specific pointer attached to the turn. + """ + + role: str + content: str + ground_truth: GroundTruth + + +Messages: TypeAlias = list[ChatMessage] +"""Ordered chat messages sent to the generation endpoint.""" + + +class CompletionMessage(TypedDict, total=False): + """Structured message payload from chat/completions style APIs. + + Fields: + content: Plain-text assistant response. + reasoning_content: Optional reasoning side channel returned by some APIs. + """ + + content: str + reasoning_content: str + + +class CompletionChoice(TypedDict, total=False): + """Normalized completion choice preserved in prediction dumps. + + Fields: + finish_reason: Backend-provided stop reason for the sampled completion. + text: Plain text returned by completion-style APIs. + message: Structured message returned by chat/completions APIs. + raw: JSON object containing the original backend payload for this choice. + """ + + finish_reason: str + text: str + message: CompletionMessage + raw: JsonObject + + +@dataclass(frozen=True) +class SamplingParams: + """Request parameters that control how a prompt is generated.""" + + temperature: float | None = None + """Sampling temperature passed to the generation backend.""" + + top_p: float | None = None + """Nucleus-sampling threshold passed to the generation backend.""" + + top_k: int | None = None + """Top-k sampling threshold passed to the generation backend.""" + + max_tokens: int | None = None + """Maximum number of generated tokens requested from the backend.""" + + seed: int | None = None + """Random seed used to make repeated sampling runs reproducible.""" + + extra_body: JsonObject = field(default_factory=dict) + """Additional request fields not covered by the explicit attributes above.""" + + def to_dict(self) -> JsonObject: + """Serialize request parameters into a generation payload fragment.""" + + payload = dict(self.extra_body) + if self.temperature is not None: + payload["temperature"] = self.temperature + if self.top_p is not None: + payload["top_p"] = self.top_p + if self.top_k is not None: + payload["top_k"] = self.top_k + if self.max_tokens is not None: + payload["max_tokens"] = self.max_tokens + if self.seed is not None: + payload["seed"] = self.seed + return payload + + +@dataclass(frozen=True) +class Prompt: + """Client-side request payload for one benchmark generation.""" + + tokens: list[int] | None = None + """Tokenized prompt content for token-based generation APIs.""" + + messages: Messages | None = None + """Structured chat messages for chat-style generation APIs.""" + + prompt_token_count: int | None = None + """Optional prompt length used for budget checks when only messages are sent.""" + + sampling_params: SamplingParams | None = None + """Generation parameters attached to this prompt from the client view.""" + + def __post_init__(self) -> None: + """Validate the request shape and derive prompt length when possible.""" + + has_tokens = self.tokens is not None + has_messages = self.messages is not None + if has_tokens == has_messages: + raise ValueError("Prompt must define exactly one of tokens or messages") + if self.prompt_token_count is not None and self.prompt_token_count < 0: + raise ValueError("Prompt.prompt_token_count must be non-negative") + if self.tokens is not None and self.prompt_token_count is None: + object.__setattr__(self, "prompt_token_count", len(self.tokens)) + + def with_sampling_params(self, sampling_params: SamplingParams) -> Prompt: + """Return a copy of this prompt with generation parameters attached.""" + + return replace(self, sampling_params=sampling_params) + + def to_dict(self) -> JsonObject: + """Serialize the prompt into a JSON-compatible structure.""" + + payload: JsonObject = {} + if self.tokens is not None: + payload["tokens"] = list(self.tokens) + if self.messages is not None: + payload["messages"] = [dict(message) for message in self.messages] + if self.prompt_token_count is not None: + payload["prompt_token_count"] = self.prompt_token_count + payload["sampling_params"] = None if self.sampling_params is None else self.sampling_params.to_dict() + return payload + + +@dataclass(frozen=True) +class BenchmarkMeta: + """Benchmark-owned metadata that is independent of evaluator runtime state.""" + + benchmark_name: str + """Stable benchmark identifier, usually matching the dataset family name.""" + + item_id: str + """Stable item identifier chosen by the benchmark implementation.""" + + context: JsonObject = field(default_factory=dict) + """Benchmark-specific scoring context not meant to be fed to the model.""" + + def to_dict(self) -> JsonObject: + """Serialize benchmark-owned metadata into a JSON-compatible structure.""" + + return { + "benchmark_name": self.benchmark_name, + "item_id": self.item_id, + "context": dict(self.context), + } + + +@dataclass(frozen=True) +class EvaluationMeta: + """Evaluator-owned runtime metadata that is benchmark-agnostic.""" + + prompt_index: int + """Index of the expanded prompt within the current evaluation run.""" + + run_index: int + """Sample index among repeated generations for the same logical item.""" + + def to_dict(self) -> JsonObject: + """Serialize evaluator-owned metadata into a JSON-compatible structure.""" + + return { + "prompt_index": self.prompt_index, + "run_index": self.run_index, + } + + +@dataclass(frozen=True) +class EvaluationCase: + """Minimal Prompt -> Generated data-flow unit used by benchmark runners.""" + + prompt: Prompt + """Prompt to send to the generation backend.""" + + benchmark: BenchmarkMeta + """Benchmark-owned context associated with the prompt.""" + + evaluation: EvaluationMeta + """Evaluator-owned runtime metadata associated with the prompt.""" + + def with_sampling_params(self, sampling_params: SamplingParams) -> EvaluationCase: + """Return a copy of the case with request parameters attached to its prompt.""" + + return EvaluationCase( + prompt=self.prompt.with_sampling_params(sampling_params), + benchmark=self.benchmark, + evaluation=self.evaluation, + ) + + +def build_demo_generated(response: str = "3") -> Generated: + """Build the smallest complete Prompt -> Generated example. + + This helper is intentionally simple so new benchmark implementations can see + the expected data flow in one place without reading the full runner stack. + """ + + case = EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "How many r are in strawberry?"}], + prompt_token_count=3, + sampling_params=SamplingParams(seed=0, max_tokens=8), + ), + benchmark=BenchmarkMeta( + benchmark_name="demo_count_r", + item_id="strawberry_r_count_0", + context={"answer": "3"}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=0), + ) + return Generated(case=case, response=response) + + +@dataclass +class Generated: + """Model output plus benchmark and evaluator metadata for offline scoring.""" + + case: EvaluationCase + """Prompt plus benchmark/evaluator metadata that produced this output.""" + + choice: CompletionChoice | None = None + """Normalized completion choice returned by the serving backend, if any.""" + + response: str = "" + """Plain-text model response extracted from the backend payload.""" + + reasoning_content: str | None = None + """Optional side-channel reasoning content returned by compatible models.""" + + error: str | None = None + """Error text captured when generation failed instead of returning a choice.""" + + @property + def finish_reason(self) -> str: + """Expose a normalized finish reason for downstream metrics.""" + + if self.error: + return "error" + if not self.choice: + return "unknown" + finish_reason = self.choice.get("finish_reason") + if isinstance(finish_reason, str) and finish_reason: + return finish_reason + return "unknown" + + def to_dict(self) -> JsonObject: + """Serialize the generated output into a JSON-compatible structure.""" + + return { + "prompt": self.case.prompt.to_dict(), + "benchmark": self.case.benchmark.to_dict(), + "evaluation": self.case.evaluation.to_dict(), + "choice": None if self.choice is None else dict(self.choice), + "response": self.response, + "reasoning_content": self.reasoning_content, + "error": self.error, + } + + +@dataclass +class BaseMetric: + """Common aggregate metric shape shared by lightweight benchmark scorers.""" + + score_avg: float + """Mean sample score across all generated outputs.""" + + score_std: float + """Population standard deviation of sample scores.""" + + pass_at_k: dict[int, float] = field(default_factory=dict) + """Unbiased HumanEval-style pass@k estimates averaged over grouped items.""" + + def __repr__(self) -> str: + """Render a stable human-readable summary string.""" + + def _format(value: float) -> str: + if math.isnan(value): + return "nan" + return f"{value:.4f}" + + pass_repr = ", ".join(f"{k}: {_format(v)}" for k, v in sorted(self.pass_at_k.items())) + return ( + f"BaseMetric(score_avg={_format(self.score_avg)}, " + f"score_std={_format(self.score_std)}, " + f"pass_at_k={{ {pass_repr} }})" + ) + + def to_dict(self) -> JsonObject: + """Serialize aggregate metric fields into a JSON-compatible structure.""" + + return { + "score_avg": self.score_avg, + "score_std": self.score_std, + "pass_at_k": {str(k): v for k, v in self.pass_at_k.items()}, + "repr": repr(self), + } + + +class BaseBenchmark(ABC): + """Minimal protocol shared by exported benchmark adapters.""" + + name: str + """Stable benchmark identifier used in prediction dumps and summaries.""" + + @abstractmethod + def get_cases(self) -> list[EvaluationCase]: + """Load evaluation cases in the exact order expected by the scorer.""" + + @abstractmethod + def evaluate(self, results: list[Generated]) -> BaseMetric: + """Aggregate per-sample outputs into a benchmark metric.""" diff --git a/steptronoss/generation/base_generatable.py b/steptronoss/generation/base_generatable.py index 35f87b6f..17c25e18 100644 --- a/steptronoss/generation/base_generatable.py +++ b/steptronoss/generation/base_generatable.py @@ -15,6 +15,17 @@ def __init__(self, meta: dict | None = None) -> None: async def generate(self) -> Any: pass + def fingerprint(self) -> str: + """Return a stable cache fingerprint for this generation request. + + Subclasses that participate in persistent generation caching should + override this method and return a deterministic string. The default + implementation keeps existing non-cache call sites working while making + unsupported cache usage fail explicitly. + """ + + raise NotImplementedError(f"{type(self).__name__} must implement fingerprint() for generation caching") + class TrainableItem(GenableItem): @abstractmethod diff --git a/steptronoss/generation/vllm/vllm_client.py b/steptronoss/generation/vllm/vllm_client.py index 7c8868c2..3a27bff4 100644 --- a/steptronoss/generation/vllm/vllm_client.py +++ b/steptronoss/generation/vllm/vllm_client.py @@ -7,7 +7,6 @@ import requests from aiohttp.client_exceptions import ClientConnectionError from loguru import logger -from vllm.entrypoints.openai.protocol import CompletionRequest, CompletionResponse from steptronoss.exp.inference import VLLMDeployConfig from steptronoss.utils.comm_utils import block_get_redis, get_exp_redis @@ -108,24 +107,30 @@ def check_server_alive(self) -> bool: def wait_for_server(self, timeout: int = 3600) -> list[str]: """Wait till all vLLM servers ready or timeout; return endpoints list.""" exp_redis = get_exp_redis() - expected_replica = block_get_redis(exp_redis, f"VLLM_NUM_WORKERS_OF_{self.cfg.model_name}") + expected_replica = block_get_redis( + exp_redis, + f"VLLM_NUM_WORKERS_OF_{self.cfg.model_name}", + timeout=timeout, + ) expected_replica = int(expected_replica) start_time = time.time() while time.time() - start_time < timeout: endpoints = self.get(f"{self._router_addr}/get_info") endpoint_addrs = [ep["endpoint"] for ep in endpoints] + logger.info(f"[wait_for_server] {len(endpoint_addrs)}/{expected_replica} endpoints ready: {endpoint_addrs}") if self.check_server_alive() and len(endpoints) >= expected_replica: - logger.info(f"[wait_for_server] All {len(endpoint_addrs)} endpoints ready: {endpoint_addrs}") assert len(endpoint_addrs) == len(set(endpoint_addrs)), "Duplicate endpoints found in endpoint_addrs" return endpoint_addrs time.sleep(max(2, timeout / 1200)) raise TimeoutError("VLLM wait server timeout!") - def completion(self, prompt_or_request: str | list[int] | CompletionRequest, max_tokens=None) -> CompletionResponse: + def completion(self, prompt_or_request: str | list[int] | dict[str, Any], max_tokens=None) -> dict[str, Any]: """发送聊天完成请求进行测试""" - if isinstance(prompt_or_request, CompletionRequest): + if hasattr(prompt_or_request, "model_dump"): payload = prompt_or_request.model_dump() + elif isinstance(prompt_or_request, dict): + payload = prompt_or_request.copy() else: payload = {"prompt": prompt_or_request, "return_token_ids": True} payload["model"] = self.cfg.model_name @@ -135,7 +140,7 @@ def completion(self, prompt_or_request: str | list[int] | CompletionRequest, max url=f"{self._router_addr}/v1/completions", json=payload, ) - return CompletionResponse(**result) + return result async def _get_model_endpoints(self) -> list[str]: """获取所有模型端点的IP和端口""" diff --git a/steptronoss/generation/vllm/vllm_router.py b/steptronoss/generation/vllm/vllm_router.py index 5b8dfd1c..2a1b446e 100644 --- a/steptronoss/generation/vllm/vllm_router.py +++ b/steptronoss/generation/vllm/vllm_router.py @@ -101,7 +101,21 @@ def _get_info_api(self) -> list[dict]: async def _get_session(self) -> aiohttp.ClientSession: if self._session is None or self._session.closed: - self._session = aiohttp.ClientSession(timeout=None) + # A client-side HTTP timeout is not part of the proxied request, so + # the closest transparent behavior is to avoid imposing any extra + # router-side timeout and let the original client decide. Also + # remove aiohttp's default 100-connection pool cap; otherwise the + # router itself becomes the hidden global concurrency bottleneck + # before requests ever reach vLLM. + self._session = aiohttp.ClientSession( + connector=aiohttp.TCPConnector(limit=0, limit_per_host=0), + timeout=aiohttp.ClientTimeout( + total=None, + connect=None, + sock_connect=None, + sock_read=None, + ), + ) return self._session def _make_proxy(self, route_name: str): diff --git a/steptronoss/utils/general.py b/steptronoss/utils/general.py index 13a89a51..9a22c3e7 100644 --- a/steptronoss/utils/general.py +++ b/steptronoss/utils/general.py @@ -14,10 +14,39 @@ import torch from configurize import DataClass from loguru import logger +from tqdm import tqdm T = TypeVar("Any") +class GroupedProgressBar: + def __init__(self, totals: dict[str, int]): + self.total_bar = tqdm( + total=sum(totals.values()), + desc="All Groups", + position=0, + dynamic_ncols=True, + ) + self.group_bars = { + name: tqdm( + total=total, + desc=name, + position=position, + dynamic_ncols=True, + ) + for position, (name, total) in enumerate(totals.items(), start=1) + } + + def update(self, name: str) -> None: + self.total_bar.update() + self.group_bars[name].update() + + def close(self) -> None: + self.total_bar.close() + for group_bar in self.group_bars.values(): + group_bar.close() + + def safediv(n, d): q, r = divmod(n, d) assert r == 0 diff --git a/tests/benchmarks/test_common_metrics.py b/tests/benchmarks/test_common_metrics.py new file mode 100644 index 00000000..78fc91f5 --- /dev/null +++ b/tests/benchmarks/test_common_metrics.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import pytest + +from playground.eval.benchmarks.common import JsonlChatBenchmark +from steptronoss.generation.base_benchmark import BenchmarkMeta, EvaluationCase, EvaluationMeta, Generated, Prompt + + +def _make_generated(*, item_id: str, run_index: int, success: bool) -> Generated: + return Generated( + case=EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "test prompt"}], + prompt_token_count=2, + ), + benchmark=BenchmarkMeta( + benchmark_name="TEST_BENCHMARK", + item_id=item_id, + context={}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=run_index), + ), + response="ok" if success else "bad", + choice={"finish_reason": "stop", "raw": {}}, + ) + + +def _build_metric(results: list[Generated]): + return JsonlChatBenchmark._build_metric( + results=results, + sample_values=[1.0 if result.response == "ok" else 0.0 for result in results], + sample_per_prompt=4, + is_success_fn=lambda result: result.response == "ok", + ) + + +def test_pass_at_k_uses_unbiased_estimator_instead_of_prefix_hits(): + results = [ + _make_generated(item_id="item_0", run_index=0, success=False), + _make_generated(item_id="item_0", run_index=1, success=False), + _make_generated(item_id="item_0", run_index=2, success=False), + _make_generated(item_id="item_0", run_index=3, success=True), + _make_generated(item_id="item_1", run_index=0, success=False), + _make_generated(item_id="item_1", run_index=1, success=False), + _make_generated(item_id="item_1", run_index=2, success=False), + _make_generated(item_id="item_1", run_index=3, success=False), + ] + + metric = _build_metric(results) + + assert metric.score_avg == pytest.approx(0.125) + assert metric.pass_at_k == pytest.approx({ + 1: 0.125, + 2: 0.25, + 4: 0.5, + }) + + +def test_pass_at_k_is_order_invariant_for_successful_samples(): + late_success_metric = _build_metric([ + _make_generated(item_id="item_0", run_index=0, success=False), + _make_generated(item_id="item_0", run_index=1, success=False), + _make_generated(item_id="item_0", run_index=2, success=False), + _make_generated(item_id="item_0", run_index=3, success=True), + ]) + early_success_metric = _build_metric([ + _make_generated(item_id="item_0", run_index=0, success=True), + _make_generated(item_id="item_0", run_index=1, success=False), + _make_generated(item_id="item_0", run_index=2, success=False), + _make_generated(item_id="item_0", run_index=3, success=False), + ]) + + assert late_success_metric.pass_at_k == pytest.approx(early_success_metric.pass_at_k) diff --git a/tests/benchmarks/test_ifbench_benchmark.py b/tests/benchmarks/test_ifbench_benchmark.py new file mode 100644 index 00000000..e31c9836 --- /dev/null +++ b/tests/benchmarks/test_ifbench_benchmark.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from playground.eval.benchmarks.IFBench import IFBenchBenchmark +from steptronoss.generation.base_benchmark import Generated + + +class DummyTokenizer: + def apply_chat_template( + self, + messages, + *, + tokenize: bool, + add_generation_prompt: bool, + **kwargs, + ): + del tokenize, add_generation_prompt, kwargs + return list(range(sum(len(message["content"].split()) for message in messages))) + + def encode(self, text: str, *, add_special_tokens: bool = False): + del add_special_tokens + return text.split() + + +def _write_official_style_prompt_file(path: Path) -> None: + records = [ + { + "key": "0", + "prompt": "Pick one option. Answer with one of the following options: Red/Blue/Green.", + "instruction_id_list": ["format:options"], + "kwargs": [{"options": "Red/Blue/Green"}], + }, + { + "key": "1", + "prompt": "Include exactly 2 numbers in the response.", + "instruction_id_list": ["count:numbers"], + "kwargs": [{"N": 2}], + }, + ] + path.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + + +def test_ifbench_directory_data_path_resolves_official_prompt_file(tmp_path): + resource_root = tmp_path / "datasets" / "IFBENCH" + resource_root.mkdir(parents=True) + _write_official_style_prompt_file(resource_root / "IFBench_test.jsonl") + + benchmark = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=1, + ) + + assert benchmark.resource_root == str(resource_root) + assert benchmark.data_path == str(resource_root / "IFBench_test.jsonl") + + +def test_ifbench_rejects_prompt_file_data_path(tmp_path): + resource_root = tmp_path / "datasets" / "IFBENCH" + resource_root.mkdir(parents=True) + prompt_file = resource_root / "IFBench_test.jsonl" + _write_official_style_prompt_file(prompt_file) + + with pytest.raises(ValueError, match="must point to the IFBENCH resource directory"): + IFBenchBenchmark( + data_path=str(prompt_file), + tokenizer=DummyTokenizer(), + sample_per_prompt=1, + ) + + +def test_ifbench_loads_official_prompt_file_and_scores_with_lazy_verifier(tmp_path): + datasets_dir = tmp_path / "datasets" + resource_root = datasets_dir / "IFBENCH" + resource_root.mkdir(parents=True) + prompt_file = resource_root / "IFBench_test.jsonl" + _write_official_style_prompt_file(prompt_file) + + benchmark = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=2, + ) + benchmark_without_strip = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=1, + strip_reasoning=False, + ) + + for module_name in list(sys.modules): + if module_name.startswith("playground.eval.benchmarks.IFBench.official."): + sys.modules.pop(module_name) + + cases = benchmark.get_cases() + assert [case.benchmark.item_id for case in cases] == ["0", "0", "1", "1"] + assert [case.prompt.messages[0]["content"] for case in cases] == [ + "Pick one option. Answer with one of the following options: Red/Blue/Green.", + "Pick one option. Answer with one of the following options: Red/Blue/Green.", + "Include exactly 2 numbers in the response.", + "Include exactly 2 numbers in the response.", + ] + assert all(case.prompt.sampling_params is not None for case in cases) + assert all(case.prompt.sampling_params.temperature == 0.0 for case in cases) + assert not any( + module_name.startswith("playground.eval.benchmarks.IFBench.official.") for module_name in sys.modules + ) + + results = [ + Generated(case=cases[0], response="hidden scratchpad Red"), + Generated(case=cases[1], response="wrong"), + Generated(case=cases[2], response="1"), + Generated(case=cases[3], response="1 2"), + ] + metric = benchmark.evaluate(results) + + assert metric.score_avg == 0.5 + assert metric.pass_at_k[1] == 0.5 + assert metric.pass_at_k[2] == 1.0 + + no_strip_metric = benchmark_without_strip.evaluate([ + Generated(case=cases[0], response="hidden scratchpad Red") + ]) + assert no_strip_metric.score_avg == 0.0 + + +def test_ifbench_metric_includes_official_strict_and_loose_reports(tmp_path): + resource_root = tmp_path / "datasets" / "IFBENCH" + resource_root.mkdir(parents=True) + _write_official_style_prompt_file(resource_root / "IFBench_test.jsonl") + + benchmark = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=2, + ) + strict_primary_benchmark = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=2, + evaluation_mode="strict", + ) + + cases = benchmark.get_cases() + results = [ + Generated(case=cases[0], response="preface\nRed"), + Generated(case=cases[1], response="wrong"), + Generated(case=cases[2], response="1"), + Generated(case=cases[3], response="1 2"), + ] + + loose_metric = benchmark.evaluate(results) + strict_metric = strict_primary_benchmark.evaluate(results) + + loose_metrics = loose_metric.to_dict()["official_metrics"] + assert loose_metric.score_avg == 0.5 + assert loose_metrics["loose"]["prompt_level_accuracy"] == 0.5 + assert loose_metrics["strict"]["prompt_level_accuracy"] == 0.25 + assert loose_metrics["loose"]["instruction_level_accuracy"] == 0.5 + assert loose_metrics["strict"]["instruction_level_accuracy"] == 0.25 + assert loose_metrics["loose"]["tier0_accuracy"] == {"count": 0.5, "format": 0.5} + assert loose_metrics["strict"]["tier0_accuracy"] == {"count": 0.5, "format": 0.0} + assert loose_metrics["loose"]["tier1_accuracy"] == {"count:numbers": 0.5, "format:options": 0.5} + assert loose_metrics["strict"]["tier1_accuracy"] == {"count:numbers": 0.5, "format:options": 0.0} + + strict_metrics = strict_metric.to_dict()["official_metrics"] + assert strict_metric.score_avg == 0.25 + assert strict_metric.to_dict()["evaluation_mode"] == "strict" + assert strict_metrics == loose_metrics diff --git a/tests/benchmarks/test_math_benchmarks.py b/tests/benchmarks/test_math_benchmarks.py new file mode 100644 index 00000000..e30db379 --- /dev/null +++ b/tests/benchmarks/test_math_benchmarks.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import pytest + +from playground.eval.benchmarks.AIME25 import AIME25Benchmark +from playground.eval.benchmarks.HMMT25 import HMMT25Benchmark +from steptronoss.generation.base_benchmark import BenchmarkMeta, EvaluationCase, EvaluationMeta, Generated, Prompt + + +def _make_generated(response: str) -> Generated: + return Generated( + case=EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "Solve the problem."}], + prompt_token_count=4, + ), + benchmark=BenchmarkMeta( + benchmark_name="TEST", + item_id="test-item", + context={}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=0), + ), + response=response, + ) + + +def test_aime_extract_answer_handles_nested_boxed_braces(): + response = r"\boxed{1 - \frac{2}{\pi}}" + assert AIME25Benchmark._extract_answer(response) == r"1 - \frac{2}{\pi}" + + +def test_hmmt_accepts_whitespace_and_dfrac_variants(): + result = _make_generated(r"\boxed{\dfrac{1}{576}}") + assert HMMT25Benchmark._is_correct(result, r"\frac{1}{576}") + + +def test_hmmt_accepts_boxed_answers_with_nested_braces(): + result = _make_generated(r"\boxed{1 - \frac{2}{\pi}}") + assert HMMT25Benchmark._is_correct(result, r"1-\frac{2}{\pi}") + + +def test_hmmt_accepts_math_verify_equivalent_forms_when_available(): + pytest.importorskip("math_verify") + result = _make_generated(r"\boxed{\frac{9}{\sqrt{23}}}") + assert HMMT25Benchmark._is_correct(result, r"\frac{9 \sqrt{23}}{23}") diff --git a/tests/test_async_generation.py b/tests/test_async_generation.py index 761c3edd..5a25d5ef 100644 --- a/tests/test_async_generation.py +++ b/tests/test_async_generation.py @@ -1,16 +1,19 @@ import asyncio +import multiprocessing as mp +import time from queue import Queue import pytest +import steptronoss.generation.async_generation as async_generation from steptronoss.exp.rl import EnvTrajectory, StopType from steptronoss.generation.async_generation import GenerationController -from steptronoss.generation.base_generatable import TrainableItem +from steptronoss.generation.base_generatable import GenableItem, TrainableItem pytestmark = pytest.mark.cpu -class FakeGenable(TrainableItem): +class FakeTrainable(TrainableItem): async def generate(self): await asyncio.sleep(0) return {"ok": True} @@ -27,18 +30,62 @@ async def generate_for_train(self): ] -def test_generation_controller_with_fake_genable(): +class FakeGeneratable(GenableItem): + async def generate(self): + await asyncio.sleep(0) + return {"ok": True} + + +class BlockingGenable(GenableItem): + def __init__(self, state, lock, release_event): + super().__init__() + self.state = state + self.lock = lock + self.release_event = release_event + + async def generate(self): + with self.lock: + active = self.state["active"] + 1 + self.state["active"] = active + self.state["entered"] += 1 + self.state["max_active"] = max(self.state["max_active"], active) + + try: + while not self.release_event.is_set(): + await asyncio.sleep(0.01) + return {"ok": True} + finally: + with self.lock: + self.state["active"] -= 1 + + +class _FakeTqdm: + def __init__(self, *, total=None, desc="", initial=0, disable=False): + self.total = total + self.desc = desc + self.n = initial + self.disable = disable + self.closed = False + + def update(self, n=1): + self.n += n + + def close(self): + self.closed = True + + +def test_generation_controller_with_fake_trainable(): controller = GenerationController(num_workers=1) try: result_queue: Queue = Queue() controller.submit_with_callback( - FakeGenable(), + FakeTrainable(), for_train=True, callback=lambda item, result, q=result_queue: q.put((item, result)), ) item, result = result_queue.get(timeout=5) - assert isinstance(item, FakeGenable) + assert isinstance(item, FakeTrainable) assert isinstance(result, list) assert len(result) == 1 traj = result[0] @@ -49,3 +96,97 @@ def test_generation_controller_with_fake_genable(): assert traj.is_gen_mask == [0, 1, 1] finally: controller.shutdown() + + +def test_generation_controller_set_tqdm(monkeypatch): + progress_bars = [] + + def _fake_tqdm(*args, **kwargs): + bar = _FakeTqdm(**kwargs) + progress_bars.append(bar) + return bar + + monkeypatch.setattr(async_generation, "tqdm", _fake_tqdm) + + controller = GenerationController(num_workers=1) + try: + controller.set_tqdm(disabled=False, total=1, desc="Friendly Progress") + result_queue: Queue = Queue() + controller.submit_with_callback( + FakeGeneratable(), + callback=lambda item, result, q=result_queue: q.put((item, result)), + ) + + item, result = result_queue.get(timeout=5) + assert isinstance(item, FakeGeneratable) + assert result == {"ok": True} + + deadline = time.time() + 5 + while time.time() < deadline and (not progress_bars or progress_bars[0].n < 1): + time.sleep(0.05) + + assert len(progress_bars) == 1 + assert progress_bars[0].desc == "Friendly Progress" + assert progress_bars[0].total == 1 + assert progress_bars[0].n == 1 + assert progress_bars[0].disable is False + finally: + controller.shutdown() + + assert progress_bars[0].closed is True + + +def test_generation_controller_with_fake_generatable(): + controller = GenerationController(num_workers=1) + try: + result_queue: Queue = Queue() + controller.submit_with_callback( + FakeGeneratable(), + callback=lambda item, result, q=result_queue: q.put((item, result)), + ) + + item, result = result_queue.get(timeout=5) + assert isinstance(item, FakeGeneratable) + assert result == {"ok": True} + finally: + controller.shutdown() + + +def test_generation_controller_max_concurrent_genables_limits_global_inflight(): + manager = mp.Manager() + state = manager.dict(active=0, entered=0, max_active=0) + lock = manager.Lock() + release_event = manager.Event() + + controller = GenerationController(num_workers=2, max_concurrent_genables=1) + try: + result_queue: Queue = Queue() + for _ in range(2): + controller.submit_with_callback( + BlockingGenable(state=state, lock=lock, release_event=release_event), + callback=lambda item, result, q=result_queue: q.put((item, result)), + ) + + deadline = time.time() + 5 + while time.time() < deadline and state["entered"] < 1: + time.sleep(0.05) + + assert state["entered"] == 1 + + time.sleep(0.3) + assert state["entered"] == 1 + assert state["max_active"] == 1 + + release_event.set() + + first_item, first_result = result_queue.get(timeout=5) + second_item, second_result = result_queue.get(timeout=5) + assert isinstance(first_item, BlockingGenable) + assert isinstance(second_item, BlockingGenable) + assert first_result == {"ok": True} + assert second_result == {"ok": True} + assert state["entered"] == 2 + assert state["max_active"] == 1 + finally: + controller.shutdown() + manager.shutdown() diff --git a/tests/test_simple_eval_cache.py b/tests/test_simple_eval_cache.py new file mode 100644 index 00000000..6aa6a8f0 --- /dev/null +++ b/tests/test_simple_eval_cache.py @@ -0,0 +1,412 @@ +import asyncio +import json +from pathlib import Path + +import pytest +from diskcache import Cache + +from playground.eval.eval_sets import simple_eval +from playground.eval.eval_sets.simple_eval import SimpleBenchmarksEvalConfig, SimpleChatGeneratable +from steptronoss.generation.base_benchmark import ( + BenchmarkMeta, + EvaluationCase, + EvaluationMeta, + Generated, + Prompt, + SamplingParams, +) + +pytestmark = pytest.mark.cpu + + +class _FakeGroupedProgressBar: + def __init__(self, totals): + self.totals = totals + self.total_updates = 0 + self.group_updates: list[str] = [] + self.closed = False + + def update(self, name: str): + self.total_updates += 1 + self.group_updates.append(name) + + def close(self): + self.closed = True + + +class _InlineGenerationController: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def set_tqdm(self, disabled: bool, total: int, desc: str) -> None: + self.disabled = disabled + self.total = total + self.desc = desc + + def generate(self, gen_items): + for item in gen_items: + yield item, asyncio.run(item.generate()) + + def shutdown(self): + return None + + +class _CountingSimpleChatGeneratable(SimpleChatGeneratable): + def __init__(self, case: EvaluationCase, response_text: str, call_counter: dict[str, int]): + super().__init__( + case=case, + endpoint_getter=lambda: "http://unused", + model_name_getter=lambda: "unused-model", + max_model_len=1024, + sampling_params=SamplingParams( + temperature=1.0, + top_p=1.0, + top_k=-1, + max_tokens=32, + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, + ), + ) + self.response_text = response_text + self.call_counter = call_counter + + async def generate(self) -> Generated: + self.call_counter[self.response_text] = self.call_counter.get(self.response_text, 0) + 1 + return Generated(case=self.case, response=self.response_text) + + +class _TestSimpleBenchmarksEvalConfig(SimpleBenchmarksEvalConfig): + def get_benchmarks(self): + return [] + + +class _DummyTokenizer: + def apply_chat_template( + self, + messages, + *, + tokenize: bool, + add_generation_prompt: bool, + **kwargs, + ): + del tokenize, add_generation_prompt, kwargs + return list(range(sum(len(message["content"].split()) for message in messages))) + + def encode(self, text: str, *, add_special_tokens: bool = False): + del add_special_tokens + return text.split() + + +class _DummyTokenizerConfig: + def build_tokenizer(self): + return _DummyTokenizer() + + +def _build_case(*, benchmark_name: str, item_id: str, run_index: int, prompt_text: str) -> EvaluationCase: + return EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": prompt_text}], + prompt_token_count=8, + ), + benchmark=BenchmarkMeta( + benchmark_name=benchmark_name, + item_id=item_id, + context={"item_id": item_id}, + ), + evaluation=EvaluationMeta( + prompt_index=0, + run_index=run_index, + ), + ) + + +def _write_simple_benchmark_record(path: Path, *, dataset: str, item_id: str, prompt: str, answer: str) -> None: + path.write_text( + json.dumps({ + "dataset": dataset, + "messages": [{"role": "user", "content": prompt}], + "source_item": { + "item_id": item_id, + "answer": answer, + }, + }) + + "\n", + encoding="utf-8", + ) + + +def _write_ifbench_prompt_file(path: Path) -> None: + path.write_text( + json.dumps({ + "key": "0", + "prompt": "Pick one option. Answer with one of the following options: Red/Blue/Green.", + "instruction_id_list": ["format:options"], + "kwargs": [{"options": "Red/Blue/Green"}], + }) + + "\n", + encoding="utf-8", + ) + + +def test_simple_chat_generatable_fingerprint_changes_with_sampling_params(): + genable_run0 = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="item-0", run_index=0, prompt_text="What is 1+1?"), + response_text="2", + call_counter={}, + ) + genable_run1 = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="item-1", run_index=1, prompt_text="What is 1+1?"), + response_text="2", + call_counter={}, + ) + + assert genable_run0.fingerprint() != genable_run1.fingerprint() + + +def test_get_sampling_params_merges_benchmark_overrides(): + cfg = _TestSimpleBenchmarksEvalConfig() + cfg.max_decode_steps = 256 + cfg.chat_template_args = {"enable_thinking": True} + + sampling_params = cfg.get_sampling_params( + SamplingParams( + temperature=0.0, + top_p=0.9, + max_tokens=32, + seed=7, + extra_body={"guided_decoding_backend": "xgrammar"}, + ) + ) + + assert sampling_params.temperature == 0.0 + assert sampling_params.top_p == 0.9 + assert sampling_params.top_k == -1 + assert sampling_params.max_tokens == 32 + assert sampling_params.seed == 7 + assert sampling_params.extra_body == { + "chat_template_kwargs": {"enable_thinking": True}, + "guided_decoding_backend": "xgrammar", + } + + +def test_get_sampling_params_uses_defaults_when_benchmark_sampling_params_is_none(): + cfg = _TestSimpleBenchmarksEvalConfig() + cfg.max_decode_steps = 256 + cfg.chat_template_args = {"enable_thinking": True} + + sampling_params = cfg.get_sampling_params(None) + + assert sampling_params.temperature == 1.0 + assert sampling_params.top_p == 1.0 + assert sampling_params.top_k == -1 + assert sampling_params.max_tokens == 256 + assert sampling_params.seed is None + assert sampling_params.extra_body == { + "chat_template_kwargs": {"enable_thinking": True}, + } + + +def test_get_prompts_loads_ifbench_from_datasets_dir(tmp_path): + datasets_dir = tmp_path / "datasets" + datasets_dir.mkdir() + _write_simple_benchmark_record( + datasets_dir / "AIME2025.jsonl", + dataset="AIME2025", + item_id="aime-0", + prompt="Solve 1+1.", + answer="2", + ) + _write_simple_benchmark_record( + datasets_dir / "GPQA_DIAMOND.jsonl", + dataset="GPQA_DIAMOND", + item_id="gpqa-0", + prompt="Choose A, B, C, or D.", + answer="A", + ) + _write_simple_benchmark_record( + datasets_dir / "HLE_TEXTONLY.jsonl", + dataset="HLE_TEXTONLY", + item_id="hle-0", + prompt="State the final answer.", + answer="42", + ) + _write_simple_benchmark_record( + datasets_dir / "HMMT25.jsonl", + dataset="HMMT25", + item_id="hmmt-0", + prompt="Compute 2+2.", + answer="4", + ) + _write_simple_benchmark_record( + datasets_dir / "MMLU_PRO.jsonl", + dataset="MMLU_PRO", + item_id="mmlu-0", + prompt="Choose A through I.", + answer="B", + ) + ifbench_dir = datasets_dir / "IFBENCH" + ifbench_dir.mkdir() + _write_ifbench_prompt_file(ifbench_dir / "IFBench_test.jsonl") + + cfg = SimpleBenchmarksEvalConfig() + cfg.datasets_dir = str(datasets_dir) + cfg.selected_datasets = "IFBENCH" + cfg.tokenizer_cfg = _DummyTokenizerConfig() + cfg.router_addr_key = "unused" + cfg.model_name_template = "unused-model" + cfg.max_model_len = 1024 + cfg.max_decode_steps = 256 + cfg.chat_template_args = {"enable_thinking": True} + + prompts = cfg.get_prompts() + + assert len(prompts) == 1 + assert all(prompt.case.benchmark.benchmark_name == "IFBENCH" for prompt in prompts) + assert all(prompt.case.prompt.messages[0]["content"].startswith("Pick one option.") for prompt in prompts) + assert all(prompt.case.prompt.sampling_params is not None for prompt in prompts) + assert all(prompt.case.prompt.sampling_params.temperature == 0.0 for prompt in prompts) + assert all(prompt.case.prompt.sampling_params.max_tokens == 256 for prompt in prompts) + assert all( + prompt.case.prompt.sampling_params.extra_body == {"chat_template_kwargs": {"enable_thinking": True}} + for prompt in prompts + ) + + +def test_get_prompts_can_enable_ifbench_thinking_via_chat_template_args(tmp_path): + datasets_dir = tmp_path / "datasets" + datasets_dir.mkdir() + _write_simple_benchmark_record( + datasets_dir / "AIME2025.jsonl", + dataset="AIME2025", + item_id="aime-0", + prompt="Solve 1+1.", + answer="2", + ) + _write_simple_benchmark_record( + datasets_dir / "GPQA_DIAMOND.jsonl", + dataset="GPQA_DIAMOND", + item_id="gpqa-0", + prompt="Choose A, B, C, or D.", + answer="A", + ) + _write_simple_benchmark_record( + datasets_dir / "HLE_TEXTONLY.jsonl", + dataset="HLE_TEXTONLY", + item_id="hle-0", + prompt="State the final answer.", + answer="42", + ) + _write_simple_benchmark_record( + datasets_dir / "HMMT25.jsonl", + dataset="HMMT25", + item_id="hmmt-0", + prompt="Compute 2+2.", + answer="4", + ) + _write_simple_benchmark_record( + datasets_dir / "MMLU_PRO.jsonl", + dataset="MMLU_PRO", + item_id="mmlu-0", + prompt="Choose A through I.", + answer="B", + ) + ifbench_dir = datasets_dir / "IFBENCH" + ifbench_dir.mkdir() + _write_ifbench_prompt_file(ifbench_dir / "IFBench_test.jsonl") + + cfg = SimpleBenchmarksEvalConfig() + cfg.datasets_dir = str(datasets_dir) + cfg.selected_datasets = "IFBENCH" + cfg.tokenizer_cfg = _DummyTokenizerConfig() + cfg.router_addr_key = "unused" + cfg.model_name_template = "unused-model" + cfg.max_model_len = 1024 + cfg.max_decode_steps = 256 + cfg.chat_template_args = {"enable_thinking": True} + + prompts = cfg.get_prompts() + + assert len(prompts) == 1 + assert prompts[0].case.prompt.sampling_params is not None + assert prompts[0].case.prompt.sampling_params.temperature == 0.0 + assert prompts[0].case.prompt.sampling_params.extra_body == {"chat_template_kwargs": {"enable_thinking": True}} + + +def test_generate_reuses_cached_generated_and_rebinds_case(monkeypatch, tmp_path): + monkeypatch.setattr(simple_eval, "GroupedProgressBar", _FakeGroupedProgressBar) + cfg = _TestSimpleBenchmarksEvalConfig() + cfg.save_dir = str(tmp_path) + cfg.run_tag = "resume-tag" + cfg.rerun_level = None + + genable = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="current-item", run_index=0, prompt_text="cached prompt"), + response_text="fresh", + call_counter={}, + ) + cached_generated = Generated( + case=_build_case(benchmark_name="bench", item_id="stale-item", run_index=0, prompt_text="cached prompt"), + response="cached", + ) + + with Cache(directory=cfg.predictions_path) as generation_cache: + generation_cache[genable.fingerprint()] = cached_generated + + results = cfg._generate([genable]) + + assert len(results) == 1 + assert results[0].response == "cached" + assert results[0].case == genable.case + assert results[0].case.benchmark.item_id == "current-item" + + +def test_generate_rerun_level_error_reruns_cached_errors_once_per_fingerprint(monkeypatch, tmp_path): + monkeypatch.setattr(simple_eval, "GenerationController", _InlineGenerationController) + monkeypatch.setattr(simple_eval, "GroupedProgressBar", _FakeGroupedProgressBar) + + cfg = _TestSimpleBenchmarksEvalConfig() + cfg.save_dir = str(tmp_path) + cfg.run_tag = "resume-tag" + cfg.rerun_level = "error" + + call_counter: dict[str, int] = {} + genable = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="item-a", run_index=0, prompt_text="shared prompt"), + response_text="fresh", + call_counter=call_counter, + ) + + with Cache(directory=cfg.predictions_path) as generation_cache: + generation_cache[genable.fingerprint()] = Generated(case=genable.case, error="cached failure") + + results = cfg._generate([genable]) + + assert call_counter == {"fresh": 1} + assert [result.response for result in results] == ["fresh"] + assert [result.case.benchmark.item_id for result in results] == ["item-a"] + + with Cache(directory=cfg.predictions_path) as generation_cache: + cached_generated = generation_cache[genable.fingerprint()] + assert isinstance(cached_generated, Generated) + assert cached_generated.error is None + assert cached_generated.response == "fresh" + + +def test_generate_rejects_duplicate_fingerprints(tmp_path): + cfg = _TestSimpleBenchmarksEvalConfig() + cfg.save_dir = str(tmp_path) + cfg.run_tag = "resume-tag" + + genable_a = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="item-a", run_index=0, prompt_text="shared prompt"), + response_text="fresh-a", + call_counter={}, + ) + genable_b = _CountingSimpleChatGeneratable( + case=_build_case(benchmark_name="bench", item_id="item-b", run_index=0, prompt_text="shared prompt"), + response_text="fresh-b", + call_counter={}, + ) + + with pytest.raises(ValueError, match="Duplicate generation fingerprint detected"): + cfg._generate([genable_a, genable_b]) diff --git a/tests/test_vllm_router.py b/tests/test_vllm_router.py index b702a34c..e67751ad 100644 --- a/tests/test_vllm_router.py +++ b/tests/test_vllm_router.py @@ -51,6 +51,13 @@ async def _get_session(): return router +class _CapturingClientSession: + def __init__(self, *, timeout, connector): + self.timeout = timeout + self.connector = connector + self.closed = False + + def test_vllm_router_streaming(monkeypatch): upstream = _FakeUpstream( status=200, @@ -82,3 +89,25 @@ def test_vllm_router_non_stream(monkeypatch): assert response.status_code == 200 assert response.content == body assert upstream.released is True + + +@pytest.mark.anyio +async def test_vllm_router_uses_no_additional_upstream_timeout(monkeypatch): + captured = {} + + def _client_session_factory(*, timeout, connector): + session = _CapturingClientSession(timeout=timeout, connector=connector) + captured["session"] = session + return session + + monkeypatch.setattr("steptronoss.generation.vllm.vllm_router.aiohttp.ClientSession", _client_session_factory) + + router = VLLMRouter(VLLMRouterConfig()) + session = await router._get_session() + assert session is captured["session"] + assert session.timeout.total is None + assert session.timeout.connect is None + assert session.timeout.sock_connect is None + assert session.timeout.sock_read is None + assert session.connector.limit == 0 + assert session.connector.limit_per_host == 0 From f777306e1b6ddf312e6511e39af79dcc448a5b3b Mon Sep 17 00:00:00 2001 From: zhouhy Date: Tue, 17 Mar 2026 11:32:43 +0800 Subject: [PATCH 2/4] update lock file --- uv.lock | 938 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 611 insertions(+), 327 deletions(-) diff --git a/uv.lock b/uv.lock index 1813b039..3d429cd1 100644 --- a/uv.lock +++ b/uv.lock @@ -2,8 +2,12 @@ version = 1 revision = 3 requires-python = ">=3.10, <4.0" resolution-markers = [ - "python_full_version >= '3.12' and sys_platform == 'linux'", - "python_full_version >= '3.12' and sys_platform != 'linux'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version < '3.11' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", @@ -851,7 +855,7 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.12.2" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "loguru" }, @@ -859,9 +863,9 @@ dependencies = [ { name = "torch" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/79/4c5c1cd14266f8cf2650bdb940f986ce7fcaeb56aad8cfa9e9afedf14e2f/compressed_tensors-0.12.2.tar.gz", hash = "sha256:5bb40856dd17f128ab73557ecc73799f80db4dd82fab6de875f1e6899b9ea0c4", size = 190409, upload-time = "2025-10-07T14:30:59.302Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/65/88dd1c58fb9d0ded51b5c86471b937a1525f91fad2211a6f051dc1ea822d/compressed_tensors-0.13.0.tar.gz", hash = "sha256:23893824d3498ea3f1a829f14a8fa85f9a5e76a34c711a038b8d7c619ca9a67c", size = 200995, upload-time = "2025-12-16T16:03:55.397Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/c0/1695b87d369e6652ec0d650912e02eca2151c5e9c29244f94d2afccfe970/compressed_tensors-0.12.2-py3-none-any.whl", hash = "sha256:e554ea761710ca2b0c0ea49276a4ef8e08658624f1591e6a7368817106b48fbe", size = 183049, upload-time = "2025-10-07T14:30:56.523Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/61ac2563c62490922b603c09113a083fd74af3630ec3931e769484d6dcb5/compressed_tensors-0.13.0-py3-none-any.whl", hash = "sha256:3518799c9baf034eb642efb551db6b0537b8713d45a64fe4def26f7f8d6cabec", size = 192620, upload-time = "2025-12-16T16:03:53.041Z" }, ] [[package]] @@ -1056,31 +1060,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" }, ] +[[package]] +name = "cuda-bindings" +version = "12.9.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/31/bfcc870f69c6a017c4ad5c42316207fc7551940db6f3639aa4466ec5faf3/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a022c96b8bd847e8dc0675523431149a4c3e872f440e3002213dbb9e08f0331a", size = 11800959, upload-time = "2025-10-21T14:51:26.458Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5", size = 12148218, upload-time = "2025-10-21T14:51:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/a9/2b/ebcbb60aa6dba830474cd360c42e10282f7a343c0a1f58d24fbd3b7c2d77/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6a429dc6c13148ff1e27c44f40a3dd23203823e637b87fd0854205195988306", size = 11840604, upload-time = "2025-10-21T14:51:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c912a3d9e6b6651853eed8eed96d6800d69c08e94052c292fec3f282c5a817c9", size = 12210593, upload-time = "2025-10-21T14:51:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c2/65bfd79292b8ff18be4dd7f7442cea37bcbc1a228c1886f1dea515c45b67/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:694ba35023846625ef471257e6b5a4bc8af690f961d197d77d34b1d1db393f56", size = 11760260, upload-time = "2025-10-21T14:51:40.79Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c1/dabe88f52c3e3760d861401bb994df08f672ec893b8f7592dc91626adcf3/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fda147a344e8eaeca0c6ff113d2851ffca8f7dfc0a6c932374ee5c47caa649c8", size = 12151019, upload-time = "2025-10-21T14:51:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/05/8b/b4b2d1c7775fa403b64333e720cfcfccef8dcb9cdeb99947061ca5a77628/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf8bfaedc238f3b115d957d1fd6562b7e8435ba57f6d0e2f87d0e7149ccb2da5", size = 11570071, upload-time = "2025-10-21T14:51:47.472Z" }, + { url = "https://files.pythonhosted.org/packages/63/56/e465c31dc9111be3441a9ba7df1941fe98f4aa6e71e8788a3fb4534ce24d/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:32bdc5a76906be4c61eb98f546a6786c5773a881f3b166486449b5d141e4a39f", size = 11906628, upload-time = "2025-10-21T14:51:49.905Z" }, + { url = "https://files.pythonhosted.org/packages/ec/07/6aff13bc1e977e35aaa6b22f52b172e2890c608c6db22438cf7ed2bf43a6/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3adf4958dcf68ae7801a59b73fb00a8b37f8d0595060d66ceae111b1002de38d", size = 11566797, upload-time = "2025-10-21T14:51:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/1e6be415e37478070aeeee5884c2022713c1ecc735e6d82d744de0252eee/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56e0043c457a99ac473ddc926fe0dc4046694d99caef633e92601ab52cbe17eb", size = 11925991, upload-time = "2025-10-21T14:51:56.535Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b5/96a6696e20c4ffd2b327f54c7d0fde2259bdb998d045c25d5dedbbe30290/cuda_bindings-12.9.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f53a7f453d4b2643d8663d036bafe29b5ba89eb904c133180f295df6dc151e5", size = 11624530, upload-time = "2025-10-21T14:52:01.539Z" }, + { url = "https://files.pythonhosted.org/packages/d1/af/6dfd8f2ed90b1d4719bc053ff8940e494640fe4212dc3dd72f383e4992da/cuda_bindings-12.9.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b72ee72a9cc1b531db31eebaaee5c69a8ec3500e32c6933f2d3b15297b53686", size = 11922703, upload-time = "2025-10-21T14:52:03.585Z" }, + { url = "https://files.pythonhosted.org/packages/39/73/d2fc40c043bac699c3880bf88d3cebe9d88410cd043795382826c93a89f0/cuda_bindings-12.9.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20f2699d61d724de3eb3f3369d57e2b245f93085cab44fd37c3bea036cea1a6f", size = 11565056, upload-time = "2025-10-21T14:52:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/6c/19/90ac264acc00f6df8a49378eedec9fd2db3061bf9263bf9f39fd3d8377c3/cuda_bindings-12.9.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d80bffc357df9988dca279734bc9674c3934a654cab10cadeed27ce17d8635ee", size = 11924658, upload-time = "2025-10-21T14:52:10.411Z" }, +] + [[package]] name = "cuda-bindings" version = "13.1.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", +] dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/60/63/579402b642f5b9b8ceb79e456b39b5771f27e132a8af3b140e54d69790fc/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4400370a83f1538e25ed4c18c34a0e9d5fad39741e282e69ce24d1479a11017d", size = 15777291, upload-time = "2025-12-09T22:05:41.109Z" }, - { url = "https://files.pythonhosted.org/packages/df/6a/3a293cfb01cd4964444a0f75917b6edb1c31ea69d0230e329975da6991ba/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f92500e2f6aec2dac00a5a1ce77d5aa77ea77b606dc484d951f1f2cc3eaa13", size = 16311623, upload-time = "2025-12-09T22:05:43.897Z" }, { url = "https://files.pythonhosted.org/packages/72/b8/a5860b9e70faa53658236dc61efc3ecc51846beff4a0b73de9151130ff98/cuda_bindings-13.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3f5bb8190267216f96597235252087accac4cbccefd1b60756cced114b2d6754", size = 15185932, upload-time = "2025-12-09T22:05:46.089Z" }, - { url = "https://files.pythonhosted.org/packages/b0/58/b8d4c7c5fb29ba46088a7e78d1065484219f8fe41a08adc4a85b1ee56149/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5f5a6ade0ad45096568bc4dd1eb3377b65884d29124338fe9a4353130ef6631", size = 15771605, upload-time = "2025-12-09T22:05:48.266Z" }, - { url = "https://files.pythonhosted.org/packages/17/af/710403f76f2d608d483d87089465e1f666351641dbd73d19bd025e652bad/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9348f69b03b257f07159dd4c869615e139722c2bd81e96c66f6b8f77615efd82", size = 16338970, upload-time = "2025-12-09T22:05:50.598Z" }, { url = "https://files.pythonhosted.org/packages/64/1c/e7ea27d4cb7d07331c88e3bbed3cacc947d2237471801086c7447b3e195d/cuda_bindings-13.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ec33b84f4bd65a86a734427f2b9cb8f221bedab2c4cfb681488cabc82f1d64ab", size = 15210672, upload-time = "2025-12-09T22:05:53.369Z" }, - { url = "https://files.pythonhosted.org/packages/53/3d/c8ed9d169843091f3f0d6b8218e826fd59520a37e0434c204feada597988/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e75ad0cb863330df784236d289612d71ca855c013d19ae00e5693574abd6915", size = 15530160, upload-time = "2025-12-09T22:05:55.386Z" }, - { url = "https://files.pythonhosted.org/packages/4a/8e/368295623ee43fba622909d780fbb6863efc1638dff55f67a0f04eac6470/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25785d1a3cdcd98f151240fd5efd025609319a6720a217dee2a929241749d488", size = 16110386, upload-time = "2025-12-09T22:05:57.71Z" }, { url = "https://files.pythonhosted.org/packages/60/1f/ecc4701ade3e85f091c625a920574527b9daf7fb354189fbfbc5516af6cd/cuda_bindings-13.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:ccde9c95c0e953b31fe7731bb08da9d0a34b1770498df9a3c156fdfdbe3951ad", size = 15250028, upload-time = "2025-12-09T22:06:00.346Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c1/0ee8fd94bab7e23116e0e3da8c0902e299f3d9edc95f1d7d8ef894c897ed/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c9822a57c8f952dc367aacd7c32fe4cb17371104383606f455ea74635bff4c7", size = 15421116, upload-time = "2025-12-09T22:06:02.994Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c2/f272fad414b96299e010dcbe510cf17fc25deaf3443e0fdb55020a8298a3/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5837f5ea422c5653626dcfe22e9ab68142cd19af9e67a226100f224cc25a1b99", size = 15940152, upload-time = "2025-12-09T22:06:05.079Z" }, { url = "https://files.pythonhosted.org/packages/2a/56/433093bec0121f031edb582ea3a72f71031e8fbebecaaf329809344da4c7/cuda_bindings-13.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e4f348cd7a779657d51e6f71aac3965fb1738f40ff3bbe75265a3242fd6f29f", size = 15216463, upload-time = "2025-12-09T22:06:07.296Z" }, - { url = "https://files.pythonhosted.org/packages/de/38/40416d037ed25db68f1dbd50e0232775a62d90c9f25af22b196c0a13b88c/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86258fe1b0d3998bea7f57dc891569e4996705b8dd00366e44c722d0a29b2090", size = 15498927, upload-time = "2025-12-09T22:06:09.476Z" }, - { url = "https://files.pythonhosted.org/packages/ac/3f/f1f88b6cdb7d41ba076f8ff10edf6d3bd17e740da9a163544b43d6349653/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:daf8468fd603b2724c2d16cbd499348c64916ed72b1d04643f1660ce13cd12ae", size = 15984539, upload-time = "2025-12-09T22:06:11.882Z" }, { url = "https://files.pythonhosted.org/packages/f6/33/7739cc5e9a3373df8e7dea9060528bee5f70cf6e28b9c14f765502816c71/cuda_bindings-13.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:f2e079182014dbc162562b46467815272c14c7afe5b988978fa968728b0ac726", size = 15373212, upload-time = "2025-12-09T22:06:13.989Z" }, - { url = "https://files.pythonhosted.org/packages/9e/0a/5c6d514e566ff86c4054bbbb6554bf49b9c55fefbc934eb456faecab53c9/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0cd96a6ec00a78235947bff9462b2139bc5b83ce8e297d865802f0b52d1e23d", size = 15403944, upload-time = "2025-12-09T22:06:16.315Z" }, - { url = "https://files.pythonhosted.org/packages/0b/5b/319cfa491a685d4d4757aa24223b6dbc0976954afac42f49fc47290ba6a3/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ff465829c6c394c2b4047250324a19925cf8c44633345b2746a4741e07bf827", size = 15911462, upload-time = "2025-12-09T22:06:18.403Z" }, { url = "https://files.pythonhosted.org/packages/e3/5c/38b92080c5b6c4ddb09f0be2536123f81c7e9e1a89e4573f20cb00347ee3/cuda_bindings-13.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:8205eee6b8b458a2110c0384923ace206855d0f1b436fc1b145fcbaa1653b501", size = 16044390, upload-time = "2025-12-09T22:06:20.945Z" }, ] @@ -1092,13 +1122,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/5e/db279a3bfbd18d59d0598922a3b3c1454908d0969e8372260afec9736376/cuda_pathfinder-1.3.4-py3-none-any.whl", hash = "sha256:fb983f6e0d43af27ef486e14d5989b5f904ef45cedf40538bfdcbffa6bb01fb2", size = 30878, upload-time = "2026-02-11T18:50:31.008Z" }, ] +[[package]] +name = "cuda-python" +version = "12.9.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cuda-bindings", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/f3/6b032a554019cfb3447e671798c1bd3e79b5f1af20d10253f56cea269ef2/cuda_python-12.9.4-py3-none-any.whl", hash = "sha256:d2cacea882a69863f1e7d27ee71d75f0684f4c76910aff839067e4f89c902279", size = 7594, upload-time = "2025-10-21T14:55:12.846Z" }, +] + [[package]] name = "cuda-python" version = "13.1.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", +] dependencies = [ - { name = "cuda-bindings" }, - { name = "cuda-pathfinder" }, + { name = "cuda-bindings", version = "13.1.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/cd/08/b5e3b9822662d72d540d830531e3ab6a7cabbda3dd56175696aabccfeb76/cuda_python-13.1.1-py3-none-any.whl", hash = "sha256:944cc4fe6482673d28dd545797a28840945a1668739328fa2ad1e9be4f7050d9", size = 8038, upload-time = "2025-12-09T22:13:10.719Z" }, @@ -1284,6 +1339,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "emoji" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/78/0d2db9382c92a163d7095fc08efff7800880f830a152cfced40161e7638d/emoji-2.15.0.tar.gz", hash = "sha256:eae4ab7d86456a70a00a985125a03263a5eac54cd55e51d7e184b1ed3b6757e4", size = 615483, upload-time = "2025-09-21T12:13:02.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/5e/4b5aaaabddfacfe36ba7768817bd1f71a7a810a43705e531f3ae4c690767/emoji-2.15.0-py3-none-any.whl", hash = "sha256:205296793d66a89d88af4688fa57fd6496732eb48917a87175a023c8138995eb", size = 608433, upload-time = "2025-09-21T12:13:01.197Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -1549,7 +1613,7 @@ wheels = [ [[package]] name = "flashinfer-python" -version = "0.5.3" +version = "0.6.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -1566,9 +1630,9 @@ dependencies = [ { name = "torch" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/91/cca69baeff24bb3efd12c7479a026432c8717ee47193694010494c528b22/flashinfer_python-0.5.3.tar.gz", hash = "sha256:100d59b0ede47878d2808cd3a1b9039d7a952d66338bc9f68dac192ae1b2e3f1", size = 4682367, upload-time = "2025-11-20T21:22:46.976Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/45/15645d2a4ee81d08206f3e132a77323e48312f510462415d7cd1122eba43/flashinfer_python-0.6.4.tar.gz", hash = "sha256:e6ab798bd1030e5ff7a3bc6952f36386c406928f60b79cf964a6db7aa7ccde75", size = 5337134, upload-time = "2026-02-19T07:33:36.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/78/6dc7e7da8cb87c9965644ea0d2439457a1bc9256c45ceda0044595be4143/flashinfer_python-0.5.3-py3-none-any.whl", hash = "sha256:b601293b72f9138bad173edc28df84b9f239a013be974e2e79d4ba98aeb38cf5", size = 6998069, upload-time = "2025-11-20T21:22:45.104Z" }, + { url = "https://files.pythonhosted.org/packages/17/9a/d2bab76d2bb15062c6a2329614653e4f8bec9c78eec9069856ef0c7c0a79/flashinfer_python-0.6.4-py3-none-any.whl", hash = "sha256:105596b505892ae330af84e250ee0eb6fc2c3a22e8dc42bd46de1b90d36004c8", size = 7819999, upload-time = "2026-02-19T07:33:34.82Z" }, ] [[package]] @@ -1751,6 +1815,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.73.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/96/a0205167fa0154f4a542fd6925bdc63d039d88dab3588b875078107e6f06/googleapis_common_protos-1.73.0.tar.gz", hash = "sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a", size = 147323, upload-time = "2026-03-06T21:53:09.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/28/23eea8acd65972bbfe295ce3666b28ac510dfcb115fac089d3edb0feb00a/googleapis_common_protos-1.73.0-py3-none-any.whl", hash = "sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8", size = 297578, upload-time = "2026-03-06T21:52:33.933Z" }, +] + [[package]] name = "griffe" version = "2.0.0" @@ -1844,6 +1920,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/b2/b096ccce418882fbfda4f7496f9357aaa9a5af1896a9a7f60d9f2b275a06/grpcio-1.78.0-cp314-cp314-win_amd64.whl", hash = "sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb", size = 4929852, upload-time = "2026-02-06T09:56:45.885Z" }, ] +[[package]] +name = "grpcio-reflection" +version = "1.78.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/06/337546aae558675f79cae2a8c1ce0c9b1952cbc5c28b01878f68d040f5bb/grpcio_reflection-1.78.0.tar.gz", hash = "sha256:e6e60c0b85dbcdf963b4d4d150c0f1d238ba891d805b575c52c0365d07fc0c40", size = 19098, upload-time = "2026-02-06T10:01:52.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/6d/4d095d27ccd049865ecdafc467754e9e47ad0f677a30dda969c3590f6582/grpcio_reflection-1.78.0-py3-none-any.whl", hash = "sha256:06fcfde9e6888cdd12e9dd1cf6dc7c440c2e9acf420f696ccbe008672ed05b60", size = 22800, upload-time = "2026-02-06T10:01:33.822Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -2099,6 +2188,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/f2/53b6e9bdd2a91202066764eaa74b572ba4dede0fe47a5a26f4de34b7541a/ijson-3.4.0.post0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a0fedf09c0f6ffa2a99e7e7fd9c5f3caf74e655c1ee015a0797383e99382ebc3", size = 54657, upload-time = "2025-10-10T05:29:24.482Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -2148,8 +2249,12 @@ name = "ipython" version = "9.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12' and sys_platform == 'linux'", - "python_full_version >= '3.12' and sys_platform != 'linux'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", ] @@ -2322,6 +2427,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -2349,6 +2463,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "kaldi-native-fbank" +version = "1.22.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/2c/84076b352107ce12d56f28c313f1aca1be332d953dd96aec7b84976e6d53/kaldi-native-fbank-1.22.3.tar.gz", hash = "sha256:387bf87225c6b83c93ae652eeaef1b4d531994b6e398e7a77189de340674f9af", size = 71013, upload-time = "2025-10-09T02:31:21.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/c1/ff7a8c85a100dbef0df6473579cb78b1527f01d34859432de3f38d5a38d1/kaldi_native_fbank-1.22.3-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:af04cae53beb6da1e28e57e053d16118513e5fbe8d16ce0b3261f1b1b396af0a", size = 244533, upload-time = "2025-10-09T02:28:39.795Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d5/be771230ba2f071ad036a9224139fa4e0ac576b8abac15209342fc63ef86/kaldi_native_fbank-1.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:947cb8fae3611244b15006bd42263fe56afe583077e7006aac4bdb0e10dc5a4f", size = 227900, upload-time = "2025-10-09T02:33:05.679Z" }, + { url = "https://files.pythonhosted.org/packages/3c/9a/2ba3bcdf8b0d78339d3bb17307f70113d32bfc5492917d254995e61fd1c0/kaldi_native_fbank-1.22.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1e6b8c587dfe4f646b3a62f4b4ee840a83f4c491fe4bdfb74ccd9937ce17cdc", size = 296954, upload-time = "2025-10-09T02:30:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/2b/09/9031a517a0655e54c5bb8f243078798c19441f037b70a3f10b8aad82d073/kaldi_native_fbank-1.22.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c710b62442a43720db853cbafbf57ea3f920b593128dfb8fb88e08d6a8225772", size = 320031, upload-time = "2025-10-09T02:30:45.244Z" }, + { url = "https://files.pythonhosted.org/packages/3e/14/3f0d13909fee89d3826279a129bc3261c677fb1f39db1c4f714d92918762/kaldi_native_fbank-1.22.3-cp310-cp310-win32.whl", hash = "sha256:4d3c97ee08b9d3d528ff4fe8a20aeb7484eea06b2ccb7a3b5a7c86fa3b065b44", size = 272314, upload-time = "2025-10-09T02:30:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/0be9a61d373449d9782dad3b259892720bdc90c553cf618cb45a1c6443c0/kaldi_native_fbank-1.22.3-cp310-cp310-win_amd64.whl", hash = "sha256:1eb9a3a9c87597872a48acc167de7321b4660bc3a10ed2f552c5f633015b1b61", size = 302723, upload-time = "2025-10-09T02:28:31.64Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d0/07ab65d7c8389f56f8c772a55f8846a81c24d973abecfc0275c2c833f63e/kaldi_native_fbank-1.22.3-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:6b9ef5b6302ee45628a51a4484cb4f41006af02141508939c09ce36899fb3f41", size = 245879, upload-time = "2025-10-09T02:28:04.7Z" }, + { url = "https://files.pythonhosted.org/packages/64/2b/3132083b930fa6411f14469f36c465b7d2fba29a8a3e121d8fd6baffc8ea/kaldi_native_fbank-1.22.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:29452f2900e771086e9022dde17a92d191217ab3e34ca7dc361bd9be53e94fb4", size = 229180, upload-time = "2025-10-09T02:29:35.356Z" }, + { url = "https://files.pythonhosted.org/packages/e3/53/720ffbe8b30de203570f397866334eb4c6364c9214699010f2086de911ff/kaldi_native_fbank-1.22.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48e5dd8e897bf4509be2c6eeb4bbab728eaaef1f214ae0510c96219c4253d17", size = 299054, upload-time = "2025-10-09T02:28:42.011Z" }, + { url = "https://files.pythonhosted.org/packages/52/3f/beb161e4fdf6710938ccf18418c147d87ba8f102903d6c6e4eda25588e22/kaldi_native_fbank-1.22.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce84c65779c9eed6ec02699797a4ba1859451977537a993be3ea8167a210ec3e", size = 321921, upload-time = "2025-10-09T02:31:21.646Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/ee42418b77dbfc5ff619857b8eb372af98a88d47c8ca8b9a2d3ca2936c96/kaldi_native_fbank-1.22.3-cp311-cp311-win32.whl", hash = "sha256:516bce595eb5e5899a91dfec1142bea56a2fa232e53425e9966785aee8cd024e", size = 273018, upload-time = "2025-10-09T02:30:31.979Z" }, + { url = "https://files.pythonhosted.org/packages/40/68/da630b035cd343311168e5fe02c39fe7b192638717e3202de92ccf8ae18e/kaldi_native_fbank-1.22.3-cp311-cp311-win_amd64.whl", hash = "sha256:bd225d0624d45b533c1780094b3c59666276a6e9f20222943441212cdf301c9e", size = 303342, upload-time = "2025-10-09T02:28:17.429Z" }, + { url = "https://files.pythonhosted.org/packages/c2/de/fbdbfcc75fad9d9a6f9a250bc986f1002902581eaa47a5948f53a7f11851/kaldi_native_fbank-1.22.3-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7f636ccdea28bd187f93b06a1e4b9275e42e43af9405b0684fc739e829299c4b", size = 249003, upload-time = "2025-10-09T02:29:48.509Z" }, + { url = "https://files.pythonhosted.org/packages/77/64/e57ce185dda028b7b9af72cdfb16825bfa52183653945681e7cb8e7c2dfa/kaldi_native_fbank-1.22.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:abd31a8bfe1db62a7ddb0beee84f3a5de9bb559fcdd2b96ca0fb729c551b9412", size = 228933, upload-time = "2025-10-09T02:31:35.8Z" }, + { url = "https://files.pythonhosted.org/packages/43/28/6f4fd8953c0b3f30de4526fd024095032abcdc25b6736c77a891687c604e/kaldi_native_fbank-1.22.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5a44b4a83cf9bf13d3f77858928068b06d3ec2238c27ff2e39393fbf7749c9f", size = 298887, upload-time = "2025-10-09T02:30:53.739Z" }, + { url = "https://files.pythonhosted.org/packages/84/90/01ef7331c52b1eaf9916f3f7a535155aac2e9e2ddad12a141613d92758c7/kaldi_native_fbank-1.22.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f16e74372fe9e20abb4183f98a8e2288d5ee4c48d04d94b6160311170e007661", size = 322002, upload-time = "2025-10-09T02:30:13.04Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/fce142bd3aeadb1292360a90ceb91f923c8e12081c21576fe69917243c5f/kaldi_native_fbank-1.22.3-cp312-cp312-win32.whl", hash = "sha256:a90f51377569575fc0d1a66ef7e89a36102bfb6dcd1d15d6c4afb930ce726672", size = 273308, upload-time = "2025-10-09T02:29:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8d/c0b0b6280edabad85d7e15093fad612c027e175fe4e0b960ce2f36485143/kaldi_native_fbank-1.22.3-cp312-cp312-win_amd64.whl", hash = "sha256:cbbeea19fe6d584c54e93fe6615a7185b10e0d78fdb6471f9e44596018437c38", size = 308023, upload-time = "2025-10-09T02:28:43.909Z" }, + { url = "https://files.pythonhosted.org/packages/0d/df/4110f685067946c8b2e59ed76cebdf51c979ae999d90f65208a9d1966cba/kaldi_native_fbank-1.22.3-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:78ca163686a4aa1693194d098aa79b517845d851aa6fd27d5b162c05e1012361", size = 249056, upload-time = "2025-10-09T02:28:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/d8/74/ef21aabdd2f32539735e2ed4d3ea072112d4e3d30dfc2d17695f6d9df072/kaldi_native_fbank-1.22.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3768ea99933aa25080cb820f93f7b612968633b9a4fa23bc8a7337e2137f3fbb", size = 229011, upload-time = "2025-10-09T02:31:50.593Z" }, + { url = "https://files.pythonhosted.org/packages/9a/72/adb11d27c545aca1db442da744ee430a6aae377a33574bfd2ec159dcf673/kaldi_native_fbank-1.22.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f74b85948328ab4b4c88522f98a59f83dd5295443b08483e945c7de2c35e5dcc", size = 299276, upload-time = "2025-10-09T02:30:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1e/496c7ae814b2a7f8f47d423dc33aae2cdfb1edf898e2faaf5c5b39b90363/kaldi_native_fbank-1.22.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3f9c6551ff5b6ae785dd15f819c3b2b7432d77bfb79ea8806748e2c7d900b5d", size = 322714, upload-time = "2025-10-09T02:30:32.698Z" }, + { url = "https://files.pythonhosted.org/packages/75/47/3fcb52e0ef081efa40b4ca5c04f207509d31157f33f3ac314578d93794f9/kaldi_native_fbank-1.22.3-cp313-cp313-win32.whl", hash = "sha256:a63d5bd6b5bd5f7c0e0af886c12c3f686fbc62347f6b886fed2694ab2f0dbd14", size = 273293, upload-time = "2025-10-09T02:30:13.979Z" }, + { url = "https://files.pythonhosted.org/packages/63/48/20bfa3f8d88605e2ec2c274c343dec1f112077e687440d64d3caa4b9136c/kaldi_native_fbank-1.22.3-cp313-cp313-win_amd64.whl", hash = "sha256:4fb769337c7d482166ada8ba041003e4a9de3a778dc970b6b5802a382e581724", size = 308032, upload-time = "2025-10-09T02:29:03.278Z" }, + { url = "https://files.pythonhosted.org/packages/b9/7e/d47f64d5332b2527e6b65490888d99793eb3280bca735d0b69348eaeb6a3/kaldi_native_fbank-1.22.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2efa8368cdd46a32c37a28c4baaa508b0a294ab1ca2aefddd3e97f62cfebc27b", size = 249216, upload-time = "2025-10-09T02:28:22.008Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/f98f72ba5a90a39675e82f2175dc5ec99a85892a88b9ccdd25f2dc916c82/kaldi_native_fbank-1.22.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8f6086073ec658a23d22f8657b3ee8c6ba69d65be57324a7284209ac7424b5ac", size = 229289, upload-time = "2025-10-09T02:31:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/d6/4b/1f3f17a7b601124df88112a1d1fcb543c8d908d6674f752f7d3322991770/kaldi_native_fbank-1.22.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:41fb506fde155d97aeef95dd6ceccc38c2c5dd4401f9b8fded9bacaf1bafef36", size = 300037, upload-time = "2025-10-09T02:30:10.203Z" }, + { url = "https://files.pythonhosted.org/packages/2b/6a/374ec4e1cf13e672f5acd8272116c1885c2a7f84be491fc652415fc6e870/kaldi_native_fbank-1.22.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f1cc2b8eeec52a33868cf59bb95d40b335fa9cff7e15a6208e0e9b67b7fd7236", size = 322854, upload-time = "2025-10-09T02:31:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/63/2a/edd85a2292d2af28af68214a3bdd029ab8ce2e6bc5aaac77255aa57ce964/kaldi_native_fbank-1.22.3-cp314-cp314-win32.whl", hash = "sha256:d6387ab52b56e2978524590e11b24cf03419d9e9361965bc8d6ff34ff9e867da", size = 279733, upload-time = "2025-10-09T02:29:41.855Z" }, +] + [[package]] name = "lark" version = "1.2.2" @@ -2679,7 +2830,7 @@ wheels = [ [[package]] name = "mistral-common" -version = "1.9.0" +version = "1.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, @@ -2691,9 +2842,9 @@ dependencies = [ { name = "tiktoken" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/41/5b/60bb9f8c424c9ec7708396096f92e34f77eca94d55f99326b72b5a322482/mistral_common-1.9.0.tar.gz", hash = "sha256:5f90ec606d1826a20a97d24aefb9bfff7f4cd4cd576b622d4857708c0577e6c2", size = 6337103, upload-time = "2026-01-29T00:28:07.982Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/22/f798c1acc3f8cf32b6201b063d96867d79aa39d31dff12478739e1a78979/mistral_common-1.10.0.tar.gz", hash = "sha256:e456ff101edbdfc094039ec6c26f7d0f73356729798d628a6e6e96c3917147bc", size = 6351515, upload-time = "2026-03-13T10:13:46.683Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/12/8a3c9aaf58b49383d24f533edb2f81f073b59822317bd56bd66d0850caae/mistral_common-1.9.0-py3-none-any.whl", hash = "sha256:e25ed2f8c73f66cf3b1a48b2ddd649e044a0db7b9d9dd1af819eeb20ee1a6d94", size = 6517668, upload-time = "2026-01-29T00:28:04.96Z" }, + { url = "https://files.pythonhosted.org/packages/87/c6/1429a0a3ab40f8530492b62b52eb792266c261b22ed62aa7f25d61d531ae/mistral_common-1.10.0-py3-none-any.whl", hash = "sha256:c594d1a05202b61e8f0d867ec6064df4c5e5d492c2c2bdb6fd8fb4872c6afd8b", size = 6525284, upload-time = "2026-03-13T10:13:44.329Z" }, ] [package.optional-dependencies] @@ -2851,13 +3002,13 @@ name = "mlx-lm" version = "0.29.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jinja2" }, + { name = "jinja2", marker = "sys_platform != 'linux'" }, { name = "mlx", marker = "sys_platform == 'darwin'" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "sentencepiece" }, - { name = "transformers" }, + { name = "numpy", marker = "sys_platform != 'linux'" }, + { name = "protobuf", marker = "sys_platform != 'linux'" }, + { name = "pyyaml", marker = "sys_platform != 'linux'" }, + { name = "sentencepiece", marker = "sys_platform != 'linux'" }, + { name = "transformers", marker = "sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/62/f46e1355256a114808517947f8e83ad6be310c7288c551db0fa678f47923/mlx_lm-0.29.1.tar.gz", hash = "sha256:b99180d8f33d33a077b814e550bfb2d8a59ae003d668fd1f4b3fff62a381d34b", size = 232302, upload-time = "2025-12-16T16:58:27.959Z" } wheels = [ @@ -3229,8 +3380,12 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12' and sys_platform == 'linux'", - "python_full_version >= '3.12' and sys_platform != 'linux'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", ] @@ -3265,6 +3420,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, ] +[[package]] +name = "nltk" +version = "3.9.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/8f/915e1c12df07c70ed779d18ab83d065718a926e70d3ea33eb0cd66ffb7c0/nltk-3.9.3.tar.gz", hash = "sha256:cb5945d6424a98d694c2b9a0264519fab4363711065a46aa0ae7a2195b92e71f", size = 2923673, upload-time = "2026-02-24T12:05:53.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/7e/9af5a710a1236e4772de8dfcc6af942a561327bb9f42b5b4a24d0cf100fd/nltk-3.9.3-py3-none-any.whl", hash = "sha256:60b3db6e9995b3dd976b1f0fa7dec22069b2677e759c28eb69b62ddd44870522", size = 1525385, upload-time = "2026-02-24T12:05:46.54Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -3405,7 +3575,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, @@ -3438,7 +3608,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, @@ -3465,9 +3635,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, @@ -3478,7 +3648,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, @@ -3494,22 +3664,36 @@ wheels = [ [[package]] name = "nvidia-cutlass-dsl" -version = "4.3.5" +version = "4.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cutlass-dsl-libs-base" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/03/678dab0383db1ddfc449da216220f40404189eb36eeed9d87a4fa4bdb0e6/nvidia_cutlass_dsl-4.4.2-py3-none-any.whl", hash = "sha256:7cfb9ef19062b055b9372c7a627004724e2755e4c8b16c3cc88807d64501a4ae", size = 10167, upload-time = "2026-03-16T02:18:59.043Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-base" +version = "4.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-python" }, + { name = "cuda-python", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, + { name = "cuda-python", version = "13.1.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, { name = "numpy" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/52/3a/89f70082c24d3b88316df9b16df861e1f2cc86389a7b36a670bc7c541977/nvidia_cutlass_dsl-4.3.5-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b4fcc50dbf9f9c6d1f4d6e1748e366c6835c95bea7b54f7111bfa6e66230f74b", size = 58736963, upload-time = "2026-01-09T01:37:55.298Z" }, - { url = "https://files.pythonhosted.org/packages/e7/92/3f39b64341e2b16dedc7434e7b63a8f457a6fdbd023346d2f00276943495/nvidia_cutlass_dsl-4.3.5-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:776f54fa72333bc8fca274e59b70552adbcd85aaef603c7d58a79ef284890046", size = 58601295, upload-time = "2026-01-09T01:39:02.461Z" }, - { url = "https://files.pythonhosted.org/packages/e8/93/9114f28351d55061d30c68dbec3ba49659ac65607966029f52dab66950e9/nvidia_cutlass_dsl-4.3.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6de9a4a7150ad1832fb8c862c92df4836f347690e4c085e9044160c846010b59", size = 58736943, upload-time = "2026-01-09T01:40:25.777Z" }, - { url = "https://files.pythonhosted.org/packages/54/b5/d2f08919a9aa9052d45b2c8adfc310a724e9474e39c612358b1b24282c54/nvidia_cutlass_dsl-4.3.5-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7a792f02ce548f311a3df313a7cdb4ac4ec1cccb6c7ff9cd68d5470b25a6daf6", size = 58602358, upload-time = "2026-01-09T01:39:28.521Z" }, - { url = "https://files.pythonhosted.org/packages/78/6c/f45c930f662e0ec7856baa5d4e6f4d1e2ca6b029678f9e05d2df54c865be/nvidia_cutlass_dsl-4.3.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6a79e94d157b16ab34069dd73fb708ff0ef31f486d699b6d5a015217f754cb0b", size = 58739895, upload-time = "2026-01-09T01:38:22.076Z" }, - { url = "https://files.pythonhosted.org/packages/76/cb/998e79b6f028268bf2653250deb4a2edb618db81244e549ced71112c6f85/nvidia_cutlass_dsl-4.3.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4687eef20c405023daa99dd4653a292fd875d6c9486f8d9a069ff6fcdb00834f", size = 58602784, upload-time = "2026-01-09T01:40:52.873Z" }, - { url = "https://files.pythonhosted.org/packages/97/09/78a2f9141006f6f1e371a3dfb7a921205bcad6fb27810731169939d3e63d/nvidia_cutlass_dsl-4.3.5-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9343a5c1335169d791b05aac6fb81e33d7f17c4f8250613a091e6ee8314ed6aa", size = 58738707, upload-time = "2026-01-09T01:39:56.445Z" }, - { url = "https://files.pythonhosted.org/packages/0f/16/41b88ded92648d99f3c83880c07a54475feded9b32b4425e30d4b34f6c63/nvidia_cutlass_dsl-4.3.5-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:11d19b7e56ae1bedaf736ea3965af3be1e7af6c2482989c414b606cdd406cf32", size = 58601867, upload-time = "2026-01-09T01:37:29.895Z" }, + { url = "https://files.pythonhosted.org/packages/5f/07/af1b456b5b6dd4a49e71a952a182a99fc863f70b9f78725324f89e0384e5/nvidia_cutlass_dsl_libs_base-4.4.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:06acb3acff3dcf4bf6630476efac7de94de30b988ded4fa00b647bbcec4224ff", size = 75471025, upload-time = "2026-03-16T02:23:49.61Z" }, + { url = "https://files.pythonhosted.org/packages/b1/12/f0770811d2874af7e04623d3baa83c445c49f38c00c4e5d20e1daae54b5d/nvidia_cutlass_dsl_libs_base-4.4.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:916bf612fba5fbc5162e300fe18196e960dac2328c1c1360c0939d3be05c7c71", size = 74355272, upload-time = "2026-03-16T02:24:44.22Z" }, + { url = "https://files.pythonhosted.org/packages/60/bf/b9d0fd1ba281b111c941d9616dd9f98a509d84bf35076e60fef27ec7abd6/nvidia_cutlass_dsl_libs_base-4.4.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:261832dafe7579dc83cd3816ab9ea845e3de3737d876c215f01fb4edff1f4473", size = 75476977, upload-time = "2026-03-16T02:26:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/a5/23/86dda6d69a3fc29d0cde2a8b54c056ad69b73a6e5e230e18d906d2ec3b7c/nvidia_cutlass_dsl_libs_base-4.4.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40c2352b2fcc80789a216cbeb9b2ee10c85c15de839cda8f5c1d18166b8249df", size = 74356100, upload-time = "2026-03-16T02:26:12.778Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/0df5e38d11e52cc72095a14d6448bc1c5d0d4b00b069a1189ca417fb225b/nvidia_cutlass_dsl_libs_base-4.4.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2ec8812eeadcbb6fe20bda2e295ed9c00653f8253b78e33cf0ab65a47b829e73", size = 75473821, upload-time = "2026-03-16T02:27:08.371Z" }, + { url = "https://files.pythonhosted.org/packages/56/98/e264964741d9cc9816625d9600d17a5249fd5cbd8c2d166fb0d0c34dfe5a/nvidia_cutlass_dsl_libs_base-4.4.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:22e37b58f7a6f2f43bba533c4df8a088012122e0b4e9a632eca23937adeafb39", size = 74355593, upload-time = "2026-03-16T02:25:11.762Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c9/2f17950ee2deb4b5f6b82f8155515a21792fe296e81bb638f164d8e2ca9b/nvidia_cutlass_dsl_libs_base-4.4.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b59a052cbfb9a25747d1b6d413615456bea38d1f377da085af07c0d86a4c8b39", size = 75477304, upload-time = "2026-03-16T02:27:35.645Z" }, + { url = "https://files.pythonhosted.org/packages/e1/68/27380038ebd9c8eab4be364e833fea144aef597704f44948921668f7adf4/nvidia_cutlass_dsl_libs_base-4.4.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8e3324a33afa7424e93beae7e54a311e80db82b9e4ed4bba2aeeda1d6c888cd9", size = 74355765, upload-time = "2026-03-16T02:24:16.778Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/0dc7f2e5b5c65106a5bb05e60654f1a79abe92e27e9b00588a73cd26ca1f/nvidia_cutlass_dsl_libs_base-4.4.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:af96c1170569138b3cb965202907fbf5ab95d7c1dcc210952d00cdf9ab7b859a", size = 75472171, upload-time = "2026-03-16T02:28:03.136Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ae/0998f328b28b956d7eb399d16f4ee681ca318b306007264444a623e86c64/nvidia_cutlass_dsl_libs_base-4.4.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:95db0c8d1d56992e2f5c2dcd5b3baab0297bedc0cbcefc1e70b57acd934e7b23", size = 74356280, upload-time = "2026-03-16T02:25:43.789Z" }, ] [[package]] @@ -3539,10 +3723,10 @@ wheels = [ [[package]] name = "nvidia-nvshmem-cu12" -version = "3.3.20" +version = "3.4.5" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, + { url = "https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:042f2500f24c021db8a06c5eec2539027d57460e1c1a762055a6554f72c369bd", size = 139103095, upload-time = "2025-09-06T00:32:31.266Z" }, ] [[package]] @@ -3628,6 +3812,132 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/90/b338326131ccb2aaa3c2c85d00f41822c0050139a4bfe723cfd95455bd2d/opencv_python_headless-4.13.0.92-cp37-abi3-win_amd64.whl", hash = "sha256:77a82fe35ddcec0f62c15f2ba8a12ecc2ed4207c17b0902c7a3151ae29f37fb6", size = 40070414, upload-time = "2026-02-05T07:02:26.448Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/1d/4049a9e8698361cc1a1aa03a6c59e4fa4c71e0c0f94a30f988a6876a2ae6/opentelemetry_api-1.40.0.tar.gz", hash = "sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f", size = 70851, upload-time = "2026-03-04T14:17:21.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/bf/93795954016c522008da367da292adceed71cca6ee1717e1d64c83089099/opentelemetry_api-1.40.0-py3-none-any.whl", hash = "sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9", size = 68676, upload-time = "2026-03-04T14:17:01.24Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/37/b6708e0eff5c5fb9aba2e0ea09f7f3bcbfd12a592d2a780241b5f6014df7/opentelemetry_exporter_otlp-1.40.0.tar.gz", hash = "sha256:7caa0870b95e2fcb59d64e16e2b639ecffb07771b6cd0000b5d12e5e4fef765a", size = 6152, upload-time = "2026-03-04T14:17:23.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/fc/aea77c28d9f3ffef2fdafdc3f4a235aee4091d262ddabd25882f47ce5c5f/opentelemetry_exporter_otlp-1.40.0-py3-none-any.whl", hash = "sha256:48c87e539ec9afb30dc443775a1334cc5487de2f72a770a4c00b1610bf6c697d", size = 7023, upload-time = "2026-03-04T14:17:03.612Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/bc/1559d46557fe6eca0b46c88d4c2676285f1f3be2e8d06bb5d15fbffc814a/opentelemetry_exporter_otlp_proto_common-1.40.0.tar.gz", hash = "sha256:1cbee86a4064790b362a86601ee7934f368b81cd4cc2f2e163902a6e7818a0fa", size = 20416, upload-time = "2026-03-04T14:17:23.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ca/8f122055c97a932311a3f640273f084e738008933503d0c2563cd5d591fc/opentelemetry_exporter_otlp_proto_common-1.40.0-py3-none-any.whl", hash = "sha256:7081ff453835a82417bf38dccf122c827c3cbc94f2079b03bba02a3165f25149", size = 18369, upload-time = "2026-03-04T14:17:04.796Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/7f/b9e60435cfcc7590fa87436edad6822240dddbc184643a2a005301cc31f4/opentelemetry_exporter_otlp_proto_grpc-1.40.0.tar.gz", hash = "sha256:bd4015183e40b635b3dab8da528b27161ba83bf4ef545776b196f0fb4ec47740", size = 25759, upload-time = "2026-03-04T14:17:24.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/6f/7ee0980afcbdcd2d40362da16f7f9796bd083bf7f0b8e038abfbc0300f5d/opentelemetry_exporter_otlp_proto_grpc-1.40.0-py3-none-any.whl", hash = "sha256:2aa0ca53483fe0cf6405087a7491472b70335bc5c7944378a0a8e72e86995c52", size = 20304, upload-time = "2026-03-04T14:17:05.942Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/fa/73d50e2c15c56be4d000c98e24221d494674b0cc95524e2a8cb3856d95a4/opentelemetry_exporter_otlp_proto_http-1.40.0.tar.gz", hash = "sha256:db48f5e0f33217588bbc00274a31517ba830da576e59503507c839b38fa0869c", size = 17772, upload-time = "2026-03-04T14:17:25.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/3a/8865d6754e61c9fb170cdd530a124a53769ee5f740236064816eb0ca7301/opentelemetry_exporter_otlp_proto_http-1.40.0-py3-none-any.whl", hash = "sha256:a8d1dab28f504c5d96577d6509f80a8150e44e8f45f82cdbe0e34c99ab040069", size = 19960, upload-time = "2026-03-04T14:17:07.153Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/77/dd38991db037fdfce45849491cb61de5ab000f49824a00230afb112a4392/opentelemetry_proto-1.40.0.tar.gz", hash = "sha256:03f639ca129ba513f5819810f5b1f42bcb371391405d99c168fe6937c62febcd", size = 45667, upload-time = "2026-03-04T14:17:31.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/b2/189b2577dde745b15625b3214302605b1353436219d42b7912e77fa8dc24/opentelemetry_proto-1.40.0-py3-none-any.whl", hash = "sha256:266c4385d88923a23d63e353e9761af0f47a6ed0d486979777fe4de59dc9b25f", size = 72073, upload-time = "2026-03-04T14:17:16.673Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/fd/3c3125b20ba18ce2155ba9ea74acb0ae5d25f8cd39cfd37455601b7955cc/opentelemetry_sdk-1.40.0.tar.gz", hash = "sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2", size = 184252, upload-time = "2026-03-04T14:17:31.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/c5/6a852903d8bfac758c6dc6e9a68b015d3c33f2f1be5e9591e0f4b69c7e0a/opentelemetry_sdk-1.40.0-py3-none-any.whl", hash = "sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1", size = 141951, upload-time = "2026-03-04T14:17:17.961Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/c0/4ae7973f3c2cfd2b6e321f1675626f0dab0a97027cc7a297474c9c8f3d04/opentelemetry_semantic_conventions-0.61b0.tar.gz", hash = "sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a", size = 145755, upload-time = "2026-03-04T14:17:32.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2", size = 231621, upload-time = "2026-03-04T14:17:19.33Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions-ai" +version = "0.4.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/75/455c15f8360b475dd31101a87eab316420388486f7941bf019cbf4e63d5b/opentelemetry_semantic_conventions_ai-0.4.15.tar.gz", hash = "sha256:12de172d1e11d21c6e82bbf578c7e8a713589a7fda76af9ed785632564a28b81", size = 18595, upload-time = "2026-03-02T15:36:50.254Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/49/819fb212386f77cfd93f81bd916d674f0e735f87c8ac2262ed14e3b852c2/opentelemetry_semantic_conventions_ai-0.4.15-py3-none-any.whl", hash = "sha256:011461f1fba30f27035c49ab3b8344367adc72da0a6c8d3c7428303c6779edc9", size = 5999, upload-time = "2026-03-02T15:36:51.44Z" }, +] + [[package]] name = "outlines-core" version = "0.2.11" @@ -4839,6 +5149,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/1b/5dbe84eefc86f48473947e2f41711aded97eecef1231f4558f1f02713c12/pyzmq-27.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c9f7f6e13dff2e44a6afeaf2cf54cee5929ad64afaf4d40b50f93c58fc687355", size = 544862, upload-time = "2025-09-08T23:09:56.509Z" }, ] +[[package]] +name = "quack-kernels" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "nvidia-cutlass-dsl" }, + { name = "torch" }, + { name = "torch-c-dlpack-ext" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/af/bc7deefb35cbe5f373bb056fc50a1af321776e6c0417fbe3dbaa771f5102/quack_kernels-0.3.3.tar.gz", hash = "sha256:74195548801202ab0ddded0cf82e7ca16a07f8605851df8b058f10eeead34d5f", size = 173675, upload-time = "2026-03-15T18:16:06.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/8d/3c3858c5415e76baab6f17a1325eda2d649d3895728dc2d1d6d551cb9fea/quack_kernels-0.3.3-py3-none-any.whl", hash = "sha256:87a529a2fede72b3f63905e469a16e6e9988b5aceb793d616fb9adc415941a1b", size = 178230, upload-time = "2026-03-15T18:16:05.409Z" }, +] + [[package]] name = "ray" version = "2.53.0" @@ -5383,143 +5708,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/5b/632a58724221ef03d78ab65062e82a1010e1bef8e8e0b9d7c6d7b8044841/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3", size = 531885, upload-time = "2025-11-19T15:18:27.146Z" }, ] -[[package]] -name = "scipy" -version = "1.15.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/2f/4966032c5f8cc7e6a60f1b2e0ad686293b9474b65246b0c642e3ef3badd0/scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c", size = 38702770, upload-time = "2025-05-08T16:04:20.849Z" }, - { url = "https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253", size = 30094511, upload-time = "2025-05-08T16:04:27.103Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b1/4deb37252311c1acff7f101f6453f0440794f51b6eacb1aad4459a134081/scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f", size = 22368151, upload-time = "2025-05-08T16:04:31.731Z" }, - { url = "https://files.pythonhosted.org/packages/38/7d/f457626e3cd3c29b3a49ca115a304cebb8cc6f31b04678f03b216899d3c6/scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92", size = 25121732, upload-time = "2025-05-08T16:04:36.596Z" }, - { url = "https://files.pythonhosted.org/packages/db/0a/92b1de4a7adc7a15dcf5bddc6e191f6f29ee663b30511ce20467ef9b82e4/scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82", size = 35547617, upload-time = "2025-05-08T16:04:43.546Z" }, - { url = "https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40", size = 37662964, upload-time = "2025-05-08T16:04:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/25/e1/3df8f83cb15f3500478c889be8fb18700813b95e9e087328230b98d547ff/scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e", size = 37238749, upload-time = "2025-05-08T16:04:55.215Z" }, - { url = "https://files.pythonhosted.org/packages/93/3e/b3257cf446f2a3533ed7809757039016b74cd6f38271de91682aa844cfc5/scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c", size = 40022383, upload-time = "2025-05-08T16:05:01.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/55bc4881973d3f79b479a5a2e2df61c8c9a04fcb986a213ac9c02cfb659b/scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13", size = 41259201, upload-time = "2025-05-08T16:05:08.166Z" }, - { url = "https://files.pythonhosted.org/packages/96/ab/5cc9f80f28f6a7dff646c5756e559823614a42b1939d86dd0ed550470210/scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b", size = 38714255, upload-time = "2025-05-08T16:05:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/4a/4a/66ba30abe5ad1a3ad15bfb0b59d22174012e8056ff448cb1644deccbfed2/scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba", size = 30111035, upload-time = "2025-05-08T16:05:20.152Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/a7e5b95afd80d24313307f03624acc65801846fa75599034f8ceb9e2cbf6/scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65", size = 22384499, upload-time = "2025-05-08T16:05:24.494Z" }, - { url = "https://files.pythonhosted.org/packages/17/99/f3aaddccf3588bb4aea70ba35328c204cadd89517a1612ecfda5b2dd9d7a/scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1", size = 25152602, upload-time = "2025-05-08T16:05:29.313Z" }, - { url = "https://files.pythonhosted.org/packages/56/c5/1032cdb565f146109212153339f9cb8b993701e9fe56b1c97699eee12586/scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889", size = 35503415, upload-time = "2025-05-08T16:05:34.699Z" }, - { url = "https://files.pythonhosted.org/packages/bd/37/89f19c8c05505d0601ed5650156e50eb881ae3918786c8fd7262b4ee66d3/scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982", size = 37652622, upload-time = "2025-05-08T16:05:40.762Z" }, - { url = "https://files.pythonhosted.org/packages/7e/31/be59513aa9695519b18e1851bb9e487de66f2d31f835201f1b42f5d4d475/scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9", size = 37244796, upload-time = "2025-05-08T16:05:48.119Z" }, - { url = "https://files.pythonhosted.org/packages/10/c0/4f5f3eeccc235632aab79b27a74a9130c6c35df358129f7ac8b29f562ac7/scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594", size = 40047684, upload-time = "2025-05-08T16:05:54.22Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a7/0ddaf514ce8a8714f6ed243a2b391b41dbb65251affe21ee3077ec45ea9a/scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb", size = 41246504, upload-time = "2025-05-08T16:06:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/37/4b/683aa044c4162e10ed7a7ea30527f2cbd92e6999c10a8ed8edb253836e9c/scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019", size = 38766735, upload-time = "2025-05-08T16:06:06.471Z" }, - { url = "https://files.pythonhosted.org/packages/7b/7e/f30be3d03de07f25dc0ec926d1681fed5c732d759ac8f51079708c79e680/scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6", size = 30173284, upload-time = "2025-05-08T16:06:11.686Z" }, - { url = "https://files.pythonhosted.org/packages/07/9c/0ddb0d0abdabe0d181c1793db51f02cd59e4901da6f9f7848e1f96759f0d/scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477", size = 22446958, upload-time = "2025-05-08T16:06:15.97Z" }, - { url = "https://files.pythonhosted.org/packages/af/43/0bce905a965f36c58ff80d8bea33f1f9351b05fad4beaad4eae34699b7a1/scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c", size = 25242454, upload-time = "2025-05-08T16:06:20.394Z" }, - { url = "https://files.pythonhosted.org/packages/56/30/a6f08f84ee5b7b28b4c597aca4cbe545535c39fe911845a96414700b64ba/scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45", size = 35210199, upload-time = "2025-05-08T16:06:26.159Z" }, - { url = "https://files.pythonhosted.org/packages/0b/1f/03f52c282437a168ee2c7c14a1a0d0781a9a4a8962d84ac05c06b4c5b555/scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49", size = 37309455, upload-time = "2025-05-08T16:06:32.778Z" }, - { url = "https://files.pythonhosted.org/packages/89/b1/fbb53137f42c4bf630b1ffdfc2151a62d1d1b903b249f030d2b1c0280af8/scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e", size = 36885140, upload-time = "2025-05-08T16:06:39.249Z" }, - { url = "https://files.pythonhosted.org/packages/2e/2e/025e39e339f5090df1ff266d021892694dbb7e63568edcfe43f892fa381d/scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539", size = 39710549, upload-time = "2025-05-08T16:06:45.729Z" }, - { url = "https://files.pythonhosted.org/packages/e6/eb/3bf6ea8ab7f1503dca3a10df2e4b9c3f6b3316df07f6c0ded94b281c7101/scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed", size = 40966184, upload-time = "2025-05-08T16:06:52.623Z" }, - { url = "https://files.pythonhosted.org/packages/73/18/ec27848c9baae6e0d6573eda6e01a602e5649ee72c27c3a8aad673ebecfd/scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759", size = 38728256, upload-time = "2025-05-08T16:06:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/1aef2184948728b4b6e21267d53b3339762c285a46a274ebb7863c9e4742/scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62", size = 30109540, upload-time = "2025-05-08T16:07:04.209Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d8/59e452c0a255ec352bd0a833537a3bc1bfb679944c4938ab375b0a6b3a3e/scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb", size = 22383115, upload-time = "2025-05-08T16:07:08.998Z" }, - { url = "https://files.pythonhosted.org/packages/08/f5/456f56bbbfccf696263b47095291040655e3cbaf05d063bdc7c7517f32ac/scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730", size = 25163884, upload-time = "2025-05-08T16:07:14.091Z" }, - { url = "https://files.pythonhosted.org/packages/a2/66/a9618b6a435a0f0c0b8a6d0a2efb32d4ec5a85f023c2b79d39512040355b/scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825", size = 35174018, upload-time = "2025-05-08T16:07:19.427Z" }, - { url = "https://files.pythonhosted.org/packages/b5/09/c5b6734a50ad4882432b6bb7c02baf757f5b2f256041da5df242e2d7e6b6/scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7", size = 37269716, upload-time = "2025-05-08T16:07:25.712Z" }, - { url = "https://files.pythonhosted.org/packages/77/0a/eac00ff741f23bcabd352731ed9b8995a0a60ef57f5fd788d611d43d69a1/scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11", size = 36872342, upload-time = "2025-05-08T16:07:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/fe/54/4379be86dd74b6ad81551689107360d9a3e18f24d20767a2d5b9253a3f0a/scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126", size = 39670869, upload-time = "2025-05-08T16:07:38.002Z" }, - { url = "https://files.pythonhosted.org/packages/87/2e/892ad2862ba54f084ffe8cc4a22667eaf9c2bcec6d2bff1d15713c6c0703/scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163", size = 40988851, upload-time = "2025-05-08T16:08:33.671Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e9/7a879c137f7e55b30d75d90ce3eb468197646bc7b443ac036ae3fe109055/scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8", size = 38863011, upload-time = "2025-05-08T16:07:44.039Z" }, - { url = "https://files.pythonhosted.org/packages/51/d1/226a806bbd69f62ce5ef5f3ffadc35286e9fbc802f606a07eb83bf2359de/scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5", size = 30266407, upload-time = "2025-05-08T16:07:49.891Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9b/f32d1d6093ab9eeabbd839b0f7619c62e46cc4b7b6dbf05b6e615bbd4400/scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e", size = 22540030, upload-time = "2025-05-08T16:07:54.121Z" }, - { url = "https://files.pythonhosted.org/packages/e7/29/c278f699b095c1a884f29fda126340fcc201461ee8bfea5c8bdb1c7c958b/scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb", size = 25218709, upload-time = "2025-05-08T16:07:58.506Z" }, - { url = "https://files.pythonhosted.org/packages/24/18/9e5374b617aba742a990581373cd6b68a2945d65cc588482749ef2e64467/scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723", size = 34809045, upload-time = "2025-05-08T16:08:03.929Z" }, - { url = "https://files.pythonhosted.org/packages/e1/fe/9c4361e7ba2927074360856db6135ef4904d505e9b3afbbcb073c4008328/scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb", size = 36703062, upload-time = "2025-05-08T16:08:09.558Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8e/038ccfe29d272b30086b25a4960f757f97122cb2ec42e62b460d02fe98e9/scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4", size = 36393132, upload-time = "2025-05-08T16:08:15.34Z" }, - { url = "https://files.pythonhosted.org/packages/10/7e/5c12285452970be5bdbe8352c619250b97ebf7917d7a9a9e96b8a8140f17/scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5", size = 38979503, upload-time = "2025-05-08T16:08:21.513Z" }, - { url = "https://files.pythonhosted.org/packages/81/06/0a5e5349474e1cbc5757975b21bd4fad0e72ebf138c5592f191646154e06/scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca", size = 40308097, upload-time = "2025-05-08T16:08:27.627Z" }, -] - -[[package]] -name = "scipy" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12' and sys_platform == 'linux'", - "python_full_version >= '3.12' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/56/3e/9cca699f3486ce6bc12ff46dc2031f1ec8eb9ccc9a320fdaf925f1417426/scipy-1.17.0.tar.gz", hash = "sha256:2591060c8e648d8b96439e111ac41fd8342fdeff1876be2e19dea3fe8930454e", size = 30396830, upload-time = "2026-01-10T21:34:23.009Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/4b/c89c131aa87cad2b77a54eb0fb94d633a842420fa7e919dc2f922037c3d8/scipy-1.17.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:2abd71643797bd8a106dff97894ff7869eeeb0af0f7a5ce02e4227c6a2e9d6fd", size = 31381316, upload-time = "2026-01-10T21:24:33.42Z" }, - { url = "https://files.pythonhosted.org/packages/5e/5f/a6b38f79a07d74989224d5f11b55267714707582908a5f1ae854cf9a9b84/scipy-1.17.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:ef28d815f4d2686503e5f4f00edc387ae58dfd7a2f42e348bb53359538f01558", size = 27966760, upload-time = "2026-01-10T21:24:38.911Z" }, - { url = "https://files.pythonhosted.org/packages/c1/20/095ad24e031ee8ed3c5975954d816b8e7e2abd731e04f8be573de8740885/scipy-1.17.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:272a9f16d6bb4667e8b50d25d71eddcc2158a214df1b566319298de0939d2ab7", size = 20138701, upload-time = "2026-01-10T21:24:43.249Z" }, - { url = "https://files.pythonhosted.org/packages/89/11/4aad2b3858d0337756f3323f8960755704e530b27eb2a94386c970c32cbe/scipy-1.17.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:7204fddcbec2fe6598f1c5fdf027e9f259106d05202a959a9f1aecf036adc9f6", size = 22480574, upload-time = "2026-01-10T21:24:47.266Z" }, - { url = "https://files.pythonhosted.org/packages/85/bd/f5af70c28c6da2227e510875cadf64879855193a687fb19951f0f44cfd6b/scipy-1.17.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc02c37a5639ee67d8fb646ffded6d793c06c5622d36b35cfa8fe5ececb8f042", size = 32862414, upload-time = "2026-01-10T21:24:52.566Z" }, - { url = "https://files.pythonhosted.org/packages/ef/df/df1457c4df3826e908879fe3d76bc5b6e60aae45f4ee42539512438cfd5d/scipy-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dac97a27520d66c12a34fd90a4fe65f43766c18c0d6e1c0a80f114d2260080e4", size = 35112380, upload-time = "2026-01-10T21:24:58.433Z" }, - { url = "https://files.pythonhosted.org/packages/5f/bb/88e2c16bd1dd4de19d80d7c5e238387182993c2fb13b4b8111e3927ad422/scipy-1.17.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb7446a39b3ae0fe8f416a9a3fdc6fba3f11c634f680f16a239c5187bc487c0", size = 34922676, upload-time = "2026-01-10T21:25:04.287Z" }, - { url = "https://files.pythonhosted.org/packages/02/ba/5120242cc735f71fc002cff0303d536af4405eb265f7c60742851e7ccfe9/scipy-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:474da16199f6af66601a01546144922ce402cb17362e07d82f5a6cf8f963e449", size = 37507599, upload-time = "2026-01-10T21:25:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/52/c8/08629657ac6c0da198487ce8cd3de78e02cfde42b7f34117d56a3fe249dc/scipy-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:255c0da161bd7b32a6c898e7891509e8a9289f0b1c6c7d96142ee0d2b114c2ea", size = 36380284, upload-time = "2026-01-10T21:25:15.632Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4a/465f96d42c6f33ad324a40049dfd63269891db9324aa66c4a1c108c6f994/scipy-1.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:85b0ac3ad17fa3be50abd7e69d583d98792d7edc08367e01445a1e2076005379", size = 24370427, upload-time = "2026-01-10T21:25:20.514Z" }, - { url = "https://files.pythonhosted.org/packages/0b/11/7241a63e73ba5a516f1930ac8d5b44cbbfabd35ac73a2d08ca206df007c4/scipy-1.17.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:0d5018a57c24cb1dd828bcf51d7b10e65986d549f52ef5adb6b4d1ded3e32a57", size = 31364580, upload-time = "2026-01-10T21:25:25.717Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1d/5057f812d4f6adc91a20a2d6f2ebcdb517fdbc87ae3acc5633c9b97c8ba5/scipy-1.17.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:88c22af9e5d5a4f9e027e26772cc7b5922fab8bcc839edb3ae33de404feebd9e", size = 27969012, upload-time = "2026-01-10T21:25:30.921Z" }, - { url = "https://files.pythonhosted.org/packages/e3/21/f6ec556c1e3b6ec4e088da667d9987bb77cc3ab3026511f427dc8451187d/scipy-1.17.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f3cd947f20fe17013d401b64e857c6b2da83cae567adbb75b9dcba865abc66d8", size = 20140691, upload-time = "2026-01-10T21:25:34.802Z" }, - { url = "https://files.pythonhosted.org/packages/7a/fe/5e5ad04784964ba964a96f16c8d4676aa1b51357199014dce58ab7ec5670/scipy-1.17.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e8c0b331c2c1f531eb51f1b4fc9ba709521a712cce58f1aa627bc007421a5306", size = 22463015, upload-time = "2026-01-10T21:25:39.277Z" }, - { url = "https://files.pythonhosted.org/packages/4a/69/7c347e857224fcaf32a34a05183b9d8a7aca25f8f2d10b8a698b8388561a/scipy-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5194c445d0a1c7a6c1a4a4681b6b7c71baad98ff66d96b949097e7513c9d6742", size = 32724197, upload-time = "2026-01-10T21:25:44.084Z" }, - { url = "https://files.pythonhosted.org/packages/d1/fe/66d73b76d378ba8cc2fe605920c0c75092e3a65ae746e1e767d9d020a75a/scipy-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9eeb9b5f5997f75507814ed9d298ab23f62cf79f5a3ef90031b1ee2506abdb5b", size = 35009148, upload-time = "2026-01-10T21:25:50.591Z" }, - { url = "https://files.pythonhosted.org/packages/af/07/07dec27d9dc41c18d8c43c69e9e413431d20c53a0339c388bcf72f353c4b/scipy-1.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:40052543f7bbe921df4408f46003d6f01c6af109b9e2c8a66dd1cf6cf57f7d5d", size = 34798766, upload-time = "2026-01-10T21:25:59.41Z" }, - { url = "https://files.pythonhosted.org/packages/81/61/0470810c8a093cdacd4ba7504b8a218fd49ca070d79eca23a615f5d9a0b0/scipy-1.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0cf46c8013fec9d3694dc572f0b54100c28405d55d3e2cb15e2895b25057996e", size = 37405953, upload-time = "2026-01-10T21:26:07.75Z" }, - { url = "https://files.pythonhosted.org/packages/92/ce/672ed546f96d5d41ae78c4b9b02006cedd0b3d6f2bf5bb76ea455c320c28/scipy-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:0937a0b0d8d593a198cededd4c439a0ea216a3f36653901ea1f3e4be949056f8", size = 36328121, upload-time = "2026-01-10T21:26:16.509Z" }, - { url = "https://files.pythonhosted.org/packages/9d/21/38165845392cae67b61843a52c6455d47d0cc2a40dd495c89f4362944654/scipy-1.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:f603d8a5518c7426414d1d8f82e253e454471de682ce5e39c29adb0df1efb86b", size = 24314368, upload-time = "2026-01-10T21:26:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/0c/51/3468fdfd49387ddefee1636f5cf6d03ce603b75205bf439bbf0e62069bfd/scipy-1.17.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:65ec32f3d32dfc48c72df4291345dae4f048749bc8d5203ee0a3f347f96c5ce6", size = 31344101, upload-time = "2026-01-10T21:26:30.25Z" }, - { url = "https://files.pythonhosted.org/packages/b2/9a/9406aec58268d437636069419e6977af953d1e246df941d42d3720b7277b/scipy-1.17.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:1f9586a58039d7229ce77b52f8472c972448cded5736eaf102d5658bbac4c269", size = 27950385, upload-time = "2026-01-10T21:26:36.801Z" }, - { url = "https://files.pythonhosted.org/packages/4f/98/e7342709e17afdfd1b26b56ae499ef4939b45a23a00e471dfb5375eea205/scipy-1.17.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9fad7d3578c877d606b1150135c2639e9de9cecd3705caa37b66862977cc3e72", size = 20122115, upload-time = "2026-01-10T21:26:42.107Z" }, - { url = "https://files.pythonhosted.org/packages/fd/0e/9eeeb5357a64fd157cbe0302c213517c541cc16b8486d82de251f3c68ede/scipy-1.17.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:423ca1f6584fc03936972b5f7c06961670dbba9f234e71676a7c7ccf938a0d61", size = 22442402, upload-time = "2026-01-10T21:26:48.029Z" }, - { url = "https://files.pythonhosted.org/packages/c9/10/be13397a0e434f98e0c79552b2b584ae5bb1c8b2be95db421533bbca5369/scipy-1.17.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe508b5690e9eaaa9467fc047f833af58f1152ae51a0d0aed67aa5801f4dd7d6", size = 32696338, upload-time = "2026-01-10T21:26:55.521Z" }, - { url = "https://files.pythonhosted.org/packages/63/1e/12fbf2a3bb240161651c94bb5cdd0eae5d4e8cc6eaeceb74ab07b12a753d/scipy-1.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6680f2dfd4f6182e7d6db161344537da644d1cf85cf293f015c60a17ecf08752", size = 34977201, upload-time = "2026-01-10T21:27:03.501Z" }, - { url = "https://files.pythonhosted.org/packages/19/5b/1a63923e23ccd20bd32156d7dd708af5bbde410daa993aa2500c847ab2d2/scipy-1.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eec3842ec9ac9de5917899b277428886042a93db0b227ebbe3a333b64ec7643d", size = 34777384, upload-time = "2026-01-10T21:27:11.423Z" }, - { url = "https://files.pythonhosted.org/packages/39/22/b5da95d74edcf81e540e467202a988c50fef41bd2011f46e05f72ba07df6/scipy-1.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d7425fcafbc09a03731e1bc05581f5fad988e48c6a861f441b7ab729a49a55ea", size = 37379586, upload-time = "2026-01-10T21:27:20.171Z" }, - { url = "https://files.pythonhosted.org/packages/b9/b6/8ac583d6da79e7b9e520579f03007cb006f063642afd6b2eeb16b890bf93/scipy-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:87b411e42b425b84777718cc41516b8a7e0795abfa8e8e1d573bf0ef014f0812", size = 36287211, upload-time = "2026-01-10T21:28:43.122Z" }, - { url = "https://files.pythonhosted.org/packages/55/fb/7db19e0b3e52f882b420417644ec81dd57eeef1bd1705b6f689d8ff93541/scipy-1.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:357ca001c6e37601066092e7c89cca2f1ce74e2a520ca78d063a6d2201101df2", size = 24312646, upload-time = "2026-01-10T21:28:49.893Z" }, - { url = "https://files.pythonhosted.org/packages/20/b6/7feaa252c21cc7aff335c6c55e1b90ab3e3306da3f048109b8b639b94648/scipy-1.17.0-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:ec0827aa4d36cb79ff1b81de898e948a51ac0b9b1c43e4a372c0508c38c0f9a3", size = 31693194, upload-time = "2026-01-10T21:27:27.454Z" }, - { url = "https://files.pythonhosted.org/packages/76/bb/bbb392005abce039fb7e672cb78ac7d158700e826b0515cab6b5b60c26fb/scipy-1.17.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:819fc26862b4b3c73a60d486dbb919202f3d6d98c87cf20c223511429f2d1a97", size = 28365415, upload-time = "2026-01-10T21:27:34.26Z" }, - { url = "https://files.pythonhosted.org/packages/37/da/9d33196ecc99fba16a409c691ed464a3a283ac454a34a13a3a57c0d66f3a/scipy-1.17.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:363ad4ae2853d88ebcde3ae6ec46ccca903ea9835ee8ba543f12f575e7b07e4e", size = 20537232, upload-time = "2026-01-10T21:27:40.306Z" }, - { url = "https://files.pythonhosted.org/packages/56/9d/f4b184f6ddb28e9a5caea36a6f98e8ecd2a524f9127354087ce780885d83/scipy-1.17.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:979c3a0ff8e5ba254d45d59ebd38cde48fce4f10b5125c680c7a4bfe177aab07", size = 22791051, upload-time = "2026-01-10T21:27:46.539Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/025cccdd738a72140efc582b1641d0dd4caf2e86c3fb127568dc80444e6e/scipy-1.17.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:130d12926ae34399d157de777472bf82e9061c60cc081372b3118edacafe1d00", size = 32815098, upload-time = "2026-01-10T21:27:54.389Z" }, - { url = "https://files.pythonhosted.org/packages/48/5f/09b879619f8bca15ce392bfc1894bd9c54377e01d1b3f2f3b595a1b4d945/scipy-1.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e886000eb4919eae3a44f035e63f0fd8b651234117e8f6f29bad1cd26e7bc45", size = 35031342, upload-time = "2026-01-10T21:28:03.012Z" }, - { url = "https://files.pythonhosted.org/packages/f2/9a/f0f0a9f0aa079d2f106555b984ff0fbb11a837df280f04f71f056ea9c6e4/scipy-1.17.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:13c4096ac6bc31d706018f06a49abe0485f96499deb82066b94d19b02f664209", size = 34893199, upload-time = "2026-01-10T21:28:10.832Z" }, - { url = "https://files.pythonhosted.org/packages/90/b8/4f0f5cf0c5ea4d7548424e6533e6b17d164f34a6e2fb2e43ffebb6697b06/scipy-1.17.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cacbaddd91fcffde703934897c5cd2c7cb0371fac195d383f4e1f1c5d3f3bd04", size = 37438061, upload-time = "2026-01-10T21:28:19.684Z" }, - { url = "https://files.pythonhosted.org/packages/f9/cc/2bd59140ed3b2fa2882fb15da0a9cb1b5a6443d67cfd0d98d4cec83a57ec/scipy-1.17.0-cp313-cp313t-win_amd64.whl", hash = "sha256:edce1a1cf66298cccdc48a1bdf8fb10a3bf58e8b58d6c3883dd1530e103f87c0", size = 36328593, upload-time = "2026-01-10T21:28:28.007Z" }, - { url = "https://files.pythonhosted.org/packages/13/1b/c87cc44a0d2c7aaf0f003aef2904c3d097b422a96c7e7c07f5efd9073c1b/scipy-1.17.0-cp313-cp313t-win_arm64.whl", hash = "sha256:30509da9dbec1c2ed8f168b8d8aa853bc6723fede1dbc23c7d43a56f5ab72a67", size = 24625083, upload-time = "2026-01-10T21:28:35.188Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2d/51006cd369b8e7879e1c630999a19d1fbf6f8b5ed3e33374f29dc87e53b3/scipy-1.17.0-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:c17514d11b78be8f7e6331b983a65a7f5ca1fd037b95e27b280921fe5606286a", size = 31346803, upload-time = "2026-01-10T21:28:57.24Z" }, - { url = "https://files.pythonhosted.org/packages/d6/2e/2349458c3ce445f53a6c93d4386b1c4c5c0c540917304c01222ff95ff317/scipy-1.17.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:4e00562e519c09da34c31685f6acc3aa384d4d50604db0f245c14e1b4488bfa2", size = 27967182, upload-time = "2026-01-10T21:29:04.107Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7c/df525fbfa77b878d1cfe625249529514dc02f4fd5f45f0f6295676a76528/scipy-1.17.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7df7941d71314e60a481e02d5ebcb3f0185b8d799c70d03d8258f6c80f3d467", size = 20139125, upload-time = "2026-01-10T21:29:10.179Z" }, - { url = "https://files.pythonhosted.org/packages/33/11/fcf9d43a7ed1234d31765ec643b0515a85a30b58eddccc5d5a4d12b5f194/scipy-1.17.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:aabf057c632798832f071a8dde013c2e26284043934f53b00489f1773b33527e", size = 22443554, upload-time = "2026-01-10T21:29:15.888Z" }, - { url = "https://files.pythonhosted.org/packages/80/5c/ea5d239cda2dd3d31399424967a24d556cf409fbea7b5b21412b0fd0a44f/scipy-1.17.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a38c3337e00be6fd8a95b4ed66b5d988bac4ec888fd922c2ea9fe5fb1603dd67", size = 32757834, upload-time = "2026-01-10T21:29:23.406Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7e/8c917cc573310e5dc91cbeead76f1b600d3fb17cf0969db02c9cf92e3cfa/scipy-1.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00fb5f8ec8398ad90215008d8b6009c9db9fa924fd4c7d6be307c6f945f9cd73", size = 34995775, upload-time = "2026-01-10T21:29:31.915Z" }, - { url = "https://files.pythonhosted.org/packages/c5/43/176c0c3c07b3f7df324e7cdd933d3e2c4898ca202b090bd5ba122f9fe270/scipy-1.17.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f2a4942b0f5f7c23c7cd641a0ca1955e2ae83dedcff537e3a0259096635e186b", size = 34841240, upload-time = "2026-01-10T21:29:39.995Z" }, - { url = "https://files.pythonhosted.org/packages/44/8c/d1f5f4b491160592e7f084d997de53a8e896a3ac01cd07e59f43ca222744/scipy-1.17.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf133ced83889583156566d2bdf7a07ff89228fe0c0cb727f777de92092ec6b", size = 37394463, upload-time = "2026-01-10T21:29:48.723Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ec/42a6657f8d2d087e750e9a5dde0b481fd135657f09eaf1cf5688bb23c338/scipy-1.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:3625c631a7acd7cfd929e4e31d2582cf00f42fcf06011f59281271746d77e061", size = 37053015, upload-time = "2026-01-10T21:30:51.418Z" }, - { url = "https://files.pythonhosted.org/packages/27/58/6b89a6afd132787d89a362d443a7bddd511b8f41336a1ae47f9e4f000dc4/scipy-1.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:9244608d27eafe02b20558523ba57f15c689357c85bdcfe920b1828750aa26eb", size = 24951312, upload-time = "2026-01-10T21:30:56.771Z" }, - { url = "https://files.pythonhosted.org/packages/e9/01/f58916b9d9ae0112b86d7c3b10b9e685625ce6e8248df139d0fcb17f7397/scipy-1.17.0-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:2b531f57e09c946f56ad0b4a3b2abee778789097871fc541e267d2eca081cff1", size = 31706502, upload-time = "2026-01-10T21:29:56.326Z" }, - { url = "https://files.pythonhosted.org/packages/59/8e/2912a87f94a7d1f8b38aabc0faf74b82d3b6c9e22be991c49979f0eceed8/scipy-1.17.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:13e861634a2c480bd237deb69333ac79ea1941b94568d4b0efa5db5e263d4fd1", size = 28380854, upload-time = "2026-01-10T21:30:01.554Z" }, - { url = "https://files.pythonhosted.org/packages/bd/1c/874137a52dddab7d5d595c1887089a2125d27d0601fce8c0026a24a92a0b/scipy-1.17.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:eb2651271135154aa24f6481cbae5cc8af1f0dd46e6533fb7b56aa9727b6a232", size = 20552752, upload-time = "2026-01-10T21:30:05.93Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f0/7518d171cb735f6400f4576cf70f756d5b419a07fe1867da34e2c2c9c11b/scipy-1.17.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:c5e8647f60679790c2f5c76be17e2e9247dc6b98ad0d3b065861e082c56e078d", size = 22803972, upload-time = "2026-01-10T21:30:10.651Z" }, - { url = "https://files.pythonhosted.org/packages/7c/74/3498563a2c619e8a3ebb4d75457486c249b19b5b04a30600dfd9af06bea5/scipy-1.17.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5fb10d17e649e1446410895639f3385fd2bf4c3c7dfc9bea937bddcbc3d7b9ba", size = 32829770, upload-time = "2026-01-10T21:30:16.359Z" }, - { url = "https://files.pythonhosted.org/packages/48/d1/7b50cedd8c6c9d6f706b4b36fa8544d829c712a75e370f763b318e9638c1/scipy-1.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8547e7c57f932e7354a2319fab613981cde910631979f74c9b542bb167a8b9db", size = 35051093, upload-time = "2026-01-10T21:30:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/e2/82/a2d684dfddb87ba1b3ea325df7c3293496ee9accb3a19abe9429bce94755/scipy-1.17.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:33af70d040e8af9d5e7a38b5ed3b772adddd281e3062ff23fec49e49681c38cf", size = 34909905, upload-time = "2026-01-10T21:30:28.704Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5e/e565bd73991d42023eb82bb99e51c5b3d9e2c588ca9d4b3e2cc1d3ca62a6/scipy-1.17.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb55bb97d00f8b7ab95cb64f873eb0bf54d9446264d9f3609130381233483f", size = 37457743, upload-time = "2026-01-10T21:30:34.819Z" }, - { url = "https://files.pythonhosted.org/packages/58/a8/a66a75c3d8f1fb2b83f66007d6455a06a6f6cf5618c3dc35bc9b69dd096e/scipy-1.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1ff269abf702f6c7e67a4b7aad981d42871a11b9dd83c58d2d2ea624efbd1088", size = 37098574, upload-time = "2026-01-10T21:30:40.782Z" }, - { url = "https://files.pythonhosted.org/packages/56/a5/df8f46ef7da168f1bc52cd86e09a9de5c6f19cc1da04454d51b7d4f43408/scipy-1.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:031121914e295d9791319a1875444d55079885bbae5bdc9c5e0f2ee5f09d34ff", size = 25246266, upload-time = "2026-01-10T21:30:45.923Z" }, -] - [[package]] name = "sentencepiece" version = "0.2.1" @@ -5776,7 +5964,9 @@ dependencies = [ { name = "asciichartpy" }, { name = "boto3" }, { name = "configurize" }, + { name = "diskcache" }, { name = "einops" }, + { name = "emoji" }, { name = "fastapi" }, { name = "hjson" }, { name = "httpx" }, @@ -5784,6 +5974,7 @@ dependencies = [ { name = "loguru" }, { name = "megfile" }, { name = "msgpack" }, + { name = "nltk" }, { name = "numpy" }, { name = "packaging" }, { name = "pip" }, @@ -5791,12 +5982,14 @@ dependencies = [ { name = "redis" }, { name = "safetensors" }, { name = "setuptools" }, + { name = "syllapy" }, { name = "tabulate" }, { name = "tensorboard" }, { name = "torch" }, { name = "tqdm" }, { name = "transformers" }, - { name = "triton" }, + { name = "triton", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, { name = "uvicorn" }, { name = "vllm" }, { name = "wandb" }, @@ -5830,7 +6023,9 @@ requires-dist = [ { name = "asciichartpy", specifier = ">=1.5.25" }, { name = "boto3" }, { name = "configurize", specifier = "==0.2.0" }, + { name = "diskcache", specifier = ">=5.6.3" }, { name = "einops" }, + { name = "emoji" }, { name = "fastapi" }, { name = "hjson" }, { name = "httpx" }, @@ -5838,6 +6033,7 @@ requires-dist = [ { name = "loguru" }, { name = "megfile" }, { name = "msgpack" }, + { name = "nltk" }, { name = "numpy" }, { name = "packaging" }, { name = "pip", specifier = ">=25.3" }, @@ -5845,14 +6041,15 @@ requires-dist = [ { name = "redis" }, { name = "safetensors" }, { name = "setuptools" }, + { name = "syllapy" }, { name = "tabulate" }, { name = "tensorboard", specifier = ">=2.20.0" }, - { name = "torch", specifier = "==2.9.0" }, + { name = "torch", specifier = ">=2.9.0" }, { name = "tqdm" }, - { name = "transformers", specifier = "<5.0" }, - { name = "triton", specifier = "==3.5.0" }, + { name = "transformers" }, + { name = "triton", specifier = ">=3.5.0" }, { name = "uvicorn" }, - { name = "vllm", specifier = ">=0.11" }, + { name = "vllm", specifier = ">=0.16" }, { name = "wandb" }, ] @@ -5885,6 +6082,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, ] +[[package]] +name = "syllapy" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/a1/7bc1ce4852e14ab5f3153262639742ae63fbfa626507dac4ab919a1e5232/syllapy-0.7.2.tar.gz", hash = "sha256:e55a7ad97d8b232e174b83f91b8f9be0c355d2a8e1208c7f6229055189605564", size = 25561, upload-time = "2022-08-29T01:55:03.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/cc/ffc9bddc146f14e8792a9b05b2bd1bc5f23f3b752a06e96b244780ce55b9/syllapy-0.7.2-py3-none-any.whl", hash = "sha256:198a7413033c32d7b31e21962efb3f284bcea80d3346e954b938ca1ebe6bee20", size = 24882, upload-time = "2022-08-29T01:55:01.1Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -6092,9 +6298,10 @@ wheels = [ [[package]] name = "torch" -version = "2.9.0" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cuda-bindings", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "filelock" }, { name = "fsspec" }, { name = "jinja2" }, @@ -6117,81 +6324,123 @@ dependencies = [ { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, { name = "sympy" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/86/245c240d2138c17ed572c943c289056c2721abab70810d772c6bf5495b28/torch-2.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:030bbfe367379ae6a4ae4042b6c44da25383343b8b3c68abaa9c7231efbaf2dd", size = 104213554, upload-time = "2025-10-15T15:45:59.798Z" }, - { url = "https://files.pythonhosted.org/packages/58/1d/fd1e88ae0948825efcab7dd66d12bec23f05d4d38ed81573c8d453c14c06/torch-2.9.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:51cb63902182a78e90886e8068befd8ea102af4b00e420263591a3d70c7d3c6c", size = 899795167, upload-time = "2025-10-15T15:47:12.695Z" }, - { url = "https://files.pythonhosted.org/packages/63/5a/496197b45c14982bef4e079b24c61dc108e3ab0d0cc9718dba9f54f45a46/torch-2.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:3f6aad4d2f0ee2248bac25339d74858ff846c3969b27d14ac235821f055af83d", size = 109310314, upload-time = "2025-10-15T15:46:16.633Z" }, - { url = "https://files.pythonhosted.org/packages/58/b0/2b4e647b0fc706e88eb6c253d05511865578f5f67b55fad639bf3272a4a1/torch-2.9.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:413e1654c9203733138858780e184d9fc59442f0b3b209e16f39354eb893db9b", size = 74452019, upload-time = "2025-10-15T15:46:04.296Z" }, - { url = "https://files.pythonhosted.org/packages/58/fe/334225e6330e672b36aef23d77451fa906ea12881570c08638a91331a212/torch-2.9.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c596708b5105d0b199215acf0c9be7c1db5f1680d88eddadf4b75a299259a677", size = 104230578, upload-time = "2025-10-15T15:46:08.182Z" }, - { url = "https://files.pythonhosted.org/packages/05/cc/49566caaa218872ec9a2912456f470ff92649894a4bc2e5274aa9ef87c4a/torch-2.9.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:51de31219c97c51cf4bf2be94d622e3deb5dcc526c6dc00e97c17eaec0fc1d67", size = 899815990, upload-time = "2025-10-15T15:48:03.336Z" }, - { url = "https://files.pythonhosted.org/packages/74/25/e9ab21d5925b642d008f139d4a3c9664fc9ee1faafca22913c080cc4c0a5/torch-2.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:dd515c70059afd95f48b8192733764c08ca37a1d19803af6401b5ecad7c8676e", size = 109313698, upload-time = "2025-10-15T15:46:12.425Z" }, - { url = "https://files.pythonhosted.org/packages/b3/b7/205ef3e94de636feffd64b28bb59a0dfac0771221201b9871acf9236f5ca/torch-2.9.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:614a185e4986326d526a91210c8fc1397e76e8cfafa78baf6296a790e53a9eec", size = 74463678, upload-time = "2025-10-15T15:46:29.779Z" }, - { url = "https://files.pythonhosted.org/packages/d1/d3/3985739f3b8e88675127bf70f82b3a48ae083e39cda56305dbd90398fec0/torch-2.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e5f7af1dc4c0a7c4a260c2534f41ddaf209714f7c89145e644c44712fbd6b642", size = 104107898, upload-time = "2025-10-15T15:46:20.883Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4b/f4bb2e6c25d0272f798cd6d7a04ed315da76cec68c602d87040c7847287f/torch-2.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:01cff95ecd9a212ea2f141db28acccdceb6a4c54f64e6c51091146f5e2a772c6", size = 899738273, upload-time = "2025-10-15T15:50:04.188Z" }, - { url = "https://files.pythonhosted.org/packages/66/11/c1c5ba6691cda6279087c35bd626536e4fd29521fe740abf5008377a9a02/torch-2.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4582b162f541651f0cb184d3e291c05c2f556c7117c64a9873e2ee158d40062b", size = 109280887, upload-time = "2025-10-15T15:46:26.228Z" }, - { url = "https://files.pythonhosted.org/packages/dd/5f/b85bd8c05312d71de9402bf5868d217c38827cfd09d8f8514e5be128a52b/torch-2.9.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:33f58e9a102a91259af289d50525c30323b5c9ae1d31322b6447c0814da68695", size = 74478983, upload-time = "2025-10-15T15:46:39.406Z" }, - { url = "https://files.pythonhosted.org/packages/c2/1c/90eb13833cdf4969ea9707586d7b57095c3b6e2b223a7256bf111689bcb8/torch-2.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c30a17fc83eeab346913e237c64b15b5ba6407fff812f6c541e322e19bc9ea0e", size = 104111330, upload-time = "2025-10-15T15:46:35.238Z" }, - { url = "https://files.pythonhosted.org/packages/0e/21/2254c54b8d523592c25ef4434769aa23e29b1e6bf5f4c0ad9e27bf442927/torch-2.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f25033b8667b57857dfd01458fbf2a9e6a6df1f8def23aef0dc46292f6aa642", size = 899750243, upload-time = "2025-10-15T15:48:57.459Z" }, - { url = "https://files.pythonhosted.org/packages/b7/a5/5cb94fa4fd1e78223455c23c200f30f6dc10c6d4a2bcc8f6e7f2a2588370/torch-2.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:d037f1b4ffd25013be4a7bf3651a0a910c68554956c7b2c92ebe87c76475dece", size = 109284513, upload-time = "2025-10-15T15:46:45.061Z" }, - { url = "https://files.pythonhosted.org/packages/66/e8/fc414d8656250ee46120b44836ffbb3266343db424b3e18ca79ebbf69d4f/torch-2.9.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e4e5b5cba837a2a8d1a497ba9a58dae46fa392593eaa13b871c42f71847503a5", size = 74830362, upload-time = "2025-10-15T15:46:48.983Z" }, - { url = "https://files.pythonhosted.org/packages/ed/5f/9474c98fc5ae0cd04b9466035428cd360e6611a86b8352a0fc2fa504acdc/torch-2.9.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:64693568f5dc4dbd5f880a478b1cea0201cc6b510d91d1bc54fea86ac5d1a637", size = 104144940, upload-time = "2025-10-15T15:47:29.076Z" }, - { url = "https://files.pythonhosted.org/packages/2d/5a/8e0c1cf57830172c109d4bd6be2708cabeaf550983eee7029291322447a0/torch-2.9.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:f8ed31ddd7d10bfb3fbe0b9fe01b1243577f13d75e6f4a0839a283915ce3791e", size = 899744054, upload-time = "2025-10-15T15:48:29.864Z" }, - { url = "https://files.pythonhosted.org/packages/6d/28/82c28b30fcb4b7c9cdd995763d18bbb830d6521356712faebbad92ffa61d/torch-2.9.0-cp313-cp313t-win_amd64.whl", hash = "sha256:eff527d4e4846e6f70d2afd8058b73825761203d66576a7e04ea2ecfebcb4ab8", size = 109517546, upload-time = "2025-10-15T15:47:33.395Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a91f96ec74347fa5fd24453fa514bc61c61ecc79196fa760b012a1873d96/torch-2.9.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:f8877779cf56d1ce431a7636703bdb13307f5960bb1af49716d8b179225e0e6a", size = 74480732, upload-time = "2025-10-15T15:47:38.002Z" }, - { url = "https://files.pythonhosted.org/packages/5c/73/9f70af34b334a7e0ef496ceec96b7ec767bd778ea35385ce6f77557534d1/torch-2.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e614fae699838038d888729f82b687c03413c5989ce2a9481f9a7e7a396e0bb", size = 74433037, upload-time = "2025-10-15T15:47:41.894Z" }, - { url = "https://files.pythonhosted.org/packages/b7/84/37cf88625901934c97109e583ecc21777d21c6f54cda97a7e5bbad1ee2f2/torch-2.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:dfb5b8cd310ba3436c7e14e8b7833ef658cf3045e50d2bdaed23c8fc517065eb", size = 104116482, upload-time = "2025-10-15T15:47:46.266Z" }, - { url = "https://files.pythonhosted.org/packages/56/8e/ca8b17866943a8d4f4664d402ea84210aa274588b4c5d89918f5caa24eec/torch-2.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b3d29524993a478e46f5d598b249cd824b7ed98d7fba538bd9c4cde6c803948f", size = 899746916, upload-time = "2025-10-15T15:50:40.294Z" }, - { url = "https://files.pythonhosted.org/packages/43/65/3b17c0fbbdab6501c5b320a52a648628d0d44e7379f64e27d9eef701b6bf/torch-2.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:71c7578984f5ec0eb645eb4816ac8435fcf3e3e2ae1901bcd2f519a9cafb5125", size = 109275151, upload-time = "2025-10-15T15:49:20.715Z" }, - { url = "https://files.pythonhosted.org/packages/83/36/74f8c051f785500396e42f93542422422dfd874a174f21f8d955d36e5d64/torch-2.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:71d9309aee457bbe0b164bce2111cd911c4ed4e847e65d5077dbbcd3aba6befc", size = 74823353, upload-time = "2025-10-15T15:49:16.59Z" }, - { url = "https://files.pythonhosted.org/packages/62/51/dc3b4e2f9ba98ae27238f0153ca098bf9340b2dafcc67fde645d496dfc2a/torch-2.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c08fb654d783899e204a32cca758a7ce8a45b2d78eeb89517cc937088316f78e", size = 104140340, upload-time = "2025-10-15T15:50:19.67Z" }, - { url = "https://files.pythonhosted.org/packages/c0/8d/b00657f8141ac16af7bb6cda2e67de18499a3263b78d516b9a93fcbc98e3/torch-2.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ec8feb0099b2daa5728fbc7abb0b05730fd97e0f359ff8bda09865aaa7bd7d4b", size = 899731750, upload-time = "2025-10-15T15:49:36.673Z" }, - { url = "https://files.pythonhosted.org/packages/fc/29/bd361e0cbb2c79ce6450f42643aaf6919956f89923a50571b0ebfe92d142/torch-2.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:695ba920f234ad4170c9c50e28d56c848432f8f530e6bc7f88fcb15ddf338e75", size = 109503850, upload-time = "2025-10-15T15:50:24.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:2b980edd8d7c0a68c4e951ee1856334a43193f98730d97408fbd148c1a933313", size = 79411457, upload-time = "2026-02-10T21:44:59.189Z" }, + { url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" }, + { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, + { url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" }, + { url = "https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321", size = 915510070, upload-time = "2026-03-11T14:16:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/7b562f1808d3f65414cd80a4f7d4bb00979d9355616c034c171249e1a303/torch-2.10.0-3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac5bdcbb074384c66fa160c15b1ead77839e3fe7ed117d667249afce0acabfac", size = 915518691, upload-time = "2026-03-11T14:15:43.147Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/39/590742415c3030551944edc2ddc273ea1fdfe8ffb2780992e824f1ebee98/torch-2.10.0-3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b1d5e2aba4eb7f8e87fbe04f86442887f9167a35f092afe4c237dfcaaef6e328", size = 915632474, upload-time = "2026-03-11T14:15:13.666Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8e/34949484f764dde5b222b7fe3fede43e4a6f0da9d7f8c370bb617d629ee2/torch-2.10.0-3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0228d20b06701c05a8f978357f657817a4a63984b0c90745def81c18aedfa591", size = 915523882, upload-time = "2026-03-11T14:14:46.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/1a/c61f36cfd446170ec27b3a4984f072fd06dab6b5d7ce27e11adb35d6c838/torch-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5276fa790a666ee8becaffff8acb711922252521b28fbce5db7db5cf9cb2026d", size = 145992962, upload-time = "2026-01-21T16:24:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/b5/60/6662535354191e2d1555296045b63e4279e5a9dbad49acf55a5d38655a39/torch-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aaf663927bcd490ae971469a624c322202a2a1e68936eb952535ca4cd3b90444", size = 915599237, upload-time = "2026-01-21T16:23:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/40/b8/66bbe96f0d79be2b5c697b2e0b187ed792a15c6c4b8904613454651db848/torch-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:a4be6a2a190b32ff5c8002a0977a25ea60e64f7ba46b1be37093c141d9c49aeb", size = 113720931, upload-time = "2026-01-21T16:24:23.743Z" }, + { url = "https://files.pythonhosted.org/packages/76/bb/d820f90e69cda6c8169b32a0c6a3ab7b17bf7990b8f2c680077c24a3c14c/torch-2.10.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:35e407430795c8d3edb07a1d711c41cc1f9eaddc8b2f1cc0a165a6767a8fb73d", size = 79411450, upload-time = "2026-01-21T16:25:30.692Z" }, + { url = "https://files.pythonhosted.org/packages/78/89/f5554b13ebd71e05c0b002f95148033e730d3f7067f67423026cc9c69410/torch-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3282d9febd1e4e476630a099692b44fdc214ee9bf8ee5377732d9d9dfe5712e4", size = 145992610, upload-time = "2026-01-21T16:25:26.327Z" }, + { url = "https://files.pythonhosted.org/packages/ae/30/a3a2120621bf9c17779b169fc17e3dc29b230c29d0f8222f499f5e159aa8/torch-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2f9edd8dbc99f62bc4dfb78af7bf89499bca3d753423ac1b4e06592e467b763", size = 915607863, upload-time = "2026-01-21T16:25:06.696Z" }, + { url = "https://files.pythonhosted.org/packages/6f/3d/c87b33c5f260a2a8ad68da7147e105f05868c281c63d65ed85aa4da98c66/torch-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:29b7009dba4b7a1c960260fc8ac85022c784250af43af9fb0ebafc9883782ebd", size = 113723116, upload-time = "2026-01-21T16:25:21.916Z" }, + { url = "https://files.pythonhosted.org/packages/61/d8/15b9d9d3a6b0c01b883787bd056acbe5cc321090d4b216d3ea89a8fcfdf3/torch-2.10.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:b7bd80f3477b830dd166c707c5b0b82a898e7b16f59a7d9d42778dd058272e8b", size = 79423461, upload-time = "2026-01-21T16:24:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/cc/af/758e242e9102e9988969b5e621d41f36b8f258bb4a099109b7a4b4b50ea4/torch-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5fd4117d89ffd47e3dcc71e71a22efac24828ad781c7e46aaaf56bf7f2796acf", size = 145996088, upload-time = "2026-01-21T16:24:44.171Z" }, + { url = "https://files.pythonhosted.org/packages/23/8e/3c74db5e53bff7ed9e34c8123e6a8bfef718b2450c35eefab85bb4a7e270/torch-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:787124e7db3b379d4f1ed54dd12ae7c741c16a4d29b49c0226a89bea50923ffb", size = 915711952, upload-time = "2026-01-21T16:23:53.503Z" }, + { url = "https://files.pythonhosted.org/packages/6e/01/624c4324ca01f66ae4c7cd1b74eb16fb52596dce66dbe51eff95ef9e7a4c/torch-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c66c61f44c5f903046cc696d088e21062644cbe541c7f1c4eaae88b2ad23547", size = 113757972, upload-time = "2026-01-21T16:24:39.516Z" }, + { url = "https://files.pythonhosted.org/packages/c9/5c/dee910b87c4d5c0fcb41b50839ae04df87c1cfc663cf1b5fca7ea565eeaa/torch-2.10.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:6d3707a61863d1c4d6ebba7be4ca320f42b869ee657e9b2c21c736bf17000294", size = 79498198, upload-time = "2026-01-21T16:24:34.704Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6f/f2e91e34e3fcba2e3fc8d8f74e7d6c22e74e480bbd1db7bc8900fdf3e95c/torch-2.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5c4d217b14741e40776dd7074d9006fd28b8a97ef5654db959d8635b2fe5f29b", size = 146004247, upload-time = "2026-01-21T16:24:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/98/fb/5160261aeb5e1ee12ee95fe599d0541f7c976c3701d607d8fc29e623229f/torch-2.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6b71486353fce0f9714ca0c9ef1c850a2ae766b409808acd58e9678a3edb7738", size = 915716445, upload-time = "2026-01-21T16:22:45.353Z" }, + { url = "https://files.pythonhosted.org/packages/6a/16/502fb1b41e6d868e8deb5b0e3ae926bbb36dab8ceb0d1b769b266ad7b0c3/torch-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2ee399c644dc92ef7bc0d4f7e74b5360c37cdbe7c5ba11318dda49ffac2bc57", size = 113757050, upload-time = "2026-01-21T16:24:19.204Z" }, + { url = "https://files.pythonhosted.org/packages/1a/0b/39929b148f4824bc3ad6f9f72a29d4ad865bcf7ebfc2fa67584773e083d2/torch-2.10.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:3202429f58309b9fa96a614885eace4b7995729f44beb54d3e4a47773649d382", size = 79851305, upload-time = "2026-01-21T16:24:09.209Z" }, + { url = "https://files.pythonhosted.org/packages/d8/14/21fbce63bc452381ba5f74a2c0a959fdf5ad5803ccc0c654e752e0dbe91a/torch-2.10.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:aae1b29cd68e50a9397f5ee897b9c24742e9e306f88a807a27d617f07adb3bd8", size = 146005472, upload-time = "2026-01-21T16:22:29.022Z" }, + { url = "https://files.pythonhosted.org/packages/54/fd/b207d1c525cb570ef47f3e9f836b154685011fce11a2f444ba8a4084d042/torch-2.10.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6021db85958db2f07ec94e1bc77212721ba4920c12a18dc552d2ae36a3eb163f", size = 915612644, upload-time = "2026-01-21T16:21:47.019Z" }, + { url = "https://files.pythonhosted.org/packages/36/53/0197f868c75f1050b199fe58f9bf3bf3aecac9b4e85cc9c964383d745403/torch-2.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ff43db38af76fda183156153983c9a096fc4c78d0cd1e07b14a2314c7f01c2c8", size = 113997015, upload-time = "2026-01-21T16:23:00.767Z" }, + { url = "https://files.pythonhosted.org/packages/0e/13/e76b4d9c160e89fff48bf16b449ea324bda84745d2ab30294c37c2434c0d/torch-2.10.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:cdf2a523d699b70d613243211ecaac14fe9c5df8a0b0a9c02add60fb2a413e0f", size = 79498248, upload-time = "2026-01-21T16:23:09.315Z" }, + { url = "https://files.pythonhosted.org/packages/4f/93/716b5ac0155f1be70ed81bacc21269c3ece8dba0c249b9994094110bfc51/torch-2.10.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:bf0d9ff448b0218e0433aeb198805192346c4fd659c852370d5cc245f602a06a", size = 79464992, upload-time = "2026-01-21T16:23:05.162Z" }, + { url = "https://files.pythonhosted.org/packages/69/2b/51e663ff190c9d16d4a8271203b71bc73a16aa7619b9f271a69b9d4a936b/torch-2.10.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:233aed0659a2503b831d8a67e9da66a62c996204c0bba4f4c442ccc0c68a3f60", size = 146018567, upload-time = "2026-01-21T16:22:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cd/4b95ef7f293b927c283db0b136c42be91c8ec6845c44de0238c8c23bdc80/torch-2.10.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:682497e16bdfa6efeec8cde66531bc8d1fbbbb4d8788ec6173c089ed3cc2bfe5", size = 915721646, upload-time = "2026-01-21T16:21:16.983Z" }, + { url = "https://files.pythonhosted.org/packages/56/97/078a007208f8056d88ae43198833469e61a0a355abc0b070edd2c085eb9a/torch-2.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:6528f13d2a8593a1a412ea07a99812495bec07e9224c28b2a25c0a30c7da025c", size = 113752373, upload-time = "2026-01-21T16:22:13.471Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/71994e7d0d5238393df9732fdab607e37e2b56d26a746cb59fdb415f8966/torch-2.10.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f5ab4ba32383061be0fb74bda772d470140a12c1c3b58a0cfbf3dae94d164c28", size = 79850324, upload-time = "2026-01-21T16:22:09.494Z" }, + { url = "https://files.pythonhosted.org/packages/e2/65/1a05346b418ea8ccd10360eef4b3e0ce688fba544e76edec26913a8d0ee0/torch-2.10.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:716b01a176c2a5659c98f6b01bf868244abdd896526f1c692712ab36dbaf9b63", size = 146006482, upload-time = "2026-01-21T16:22:18.42Z" }, + { url = "https://files.pythonhosted.org/packages/1d/b9/5f6f9d9e859fc3235f60578fa64f52c9c6e9b4327f0fe0defb6de5c0de31/torch-2.10.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d8f5912ba938233f86361e891789595ff35ca4b4e2ac8fe3670895e5976731d6", size = 915613050, upload-time = "2026-01-21T16:20:49.035Z" }, + { url = "https://files.pythonhosted.org/packages/66/4d/35352043ee0eaffdeff154fad67cd4a31dbed7ff8e3be1cc4549717d6d51/torch-2.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:71283a373f0ee2c89e0f0d5f446039bdabe8dbc3c9ccf35f0f784908b0acd185", size = 113995816, upload-time = "2026-01-21T16:22:05.312Z" }, +] + +[[package]] +name = "torch-c-dlpack-ext" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/de/921b6491efce5c389a5ef9bbed3d2d6660005840dae488124173180859ab/torch_c_dlpack_ext-0.1.5.tar.gz", hash = "sha256:d06f0357d575d22a168cc77acb9020fc4bae30968ceb6718a055dcbe92bacabe", size = 12913, upload-time = "2026-01-12T11:25:08.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/49/67a66932ab2fcdda3c5a4dcf606e713d86883a4a9a99a3bb832815b52b8e/torch_c_dlpack_ext-0.1.5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:e0f6c197d5293884898b9ebf13d07501de39cb94799b374ed43f91731087d557", size = 7056755, upload-time = "2026-01-12T11:24:31.817Z" }, + { url = "https://files.pythonhosted.org/packages/ae/28/d2d6bf90e01a1f4da3277c9a56d9ecac648b6d6adaa8e20c17f802deb7fb/torch_c_dlpack_ext-0.1.5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba3d88f0f7d5e1d9c3d4a3179037fc8e261c3b77ac1fad23edc0d3a9214ef193", size = 432066, upload-time = "2026-01-12T11:24:33.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e9/a1f9584a3af4ac6ae5ad5cf86927d8c3a9b6bb50d54e54d19313411216a0/torch_c_dlpack_ext-0.1.5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7468df84ec152d930fbc3acf460c44a60b3462b95af3d3a676d133629c7e176", size = 879488, upload-time = "2026-01-12T11:24:34.837Z" }, + { url = "https://files.pythonhosted.org/packages/6c/08/478cfcb5814e29f9b720111bdef315fc2fbc8b276e4b1183c8b9c9414a4f/torch_c_dlpack_ext-0.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:78dd4904bd26170a2dd7c0eab56367756ee0a15672ce9b84146169e68f0c6ddc", size = 1461437, upload-time = "2026-01-12T11:24:36.385Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/c12a9bb3a5ddc0962c00467891bf1ffdda39a4d4780bf0fbbf54523ff34e/torch_c_dlpack_ext-0.1.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:56bd25a2af19280bf8a06aa62cff5510106f43235b9327d8561b3e9a659c4d84", size = 5076782, upload-time = "2026-01-12T11:24:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/20/e1/64e1e579d107064785549e70758e38a42376ab7e73d86897ed4beab10e74/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fba674110e1fab0b176bb5a28223e157db65c90767d4ba74abdbee9f537b0e9d", size = 440949, upload-time = "2026-01-12T11:24:39.716Z" }, + { url = "https://files.pythonhosted.org/packages/64/5c/3e1382a620824f92920ab3fae132d8fb4e85898284c99e0c6a7764e452ce/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3448c4f0d64104d0b2e58080a7efa72304a04960c18f338024b80b13cd3eca26", size = 897768, upload-time = "2026-01-12T11:24:41.209Z" }, + { url = "https://files.pythonhosted.org/packages/54/4f/76ea1006b9038b496d01e916c91efd17cb782abde2491a261cf203f57e30/torch_c_dlpack_ext-0.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:74676474e0afa9a4216c4755ea7cf05e8158be1d168f6bda669ba91097c263f2", size = 1479088, upload-time = "2026-01-12T11:24:42.436Z" }, + { url = "https://files.pythonhosted.org/packages/b1/67/10d236698525d7b7db4d74ec0a4b01f5b2db33968995fdd9ac6b4635e327/torch_c_dlpack_ext-0.1.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c0f2bd51fcd99c0e5b50314e1985f2728c4941bfa821f065e6c30951d1f995ca", size = 5291237, upload-time = "2026-01-12T11:24:44.011Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8d760997307a5c3be4384424667bf31aae0a42060838c532c7d846516175/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3562ee411258676f9c38b8ad39306d1c8d027b6a86f6a87c920d2d009a9d1510", size = 443069, upload-time = "2026-01-12T11:24:45.451Z" }, + { url = "https://files.pythonhosted.org/packages/e2/79/a914539b4785f3e44f891aa012a886edb8bc10fe081c440981c57543ce21/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6f9da4bb9af70e27facc777458be62e10dbbbddda7672d16138db0553c5a524", size = 897846, upload-time = "2026-01-12T11:24:48.168Z" }, + { url = "https://files.pythonhosted.org/packages/3a/e6/7d7a97a3953208d6d6ce749180c34d1dab48464ded9a76cecabe9d021ce6/torch_c_dlpack_ext-0.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:670fbbab70123cc228bed41693a3720757af57a0ad22669063c9db25321e8f55", size = 1482855, upload-time = "2026-01-12T11:24:49.581Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/65346a201d921b616731311fc9941f15137672b444cebdad702cb52ccee0/torch_c_dlpack_ext-0.1.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:74acea2ed395cadda63342845b9e9ee7cd4537846223dacfb4431b4610109265", size = 1993243, upload-time = "2026-01-12T11:24:51.079Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ec/faf10be09a5812b1c5ec9922b53fb5def5fc4080b81a653b9347bb169ebb/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f1e99d13c64e22dac0a34a1560e9e5a398a49a9fa81df83053e04fde6ec5bd", size = 443798, upload-time = "2026-01-12T11:24:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/2d/68/f434b48700f3e04f33882f54d8d3910327b935f55e14ec49da7d607bf470/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:debe62e5ef93e631065d6b9f6e60d3d39bae6b89fa1b25d9523f40b3efbf8aba", size = 755004, upload-time = "2026-01-12T11:24:54.004Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/cc64e563f05ea99bd79bdb43f71f0f46452d3acd734da4843ede5fc73a35/torch_c_dlpack_ext-0.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:30e3eab616dbc81dfdb7492aca557be551a9163ba9b585f97394a42b336b113a", size = 999126, upload-time = "2026-01-12T11:24:55.44Z" }, + { url = "https://files.pythonhosted.org/packages/96/5e/449324ca8e81573e650b6851fc31c1038f750d1de85d0b185d788e1c7a3a/torch_c_dlpack_ext-0.1.5-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:cac94a4905d391889e679a8da31e46dc325af5d55d13b7c70c0ce3d71d1ced6d", size = 1982154, upload-time = "2026-01-12T11:24:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/20/62/11c05b99f69aa5152bca0313e0dfa6d125a020cf890dc888ef009aa7891c/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a58fdf45fb0bda7bc459632cec891570f31c11636d5851c825cf308ec8b73c2", size = 163825, upload-time = "2026-01-12T11:24:59.474Z" }, + { url = "https://files.pythonhosted.org/packages/15/b5/be613cd8e71c9982bd07af530f86c5a7f30df7831d14cec5414857af7149/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b985a324c68241cf83a9474b28015524b66775b12a91930dd4c0760aa628d01", size = 171740, upload-time = "2026-01-12T11:25:00.776Z" }, + { url = "https://files.pythonhosted.org/packages/5c/11/52e291f1659e2ec70a09f5ca4ad27e015eb4f0a1371ae68d23a9fbd1c704/torch_c_dlpack_ext-0.1.5-cp314-cp314-win_amd64.whl", hash = "sha256:d794e19fa3f330ab7a29987c07e031fc08e4953aec516d35701d0827863e356b", size = 277086, upload-time = "2026-01-12T11:25:01.901Z" }, ] [[package]] name = "torchaudio" -version = "2.9.0" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "torch" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/78/aa/7fce684dc0e21f8ea3ecf4a9f37253f8fa0b51aa0973202b58f33b9dc031/torchaudio-2.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:214d2e8bec2b204ac3f552f3dceae51550e06a91c5863d5dc341d81691ef655e", size = 806922, upload-time = "2025-10-15T15:51:53.069Z" }, - { url = "https://files.pythonhosted.org/packages/0b/c2/212181b1df762487462b3a092f6a9ae6ba87df02df71bb2121c100b13b8d/torchaudio-2.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1e84e45f74bf5b208b5ce59b36f26ec1e5f63596542c3ebee6edeadf85e73563", size = 473802, upload-time = "2025-10-15T15:51:55.626Z" }, - { url = "https://files.pythonhosted.org/packages/39/27/75184741da9aa1e94ec136319781e1275a560d1c311a293cc22aba747863/torchaudio-2.9.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:905f2c916e392b6dde375c002abe98f6fc64705fdf1192c90a6df2de235305f3", size = 2055464, upload-time = "2025-10-15T15:51:57.996Z" }, - { url = "https://files.pythonhosted.org/packages/43/af/f12349d7cb325b9b36452192953eb8c4ca9a6c28c8335c2d2f5e576be7f3/torchaudio-2.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ed556da9de16f69ccbe804df510ae8fefdf995cbdc2fcf26ea7532d25463326", size = 663878, upload-time = "2025-10-15T15:52:07.274Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a2/7696b9579ad0c40b78ce2774fb24875c43257f3d0d24540e1cfa946c13b4/torchaudio-2.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:662eb49ab25e1a2b7367bb072a8ad05c8a4b650ebbe7090a5af1a1eb1d40767c", size = 808368, upload-time = "2025-10-15T15:51:56.56Z" }, - { url = "https://files.pythonhosted.org/packages/55/1a/48d528cae6050b9a5f07c1c942b547143237e9f080f4a2ccb80ba88486df/torchaudio-2.9.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:914f1408142bdeda1ca9f834dd04967625fccc75893bd1504a018a13a04f1b66", size = 475720, upload-time = "2025-10-15T15:51:59.111Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/7aba77bc89d06df993c1519b66b7e0b09661d297d0eb8c044ab2c5af665f/torchaudio-2.9.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:86b15ce1d74814d5ca14bfac0d3b33f325c8cac4a6f09dcc5b82748133a96792", size = 2058688, upload-time = "2025-10-15T15:52:01.885Z" }, - { url = "https://files.pythonhosted.org/packages/96/64/93944c24d7ec76dff3315f9aaf382e86d09fa2c865942c3d6b48666e5b1d/torchaudio-2.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:840487d748128ded45bd65b213b55db701ad047544e77ae3c57ea48f55623a77", size = 664692, upload-time = "2025-10-15T15:52:02.908Z" }, - { url = "https://files.pythonhosted.org/packages/b7/63/3c0ede3aa3d19a8a6698ddd107fa88660549360b51bf8ce2717cd498d800/torchaudio-2.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab4cbcccfd873b0fb41fcb39c9869e59ef84bb95b093f6f58e2d05172a7500d2", size = 809116, upload-time = "2025-10-15T15:52:00.911Z" }, - { url = "https://files.pythonhosted.org/packages/be/d5/25e58745defe9d05893d3cba5c0e1a76aeaac503ac5ec4d9f83c871df71c/torchaudio-2.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7f93388b6e536c14d6015b6f75277a8b45efc532f61b35adc1ed06c98a86003e", size = 476020, upload-time = "2025-10-15T15:51:59.967Z" }, - { url = "https://files.pythonhosted.org/packages/f0/9c/58b8b49dfba2ae85e41ca86b0c52de45bbbea01987490de219c99c523a58/torchaudio-2.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:508318a2130b40ad51378f90caf8727a4bd3ac2b296f2b90c900b44e6068a940", size = 2059901, upload-time = "2025-10-15T15:51:54.634Z" }, - { url = "https://files.pythonhosted.org/packages/d7/eb/58b05f75d12f69ccc460893a20c999da082e063082120ed06e05cca3a053/torchaudio-2.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:82117e3a605f2959dc09b4cd8a11178d6e92727d5f85e5d4f9fe47502f84ee96", size = 665350, upload-time = "2025-10-15T15:52:08.384Z" }, - { url = "https://files.pythonhosted.org/packages/6c/66/974371d4e4042d186931b72365817d9d3a509f2bc570888a48612448c060/torchaudio-2.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5549c25db4c2da306e179e9aa99980e7f5b1826a8d2d7de08125f3943a5620b2", size = 809149, upload-time = "2025-10-15T15:52:16.133Z" }, - { url = "https://files.pythonhosted.org/packages/09/61/8f7b875a2d879666f2f121e458817703e5499988a86105d2a25afecb9987/torchaudio-2.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1eb0d1dac8cefbc4a54afb21aac72a1c25a91f73e9c3bd85f6684930a4a1be5d", size = 475699, upload-time = "2025-10-15T15:52:06.349Z" }, - { url = "https://files.pythonhosted.org/packages/26/db/10ba200f90b76f7b859f46b5ba30cdded69f71bcb0fe3c59bb215532cd2b/torchaudio-2.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:266d304dd4ed738a10148b020e3d066e81272ee851f6f92193fe549df96af868", size = 2060349, upload-time = "2025-10-15T15:52:09.329Z" }, - { url = "https://files.pythonhosted.org/packages/be/53/5f9adbea55e48f91532ee4f041283900939ee5cb6bc1395587214e67a629/torchaudio-2.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:7d3926129389d934aa048bd6c6f68fbf3ef26828ebbbbeac99794ea00e90dc1c", size = 665310, upload-time = "2025-10-15T15:52:05.101Z" }, - { url = "https://files.pythonhosted.org/packages/e3/41/88b989aab1e11134d858350196fcf3afd4c2a6821d74efb3c1b9ab23b8cf/torchaudio-2.9.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:967d664477fb91dffad82ef64ea3695801c0cc35304baec71be875b569440872", size = 813491, upload-time = "2025-10-15T15:52:10.346Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c1/8d0481fc921cb72d6cadbacd338fa71db0052e8fdb1bf33127c694bbf257/torchaudio-2.9.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:276871d6f5fed5268a87c5da303a13ca2e06b9d29a4c44663b960f0a2e2f46d7", size = 477749, upload-time = "2025-10-15T15:52:04.189Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d3/d085cd76413b9f3f792e61933235d982caf5cdbdf60f0e4fdae71879becc/torchaudio-2.9.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3d5657d929d6ca07b08cfa005988f2ea8caacf9af42f20bc7eff10f88812ce30", size = 2062165, upload-time = "2025-10-15T15:52:12.784Z" }, - { url = "https://files.pythonhosted.org/packages/f2/41/d9876f5b19b4b2f98a6131d1a98ee6d5d8f707c01311bbba7cc3bb02f4bf/torchaudio-2.9.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3fe9cac0c2ee713e07f8c88d09528d55e0fa74987b0122e27911dfb720f39054", size = 669260, upload-time = "2025-10-15T15:52:13.8Z" }, - { url = "https://files.pythonhosted.org/packages/97/ad/db50c49d73d1904152bbaaaa281e03a41ec519dd6a9df48cc69ea5cd48b9/torchaudio-2.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3fa41447a21103fcde930b4ad2bd2634565a0becff1a5425535b4f0116c0d5df", size = 810532, upload-time = "2025-10-15T15:52:17.197Z" }, - { url = "https://files.pythonhosted.org/packages/a8/00/aa8ed83a169a87af72d6cdc17e0350f418b3cba3bd7397b0cca873274789/torchaudio-2.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:69f46f21bd67e90ade33a7d0f0cf98270cd61b98f5f8249d3893be0a16b3e31f", size = 475864, upload-time = "2025-10-15T15:52:11.446Z" }, - { url = "https://files.pythonhosted.org/packages/4b/bb/7ca64ed0556afa08d3a7a47c887ee9b1c4f3eebd193baf47505b6fac479c/torchaudio-2.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:631b0f43564a25e27e615b217454c334f52162679f39ae10b9fa7562ed587dfc", size = 2060360, upload-time = "2025-10-15T15:52:14.992Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/4407b79ddedc9ea95d88fa54c3758df21f0117683fceba4bacd98ceaa772/torchaudio-2.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:ed6df9f14431e13498b984dc87df1aabb2156b9ce0ce7268ce4a61650197310a", size = 665048, upload-time = "2025-10-15T15:52:19.116Z" }, - { url = "https://files.pythonhosted.org/packages/7d/1a/d3cd6b67b5c68ff4211be923978d1d7c10ea2f44f826d4cd15b775f52c11/torchaudio-2.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:93358d8f2f24969ba3f368f4eec33295df830af54836c7fd3336740228f9af16", size = 813499, upload-time = "2025-10-15T15:52:20.412Z" }, - { url = "https://files.pythonhosted.org/packages/ab/65/a35a182519b40dcd2cedaf5fdcac6f724ae2451c534dfcece6ff5f85f983/torchaudio-2.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:742143d9d62769bc4b9a2977ca4f4720e0a5e922bdc5df585c155e0a1f545461", size = 477752, upload-time = "2025-10-15T15:52:18.14Z" }, - { url = "https://files.pythonhosted.org/packages/6f/1c/30272b71ae08817eaca00bb856ebef25dd44041329579903c1915b57f0c9/torchaudio-2.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0a234634e1142fb2652c49e935a98b4d9656fd0af9e4aa14b1b05a80c3cf8e78", size = 2062173, upload-time = "2025-10-15T15:52:22.724Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d6/d007f6bc55a16a86e64e9bba295b90485011cc6a113d8f56b503b4f34a7d/torchaudio-2.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:cbf5d6da8fd2ed545c78218b39fd6aacaa4dd5e265c5f85b248a2fac223f0bd6", size = 669272, upload-time = "2025-10-15T15:52:21.696Z" }, + { url = "https://files.pythonhosted.org/packages/04/59/88ab8ebff9d91f1f1365088b30f1b9ccce07c5eeac666038a5dee5e2f9b1/torchaudio-2.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cde383582a6240c1315443df5c5638863e96b03acf1cb44a298aff07a72d373", size = 734944, upload-time = "2026-01-21T16:28:49.535Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d6/41f25f9ae9b37c191bed4cd474e403626685d2be8f7d20d011e6601fede1/torchaudio-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:cfb2ad4b7847d81931989127d803487263c8284f21156e9000daec1ac16c0831", size = 390449, upload-time = "2026-01-21T16:28:48.585Z" }, + { url = "https://files.pythonhosted.org/packages/43/ac/a14425fddd1cf56bb052a3bfd38880258008f8c3cd17f37bba55b3a88ce7/torchaudio-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:316cdb15fb37290fca89894b095d97b4dc14a90c4c61148ae5c96bb334d962cd", size = 1891070, upload-time = "2026-01-21T16:28:47.323Z" }, + { url = "https://files.pythonhosted.org/packages/6e/03/d1898db1bf7ecd47ca9b4e1b70927597d236cf721e3736d953d555901832/torchaudio-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:013079d1ba2a652184703e671b8339cbc7991f17e4ed927071fe7635f908a4a1", size = 474045, upload-time = "2026-01-21T16:28:46.191Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e7/401fe1d024bf9352371d854be6f339ad9928669e6bc8a5ba08e9dbce81cf/torchaudio-2.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bcab0e39eb18da84cba1a0c87f600abb6ce97c882200cb46e841caea106f037f", size = 736373, upload-time = "2026-01-21T16:28:41.589Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b7/c66dc34a27441d78997e20d0ffe2f5ad73db9f7b1267511be255bb94ac9b/torchaudio-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:87c841a21e82703ebd4a29170c4e60c25a2b47312dc212930087ad58965ac0c8", size = 391843, upload-time = "2026-01-21T16:28:43.093Z" }, + { url = "https://files.pythonhosted.org/packages/13/ae/a2a34a64947c4fa4a61b4c86d8f36fbcb4ebfec30fdde140267db260f96c/torchaudio-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b2c77fb9114dd463dc805560bf55a1ac2a52e219794cc32b7b32cf2aeffd2826", size = 1894140, upload-time = "2026-01-21T16:28:35.892Z" }, + { url = "https://files.pythonhosted.org/packages/69/26/cd2aec609b4f8918e4e85e5c6a3f569bc7b5f72a7ecba3f784077102749c/torchaudio-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:4c6e9609046143b30a30183893d23ff1ce5de603dbe914b3cce5cc29f5aa5a9c", size = 474792, upload-time = "2026-01-21T16:28:45.254Z" }, + { url = "https://files.pythonhosted.org/packages/0f/36/28a6f3e857616cf7576bdbf8170e483b8c5d0a1f8d349ecb2b75921236aa/torchaudio-2.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d0fbdbfd2f621c51d28571050d6d0c7287791034e5c7303b31480af1258f33f", size = 737144, upload-time = "2026-01-21T16:28:44.189Z" }, + { url = "https://files.pythonhosted.org/packages/ea/3f/df620439a76ece170472d41438d11a1545d5db5dc9f1eaeab8c6e055a328/torchaudio-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:42b148a0921a3721abd1f6ae098b1ec9f89703e555c4f7a0d44da87b8decbcb9", size = 391973, upload-time = "2026-01-21T16:28:39.732Z" }, + { url = "https://files.pythonhosted.org/packages/98/25/e55a30d7138f8fe56ed006df25b0a3c27681f0ec7bc9989e1778e6d559c3/torchaudio-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0e77b2956448d63790a99beed0b74ac8b8cd3a94dcdd9ad01974411078f46278", size = 1895234, upload-time = "2026-01-21T16:28:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/be/a0/da53c7d20fac15f66f8838653b91162de1bf21fb40fee88cf839e4ef5174/torchaudio-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f76a01ecebf1869e1f2c50a261f1cf07e5fccb24402b4e9bbb82d6725b9c7dd", size = 475470, upload-time = "2026-01-21T16:28:40.615Z" }, + { url = "https://files.pythonhosted.org/packages/b6/02/341e7bd588355f82c5180103cb2f8070a72ab1be920ab27553a1135d4aa6/torchaudio-2.10.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8fd38d28ee150c584d3ee3b05f39e021f0ad8a8ec8fec1f26dfe150c9db9b2f5", size = 737164, upload-time = "2026-01-21T16:28:38.354Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/831c2595c81b17141180ca11ab3c0836cc544ef13e15aa0e7b2cb619e582/torchaudio-2.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5bc39ff3ea341097ce1ab023dd88c9dd8ca5f96ebf48821e7d23766137bb55d7", size = 392757, upload-time = "2026-01-21T16:28:33.631Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d8/405c80c57dc68ca5855bddfaae57c3d84ea7397bf1eb2aa5d59c9fa1d3a9/torchaudio-2.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3057c4286db5673d266124a2a10ca54e19f516772e9057f44573a7da5b85e328", size = 1897099, upload-time = "2026-01-21T16:28:24.793Z" }, + { url = "https://files.pythonhosted.org/packages/73/cf/0e48d67788c935e3b3d00e6f55a930a54a67f432e04c33ef80a38cb764fd/torchaudio-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:99e74d1901742bc10961d807fe75c0dd9496f4a4a4ff4bb317c5de4a0b6f24e6", size = 475476, upload-time = "2026-01-21T16:28:28.249Z" }, + { url = "https://files.pythonhosted.org/packages/48/29/30bcce0f17a8279b051b09250993691a828f89a03278306b23571c18df04/torchaudio-2.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6cfe98ef0ea9bee6d6297493ce67ce0c54a38d80caf6535a3ae48900fd5f3769", size = 742449, upload-time = "2026-01-21T16:28:29.556Z" }, + { url = "https://files.pythonhosted.org/packages/43/8c/653e7f67855424bf3b7cbb48335f8316f7fb02bb01a6cab38f6bf9555676/torchaudio-2.10.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:b41b254d958632dc00dc7768431cadda516c91641d798775cbb19bcd4f0d2be4", size = 393430, upload-time = "2026-01-21T16:28:34.855Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1f/f91fcb9dd47a19b720fb48042a2f6f023651948e73726e98fff60d5ed5c7/torchaudio-2.10.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:da1081d1018a1e95f5a13947402aeb037cf5ac8861219a6164df004898a96bb1", size = 1897271, upload-time = "2026-01-21T16:28:23.519Z" }, + { url = "https://files.pythonhosted.org/packages/57/27/270c26890f43838e8faa5d3e52f079bd9d9d09f9a535a11cf6b94e20ed21/torchaudio-2.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f1afa53146a5655258d3a86e689c6879dfe78581d9bee9ef611ace98722f86bb", size = 478966, upload-time = "2026-01-21T16:28:32.491Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5c/0e54b162bd0d1ec2f87b545553af839f906b940888d0122cdef04b965385/torchaudio-2.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1f2897fbf776d55afcb5f6d9b7bdfaea850ca7a129c8f5e4b3a4b025c431130d", size = 739544, upload-time = "2026-01-21T16:28:26.947Z" }, + { url = "https://files.pythonhosted.org/packages/57/a1/ef5571406858f4ea89c18d6ad844d21cb9858708149e6bbd9a789ee30ea5/torchaudio-2.10.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b2d5e11a2bec08f02a4f5fb7d1902ff82d48c533a27ceedc21e6ade650cf65b3", size = 393061, upload-time = "2026-01-21T16:28:25.802Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0f/a0cf0ebc6f71b1868ea056dd4cd4f1a2244b8da8bc38372a1adc984a7c1f/torchaudio-2.10.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:77f6cf11a3b61af1b0967cd642368ecd30a86d70f622b22410ae6cb42d980b72", size = 1897137, upload-time = "2026-01-21T16:28:15.366Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/98e6710a4601e190bc923c3683629c29d41fb18a818a9328515541f023ed/torchaudio-2.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:4711c2a86a005685ca3b5da135b2f370d81ac354e3dcb142ef45fe2c78b9c9c4", size = 475154, upload-time = "2026-01-21T16:28:22.438Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9b/cd02f8add38bd98761548b0821a5e54c564117a9bbeafaf95f665ab0fd72/torchaudio-2.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13bdc1bde0c88e999699d1503304a56fc9dea6401b76bc08a5f268368129d46c", size = 742453, upload-time = "2026-01-21T16:28:20.989Z" }, + { url = "https://files.pythonhosted.org/packages/53/8a/946aa07393845b918d318b5e34b3bd0359fd27fc9fac10a85fae2bb86382/torchaudio-2.10.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ed912de8ec1b400e17a5172badcfcddc601a9cd4e02d200f3a9504fc8e54961c", size = 393434, upload-time = "2026-01-21T16:28:18.668Z" }, + { url = "https://files.pythonhosted.org/packages/e1/68/e37e8fbbae986afa80f8851e08fc017eb8ae5f7b398ee28ed92303da163e/torchaudio-2.10.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:f7aa33a8198e87949896e16ea245ea731906445becdf10130e8823c68494a94a", size = 1897289, upload-time = "2026-01-21T16:28:17.059Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/0e1f464463b85bc677036faffdfd23493aa17e8c3fc3a649abca8c019701/torchaudio-2.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e49f6a18a8552620c4394f8529b7551eda9312d46dfdd3500bd2be459c86aea4", size = 478968, upload-time = "2026-01-21T16:28:19.542Z" }, ] [[package]] name = "torchvision" -version = "0.24.0" +version = "0.25.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -6199,34 +6448,34 @@ dependencies = [ { name = "torch" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/63/5b/1404eeab00819df71a30e916c2081654366741f7838fcc4fff86b7bd9e7e/torchvision-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e8d5e667deff87bd66d26df6d225f46224bb0782d4f3f8f5d2f3068b5fd4492", size = 1891723, upload-time = "2025-10-15T15:51:08.5Z" }, - { url = "https://files.pythonhosted.org/packages/88/e3/1b003ecd52bd721f8304aeb66691edfbc2002747ec83d36188ad6abab506/torchvision-0.24.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a110a51c75e89807a8382b0d8034f5e180fb9319570be3389ffd3d4ac4fd57a9", size = 2418988, upload-time = "2025-10-15T15:51:25.195Z" }, - { url = "https://files.pythonhosted.org/packages/56/2e/3c19a35e62da0f606baf8f6e2ceeab1eb66aaa2f84c6528538b06b416d54/torchvision-0.24.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:81d5b12a6df1bb2cc8bdbad837b637d6ea446f2866e6d94f1b5d478856331be3", size = 8046769, upload-time = "2025-10-15T15:51:15.221Z" }, - { url = "https://files.pythonhosted.org/packages/e0/1d/e7ab614a1ace820a2366eab1532679fbe81bd9501ffd6a1b7be14936366d/torchvision-0.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:0839dbb305d34671f5a64f558782095134b04bbeff8b90f11eb80515d7d50092", size = 3686529, upload-time = "2025-10-15T15:51:20.982Z" }, - { url = "https://files.pythonhosted.org/packages/a3/17/54ed2ec6944ea972b461a86424c8c7f98835982c90cbc45bf59bd962863a/torchvision-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f771cf918351ad509a28488be475f3e9cc71a750d6b1467842bfb64863a5e986", size = 1891719, upload-time = "2025-10-15T15:51:10.384Z" }, - { url = "https://files.pythonhosted.org/packages/f8/07/0cd6776eee784742ad3cb2bfd3295383d84cb2f9e87386119333d1587f0f/torchvision-0.24.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbd63bf4ebff84c48c50123eba90526cc9f794fe45bc9f5dd07cec19e8c62bce", size = 2420513, upload-time = "2025-10-15T15:51:18.087Z" }, - { url = "https://files.pythonhosted.org/packages/1a/f4/6026c08011ddcefcbc14161c5aa9dce55c35c6b045e04ef0952e88bf4594/torchvision-0.24.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:78fe414b3bb6dbf7e6f6da6f733ba96881f6b29a9b997228de7c5f603e5ed940", size = 8048018, upload-time = "2025-10-15T15:51:13.579Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b4/362b4e67ed87cee0fb4f8f0363a852eaeef527968bf62c07ed56f764d729/torchvision-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:629584b94e52f32a6278f2a35d85eeaae95fcc38730fcb765064f26c3c96df5d", size = 4027686, upload-time = "2025-10-15T15:51:19.189Z" }, - { url = "https://files.pythonhosted.org/packages/47/ef/81e4e69e02e2c4650b30e8c11c8974f946682a30e0ab7e9803a831beff76/torchvision-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c61d40bcd2e2451e932902a702ad495ba1ec6f279e90b1e15cef2bb55dc911e2", size = 1891726, upload-time = "2025-10-15T15:51:16.977Z" }, - { url = "https://files.pythonhosted.org/packages/00/7b/e3809b3302caea9a12c13f3adebe4fef127188438e719fd6c8dc93db1da6/torchvision-0.24.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b0531d1483fc322d7da0d83be52f0df860a75114ab87dbeeb9de765feaeda843", size = 2419495, upload-time = "2025-10-15T15:51:11.885Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e6/7324ead6793075a8c75c56abeed1236d1750de16a5613cfe2ddad164a92a/torchvision-0.24.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:26b9dd9c083f8e5f7ac827de6d5b88c615d9c582dc87666770fbdf16887e4c25", size = 8050480, upload-time = "2025-10-15T15:51:24.012Z" }, - { url = "https://files.pythonhosted.org/packages/3e/ad/3c56fcd2a0d6e8afa80e115b5ade4302232ec99655220a51d05709819523/torchvision-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:060b7c50ed4b3fb0316b08e2e31bfd874ec2f63ef5ae02f81e54341ca4e88703", size = 4292225, upload-time = "2025-10-15T15:51:27.699Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b5/b2008e4b77a8d6aada828dd0f6a438d8f94befa23fdd2d62fa0ac6e60113/torchvision-0.24.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84d79cfc6457310107ce4d712de7a3d388b24484bc9aeded4a76d8f8e3a2813d", size = 1891722, upload-time = "2025-10-15T15:51:28.854Z" }, - { url = "https://files.pythonhosted.org/packages/8f/02/e2f6b0ff93ca4db5751ac9c5be43f13d5e53d9e9412324f464dca1775027/torchvision-0.24.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fec12a269cf80f6b0b71471c8d498cd3bdd9d8e892c425bf39fecb604852c3b0", size = 2371478, upload-time = "2025-10-15T15:51:37.842Z" }, - { url = "https://files.pythonhosted.org/packages/77/85/42e5fc4f716ec7b73cf1f32eeb5c77961be4d4054b26cd6a5ff97f20c966/torchvision-0.24.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:7323a9be5e3da695605753f501cdc87824888c5655d27735cdeaa9986b45884c", size = 8050200, upload-time = "2025-10-15T15:51:46.276Z" }, - { url = "https://files.pythonhosted.org/packages/93/c2/48cb0b6b26276d2120b1e0dbc877579a748eae02b4091a7522ce54f6d5e1/torchvision-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:08cad8b204196e945f0b2d73adee952d433db1c03645851d52b22a45f1015b13", size = 4309939, upload-time = "2025-10-15T15:51:39.002Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d7/3dd10830b047eeb46ae6b465474258d7b4fbb7d8872dca69bd42449f5c82/torchvision-0.24.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6ab956a6e588623353e0f20d4b03eb1656cb4a3c75ca4dd8b4e32e01bc43271a", size = 2028355, upload-time = "2025-10-15T15:51:22.384Z" }, - { url = "https://files.pythonhosted.org/packages/f7/cf/2d7e43409089ce7070f5336161f9216d58653ee1cb26bcb5d6c84cc2de36/torchvision-0.24.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:b1b3db80609c32a088554e8e94b4fc31f1033fe5bb4ac0673ec49c3eb03fb4da", size = 2374466, upload-time = "2025-10-15T15:51:35.382Z" }, - { url = "https://files.pythonhosted.org/packages/e9/30/8f7c328fd7e0a9665da4b6b56b1c627665c18470bfe62f3729ad3eda9aec/torchvision-0.24.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:e6635f100d455c80b43f297df4b8585a76c6a2e114802f6567ddd28d7b5479b0", size = 8217068, upload-time = "2025-10-15T15:51:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/55/a2/b6f9e40e2904574c80b3bb872c66af20bbd642053e7c8e1b9e99ab396535/torchvision-0.24.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4ce158bbdc3a9086034bced0b5212888bd5b251fee6d08a9eff151d30b4b228a", size = 4273912, upload-time = "2025-10-15T15:51:33.866Z" }, - { url = "https://files.pythonhosted.org/packages/1b/24/790a39645cc8c71bf442d54a76da9bda5caeb2a44c5f7e02498649cd99d4/torchvision-0.24.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4bdfc85a5ed706421555f32cdc5e3ddb6d40bf65ef03a274ce3c176393e2904b", size = 2028335, upload-time = "2025-10-15T15:51:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d7/69479a066ea773653e88eda99031e38681e9094046f87cb957af5036db0e/torchvision-0.24.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:73576a9c4a593223fbae85a64e8bbd77049abd1101893ecf3c5e981284fd58b4", size = 2371609, upload-time = "2025-10-15T15:51:29.859Z" }, - { url = "https://files.pythonhosted.org/packages/46/64/3c7fdb3771ec992b9445a1f7a969466b23ce2cdb14e09303b3db351a0655/torchvision-0.24.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:dd565b1b06666ff399d0801d4d1824fa570c0167a179ca700a5be232527b3c62", size = 8214918, upload-time = "2025-10-15T15:51:41.465Z" }, - { url = "https://files.pythonhosted.org/packages/58/51/abc416bc34d574ad479af738e413d9ebf93027ee92d0f4ae38f966b818f7/torchvision-0.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:eb45d12ac48d757738788fd3fb8e88e647d6b2ab2424134ca87556efc72d81b5", size = 4257776, upload-time = "2025-10-15T15:51:42.642Z" }, - { url = "https://files.pythonhosted.org/packages/08/f7/261d1353c611820541ecd43046b89da3f1ae998dc786e4288b890a009883/torchvision-0.24.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:68120e7e03c31900e499a10bb7fdd63cfd67f0054c9fa108e7e27f9cd372f315", size = 2028359, upload-time = "2025-10-15T15:51:32.119Z" }, - { url = "https://files.pythonhosted.org/packages/a2/fd/615d8a86db1578345de7fa1edaf476fbcf4f057bf7e4fd898306b620c487/torchvision-0.24.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:64e54494043eecf9f57a9881c6fdea49c62282782e737c002ae8b1639e6ea80e", size = 2374469, upload-time = "2025-10-15T15:51:40.19Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/bac11e8fdbf00d6c398246ff2781370aa72c99f2ac685c01ce79354c9a32/torchvision-0.24.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:75ef9546323b321a451239d886f0cb528f7e98bb294da47a3200effd4e572064", size = 8217060, upload-time = "2025-10-15T15:51:45.033Z" }, - { url = "https://files.pythonhosted.org/packages/47/6f/9fba8abc468c904570699eceeb51588f9622172b8fffa4ab11bcf15598c2/torchvision-0.24.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2efb617667950814fc8bb9437e5893861b3616e214285be33cbc364a3f42c599", size = 4358490, upload-time = "2025-10-15T15:51:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/50/ae/cbf727421eb73f1cf907fbe5788326a08f111b3f6b6ddca15426b53fec9a/torchvision-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a95c47abb817d4e90ea1a8e57bd0d728e3e6b533b3495ae77d84d883c4d11f56", size = 1874919, upload-time = "2026-01-21T16:27:47.617Z" }, + { url = "https://files.pythonhosted.org/packages/64/68/dc7a224f606d53ea09f9a85196a3921ec3a801b0b1d17e84c73392f0c029/torchvision-0.25.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:acc339aba4a858192998c2b91f635827e40d9c469d9cf1455bafdda6e4c28ea4", size = 2343220, upload-time = "2026-01-21T16:27:44.26Z" }, + { url = "https://files.pythonhosted.org/packages/f9/fa/8cce5ca7ffd4da95193232493703d20aa06303f37b119fd23a65df4f239a/torchvision-0.25.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0d9a3f925a081dd2ebb0b791249b687c2ef2c2717d027946654607494b9b64b6", size = 8068106, upload-time = "2026-01-21T16:27:37.805Z" }, + { url = "https://files.pythonhosted.org/packages/8b/b9/a53bcf8f78f2cd89215e9ded70041765d50ef13bf301f9884ec6041a9421/torchvision-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:b57430fbe9e9b697418a395041bb615124d9c007710a2712fda6e35fb310f264", size = 3697295, upload-time = "2026-01-21T16:27:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/3e/be/c704bceaf11c4f6b19d64337a34a877fcdfe3bd68160a8c9ae9bea4a35a3/torchvision-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db74a551946b75d19f9996c419a799ffdf6a223ecf17c656f90da011f1d75b20", size = 1874923, upload-time = "2026-01-21T16:27:46.574Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/f143cd71232430de1f547ceab840f68c55e127d72558b1061a71d0b193cd/torchvision-0.25.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f49964f96644dbac2506dffe1a0a7ec0f2bf8cf7a588c3319fed26e6329ffdf3", size = 2344808, upload-time = "2026-01-21T16:27:43.191Z" }, + { url = "https://files.pythonhosted.org/packages/43/ae/ad5d6165797de234c9658752acb4fce65b78a6a18d82efdf8367c940d8da/torchvision-0.25.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:153c0d2cbc34b7cf2da19d73450f24ba36d2b75ec9211b9962b5022fb9e4ecee", size = 8070752, upload-time = "2026-01-21T16:27:33.748Z" }, + { url = "https://files.pythonhosted.org/packages/23/19/55b28aecdc7f38df57b8eb55eb0b14a62b470ed8efeb22cdc74224df1d6a/torchvision-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:ea580ffd6094cc01914ad32f8c8118174f18974629af905cea08cb6d5d48c7b7", size = 4038722, upload-time = "2026-01-21T16:27:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/56/3a/6ea0d73f49a9bef38a1b3a92e8dd455cea58470985d25635beab93841748/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2abe430c90b1d5e552680037d68da4eb80a5852ebb1c811b2b89d299b10573b", size = 1874920, upload-time = "2026-01-21T16:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/51/f8/c0e1ef27c66e15406fece94930e7d6feee4cb6374bbc02d945a630d6426e/torchvision-0.25.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b75deafa2dfea3e2c2a525559b04783515e3463f6e830cb71de0fb7ea36fe233", size = 2344556, upload-time = "2026-01-21T16:27:40.125Z" }, + { url = "https://files.pythonhosted.org/packages/68/2f/f24b039169db474e8688f649377de082a965fbf85daf4e46c44412f1d15a/torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f25aa9e380865b11ea6e9d99d84df86b9cc959f1a007cd966fc6f1ab2ed0e248", size = 8072351, upload-time = "2026-01-21T16:27:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/ad/16/8f650c2e288977cf0f8f85184b90ee56ed170a4919347fc74ee99286ed6f/torchvision-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9c55ae8d673ab493325d1267cbd285bb94d56f99626c00ac4644de32a59ede3", size = 4303059, upload-time = "2026-01-21T16:27:11.08Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5b/1562a04a6a5a4cf8cf40016a0cdeda91ede75d6962cff7f809a85ae966a5/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:24e11199e4d84ba9c5ee7825ebdf1cd37ce8deec225117f10243cae984ced3ec", size = 1874918, upload-time = "2026-01-21T16:27:39.02Z" }, + { url = "https://files.pythonhosted.org/packages/36/b1/3d6c42f62c272ce34fcce609bb8939bdf873dab5f1b798fd4e880255f129/torchvision-0.25.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f271136d2d2c0b7a24c5671795c6e4fd8da4e0ea98aeb1041f62bc04c4370ef", size = 2309106, upload-time = "2026-01-21T16:27:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/c7/60/59bb9c8b67cce356daeed4cb96a717caa4f69c9822f72e223a0eae7a9bd9/torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:855c0dc6d37f462482da7531c6788518baedca1e0847f3df42a911713acdfe52", size = 8071522, upload-time = "2026-01-21T16:27:29.392Z" }, + { url = "https://files.pythonhosted.org/packages/32/a5/9a9b1de0720f884ea50dbf9acb22cbe5312e51d7b8c4ac6ba9b51efd9bba/torchvision-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:cef0196be31be421f6f462d1e9da1101be7332d91984caa6f8022e6c78a5877f", size = 4321911, upload-time = "2026-01-21T16:27:35.195Z" }, + { url = "https://files.pythonhosted.org/packages/52/99/dca81ed21ebaeff2b67cc9f815a20fdaa418b69f5f9ea4c6ed71721470db/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a8f8061284395ce31bcd460f2169013382ccf411148ceb2ee38e718e9860f5a7", size = 1896209, upload-time = "2026-01-21T16:27:32.159Z" }, + { url = "https://files.pythonhosted.org/packages/28/cc/2103149761fdb4eaed58a53e8437b2d716d48f05174fab1d9fcf1e2a2244/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:146d02c9876858420adf41f3189fe90e3d6a409cbfa65454c09f25fb33bf7266", size = 2310735, upload-time = "2026-01-21T16:27:22.327Z" }, + { url = "https://files.pythonhosted.org/packages/76/ad/f4c985ad52ddd3b22711c588501be1b330adaeaf6850317f66751711b78c/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c4d395cb2c4a2712f6eb93a34476cdf7aae74bb6ea2ea1917f858e96344b00aa", size = 8089557, upload-time = "2026-01-21T16:27:27.666Z" }, + { url = "https://files.pythonhosted.org/packages/63/cc/0ea68b5802e5e3c31f44b307e74947bad5a38cc655231d845534ed50ddb8/torchvision-0.25.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5e6b449e9fa7d642142c0e27c41e5a43b508d57ed8e79b7c0a0c28652da8678c", size = 4344260, upload-time = "2026-01-21T16:27:17.018Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1f/fa839532660e2602b7e704d65010787c5bb296258b44fa8b9c1cd6175e7d/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:620a236288d594dcec7634c754484542dc0a5c1b0e0b83a34bda5e91e9b7c3a1", size = 1896193, upload-time = "2026-01-21T16:27:24.785Z" }, + { url = "https://files.pythonhosted.org/packages/80/ed/d51889da7ceaf5ff7a0574fb28f9b6b223df19667265395891f81b364ab3/torchvision-0.25.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b5e7f50002a8145a98c5694a018e738c50e2972608310c7e88e1bd4c058f6ce", size = 2309331, upload-time = "2026-01-21T16:27:19.97Z" }, + { url = "https://files.pythonhosted.org/packages/90/a5/f93fcffaddd8f12f9e812256830ec9c9ca65abbf1bc369379f9c364d1ff4/torchvision-0.25.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:632db02300e83793812eee4f61ae6a2686dab10b4cfd628b620dc47747aa9d03", size = 8088713, upload-time = "2026-01-21T16:27:15.281Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/d0096eed5690d962853213f2ee00d91478dfcb586b62dbbb449fb8abc3a6/torchvision-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1abd5ed030c708f5dbf4812ad5f6fbe9384b63c40d6bd79f8df41a4a759a917", size = 4325058, upload-time = "2026-01-21T16:27:26.165Z" }, + { url = "https://files.pythonhosted.org/packages/97/36/96374a4c7ab50dea9787ce987815614ccfe988a42e10ac1a2e3e5b60319a/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad9a8a5877782944d99186e4502a614770fe906626d76e9cd32446a0ac3075f2", size = 1896207, upload-time = "2026-01-21T16:27:23.383Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e2/7abb10a867db79b226b41da419b63b69c0bd5b82438c4a4ed50e084c552f/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:40a122c3cf4d14b651f095e0f672b688dde78632783fc5cd3d4d5e4f6a828563", size = 2310741, upload-time = "2026-01-21T16:27:18.712Z" }, + { url = "https://files.pythonhosted.org/packages/08/e6/0927784e6ffc340b6676befde1c60260bd51641c9c574b9298d791a9cda4/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:846890161b825b38aa85fc37fb3ba5eea74e7091ff28bab378287111483b6443", size = 8089772, upload-time = "2026-01-21T16:27:14.048Z" }, + { url = "https://files.pythonhosted.org/packages/b6/37/e7ca4ec820d434c0f23f824eb29f0676a0c3e7a118f1514f5b949c3356da/torchvision-0.25.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f07f01d27375ad89d72aa2b3f2180f07da95dd9d2e4c758e015c0acb2da72977", size = 4425879, upload-time = "2026-01-21T16:27:12.579Z" }, ] [[package]] @@ -6312,21 +6561,40 @@ wheels = [ name = "triton" version = "3.5.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", +] + +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", +] wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/22/507b6f58a35e05e84381630b2dc2a3cee1a7a2a7eaf4cba857c638a18a24/triton-3.5.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f90de6a6566bb619b4c0adc9855729e1b1b5e26533fca1bf6206e96b6d277a3", size = 159827599, upload-time = "2025-10-15T19:15:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/0b/eb/09e31d107a5d00eb281aa7e6635ca463e9bca86515944e399480eadb71f8/triton-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5d3b3d480debf24eaa739623c9a42446b0b77f95593d30eb1f64cd2278cc1f0", size = 170333110, upload-time = "2025-10-13T16:37:49.588Z" }, - { url = "https://files.pythonhosted.org/packages/79/f9/b6f60f978397c616fd8dacca2305759fe4f80d397b20ef72534803244bd5/triton-3.5.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8457b22148defefdcb7fa8144b05ce211b9faefad650a1ce85b23df488d5549c", size = 159926731, upload-time = "2025-10-15T19:15:49.682Z" }, - { url = "https://files.pythonhosted.org/packages/3d/78/949a04391c21956c816523678f0e5fa308eb5b1e7622d88c4e4ef5fceca0/triton-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f34bfa21c5b3a203c0f0eab28dcc1e49bd1f67d22724e77fb6665a659200a4ec", size = 170433488, upload-time = "2025-10-13T16:37:57.132Z" }, - { url = "https://files.pythonhosted.org/packages/87/9b/30988039e1e84df7554fba24e6a734d2d0e847af33cabdf9b532b3c51456/triton-3.5.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da21fccceafc163e3a5e857abe34351ef76345af06cabf9637a914742671f0b", size = 159946647, upload-time = "2025-10-15T19:15:56.325Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3a/e991574f3102147b642e49637e0281e9bb7c4ba254edb2bab78247c85e01/triton-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9e71db82261c4ffa3921cd050cd5faa18322d2d405c30eb56084afaff3b0833", size = 170476535, upload-time = "2025-10-13T16:38:05.18Z" }, - { url = "https://files.pythonhosted.org/packages/cd/85/e37f1197acb04c8f3d83851d23d5d6ed5060ef74580668b112e23fdfa203/triton-3.5.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:188da5b81fa2f8322c27fec1627703eac24cb9bb7ab0dfbe9925973bc1b070d3", size = 159958970, upload-time = "2025-10-15T19:16:01.717Z" }, - { url = "https://files.pythonhosted.org/packages/6c/29/10728de8a6e932e517c10773486b8e99f85d1b1d9dd87d9a9616e1fef4a1/triton-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6bb9aa5519c084a333acdba443789e50012a4b851cd486c54f0b8dc2a8d3a12", size = 170487289, upload-time = "2025-10-13T16:38:11.662Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1d/38258f05010ac17a7b058c022911c9cae6526e149b7397134a048cf5a6c2/triton-3.5.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:03127d9b33aaf979c856676b394bc059ec1d68cb6da68ae03f62dd8ad77a04ae", size = 160073012, upload-time = "2025-10-15T19:16:07.477Z" }, - { url = "https://files.pythonhosted.org/packages/5c/38/db80e48b9220c9bce872b0f616ad0446cdf554a40b85c7865cbca99ab3c2/triton-3.5.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c83f2343e1a220a716c7b3ab9fccfcbe3ad4020d189549200e2d2e8d5868bed9", size = 170577179, upload-time = "2025-10-13T16:38:17.865Z" }, - { url = "https://files.pythonhosted.org/packages/91/fe/8f5771d00227f4eb1ee034f218ed427102b989366d2275fe3b3c105a3921/triton-3.5.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:468936651d383f4a6d10068d34a627505e13af55be5d002b9f27b987e7a5f0ac", size = 159957460, upload-time = "2025-10-15T19:16:12.626Z" }, - { url = "https://files.pythonhosted.org/packages/ff/60/1810655d1d856c9a4fcc90ee8966d85f552d98c53a6589f95ab2cbe27bb8/triton-3.5.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da0fa67ccd76c3dcfb0bffe1b1c57c685136a6bd33d141c24d9655d4185b1289", size = 170487949, upload-time = "2025-10-13T16:38:24.881Z" }, - { url = "https://files.pythonhosted.org/packages/78/59/99edd103958fe6e42b50b9ad8ce4f223ddf4ccf475259cf7d2b53381dc6c/triton-3.5.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7ceef21410229ac23173a28eee5cfc0e37c1dfdb8b4bc11ecda2e3ecec7c686", size = 160075629, upload-time = "2025-10-15T19:16:18.746Z" }, - { url = "https://files.pythonhosted.org/packages/fb/b7/1dec8433ac604c061173d0589d99217fe7bf90a70bdc375e745d044b8aad/triton-3.5.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:317fe477ea8fd4524a6a8c499fb0a36984a56d0b75bf9c9cb6133a1c56d5a6e7", size = 170580176, upload-time = "2025-10-13T16:38:31.14Z" }, + { url = "https://files.pythonhosted.org/packages/44/ba/b1b04f4b291a3205d95ebd24465de0e5bf010a2df27a4e58a9b5f039d8f2/triton-3.6.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781", size = 175972180, upload-time = "2026-01-20T16:15:53.664Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea", size = 188105201, upload-time = "2026-01-20T16:00:29.272Z" }, + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, + { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, + { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, + { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, ] [[package]] @@ -6485,7 +6753,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.13.0" +version = "0.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -6502,7 +6770,10 @@ dependencies = [ { name = "filelock" }, { name = "flashinfer-python" }, { name = "gguf" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, { name = "ijson" }, + { name = "kaldi-native-fbank" }, { name = "lark" }, { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, { name = "lm-format-enforcer" }, @@ -6513,9 +6784,15 @@ dependencies = [ { name = "ninja" }, { name = "numba" }, { name = "numpy" }, + { name = "nvidia-cudnn-frontend" }, + { name = "nvidia-cutlass-dsl" }, { name = "openai" }, { name = "openai-harmony" }, { name = "opencv-python-headless" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions-ai" }, { name = "outlines-core" }, { name = "partial-json-parser" }, { name = "pillow" }, @@ -6529,11 +6806,10 @@ dependencies = [ { name = "python-json-logger" }, { name = "pyyaml" }, { name = "pyzmq" }, + { name = "quack-kernels" }, { name = "ray", extra = ["cgraph"] }, { name = "regex" }, { name = "requests" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "sentencepiece" }, { name = "setproctitle" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, @@ -6549,10 +6825,10 @@ dependencies = [ { name = "watchfiles" }, { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/11/12/b922f96778d07df1c28dfa9a81fbc9706c13c5d0a4e8d154060818a79705/vllm-0.13.0.tar.gz", hash = "sha256:4ad43db45fef37114b550d03a4f423fb3fa3a31d8bc09ee810ef8b9cdcd4b5fe", size = 17828199, upload-time = "2025-12-19T03:30:32.741Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/7c/79ef306c71f3de2e73beb3c03c3f2966f966df61b8dd1f01dbdc65184050/vllm-0.17.1.tar.gz", hash = "sha256:d26a95dcb92e2ff78ed4b48bff247d845b0c768edf6c0acf2401376a56c57b61", size = 30547577, upload-time = "2026-03-11T11:03:58.693Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/82/e6194ac86862c50e9ff3f58ab3eb63d71604f96723bead2fcc610821197f/vllm-0.13.0-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:464b722c5c5d67a39593ada4a228f7558e860a732cb74a3bfa61c1b442b57581", size = 442031402, upload-time = "2025-12-19T03:31:07.026Z" }, - { url = "https://files.pythonhosted.org/packages/46/ae/36f87f514811c1389ff1a16e4e5b0b55f25ce782eb0eff2d7eaa92ff7deb/vllm-0.13.0-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:12b3d0a3b91c32a0091349de64b464f1c3d499a5b3a5d0ec387fef94ed5df6ee", size = 474942618, upload-time = "2025-12-19T03:31:35.593Z" }, + { url = "https://files.pythonhosted.org/packages/f5/02/ff63919abb341b0819f33a400c83698d095e5fd461ae3e44f3ff91f6489f/vllm-0.17.1-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:f04d63a94d0415b2323b0a0d3ab89a8d4d9bd346251ff60d47a7df679f7b3ff8", size = 385333057, upload-time = "2026-03-11T11:06:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/f85e67b390082481298e27a5c9f1da540d2d5abb1a06a594545cdc320818/vllm-0.17.1-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:c52e892309532b4e51cb94d022c5e3c0087300cdb56e4645708601443299d871", size = 432931666, upload-time = "2026-03-11T11:06:00.955Z" }, ] [[package]] @@ -6834,38 +7110,37 @@ wheels = [ [[package]] name = "xgrammar" -version = "0.1.27" +version = "0.1.29" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, - { name = "ninja" }, { name = "numpy" }, { name = "pydantic" }, { name = "torch" }, { name = "transformers" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/e1/b522b1e50fddd773d368c2945ef5ed628aa90c0c972027f9aa5a51d6d4f9/xgrammar-0.1.27.tar.gz", hash = "sha256:40af7bb2891f1633ec7f660723c74a92a963307d283aca9e3b4e53a0feaf1d46", size = 2303435, upload-time = "2025-11-04T03:11:53.512Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/55/03a548a116fa5cc716ce70e15240ca61ddaae046ed34c711e63d3d91d047/xgrammar-0.1.27-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6e3ea7cd74a7d4188744f90878507637ce9ac5f671cb9d1bda9d53305a46889e", size = 664256, upload-time = "2025-11-04T03:11:08.471Z" }, - { url = "https://files.pythonhosted.org/packages/84/a5/45a430a7fb44f70303742c59e7d792ca6d4b7960e9252ec5238f1112bbcd/xgrammar-0.1.27-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:73ca9ec86e81a7f936c5668b7e6dda6929c078d1748b7615c8da504584b6c24a", size = 637358, upload-time = "2025-11-04T03:11:10.975Z" }, - { url = "https://files.pythonhosted.org/packages/cb/2b/3867379f76b97fb7cb03bc0fa1d0f81f19d13df3c313bd22878b636b0f50/xgrammar-0.1.27-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa8b7cc167737a9b4c3e1012faa7b488cc5b451ea8403c4d77ec1d53b58e9266", size = 8687578, upload-time = "2025-11-04T03:11:13.304Z" }, - { url = "https://files.pythonhosted.org/packages/ae/35/fe718ec90c210ab892a845af6d4e6e876a3d3c7dcc1bacaa98abfec42c0f/xgrammar-0.1.27-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0c5899b59c8e45ba3a6f3b9e7fb2ef23243f09b164f724d59c7734173bb3db", size = 8869161, upload-time = "2025-11-04T03:11:15.572Z" }, - { url = "https://files.pythonhosted.org/packages/e8/44/674724714407e0265d088ad40a17d367c00d72d206f2b15d559a644a36dc/xgrammar-0.1.27-cp310-cp310-win_amd64.whl", hash = "sha256:1ce2558992b0ffda65f46772bae94b051d139f0036968853078904bc167d4a8d", size = 709212, upload-time = "2025-11-04T03:11:17.572Z" }, - { url = "https://files.pythonhosted.org/packages/93/bb/e6d30457c99a0ce11247154ecb1f3f9fab5960192a0564c2862ba9b98897/xgrammar-0.1.27-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c995c71ea94b153eac0e08c36eb82a898d7d71e4b77ce93f3b9fe648bd2d3a04", size = 664112, upload-time = "2025-11-04T03:11:18.932Z" }, - { url = "https://files.pythonhosted.org/packages/7e/81/caab5c46d314c1b005e36c9ec8aef124f7c52619d980f2bbd2d4cf4cd491/xgrammar-0.1.27-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:456f2f74135a414f44413599d90a382f5b22e6b515e4ae7e8938a28f7efacbaa", size = 637181, upload-time = "2025-11-04T03:11:20.29Z" }, - { url = "https://files.pythonhosted.org/packages/a4/29/7f78ed69b5f221206af0b68b0517335f9c09459def5d63065827a79fec74/xgrammar-0.1.27-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed23e6960218e791ecaccbbbb66d7caa5c0ed8636aca85807d81b89ba87a7f33", size = 8674617, upload-time = "2025-11-04T03:11:22.255Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a2/afcce6a59b83644ffe19ffebe8107355febb15d8084ce5316eccd457e3c8/xgrammar-0.1.27-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02fe3b137d041649b8f7a180a0aa7f3466d47579ce4e9fbdb77208b59621b2ab", size = 8869958, upload-time = "2025-11-04T03:11:24.751Z" }, - { url = "https://files.pythonhosted.org/packages/76/fb/a4a3254041174013ff09e99c298f2bc6c03f34891df458839de7cbb53e4b/xgrammar-0.1.27-cp311-cp311-win_amd64.whl", hash = "sha256:db0c74f7cc4fb2b5d566eee873e4d18920ed5ee0fe500178b412408d0dad3686", size = 709137, upload-time = "2025-11-04T03:11:26.672Z" }, - { url = "https://files.pythonhosted.org/packages/39/b6/09b43e2adff45d30ebcf9110d0ff753f4c96b368adaa2d166df3dee88d5f/xgrammar-0.1.27-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:6404a7714440eb86ab0379d749f33591274eeef04787dc00d61f22069f3ed51d", size = 663319, upload-time = "2025-11-04T03:11:28.682Z" }, - { url = "https://files.pythonhosted.org/packages/88/8b/53eb5c6d0df8df9f6350f182516a5b8c7b8b11d62650300d2c04af2bc4ea/xgrammar-0.1.27-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d01fa9894bc44a7f6a70b0301b59f3e310c0e0e7b7ea4cf5ce190b12d8220dd8", size = 636168, upload-time = "2025-11-04T03:11:30.373Z" }, - { url = "https://files.pythonhosted.org/packages/08/1b/53d30395bb973f13255d3e3a72961f95fdfb4083877c3f93bb626e3d1522/xgrammar-0.1.27-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:906c0601bac9170e1bab77ca985259035ff9c386c347efcb191555eab86e984e", size = 8676340, upload-time = "2025-11-04T03:11:32.203Z" }, - { url = "https://files.pythonhosted.org/packages/48/74/70cfac0171d9f309cfe18c5384330e3edc9466c436b258495fd30ecf29a3/xgrammar-0.1.27-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb68988a122f544301c496f2cac8ee82960ca7f5b3a42a952b2a00c0a55e6ca5", size = 8870650, upload-time = "2025-11-04T03:11:34.322Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a1/0392aa9c7669c56f7f88e4423b246476a74a72c3bb9db944e1bfc029985e/xgrammar-0.1.27-cp312-cp312-win_amd64.whl", hash = "sha256:3aac335ea052afc8f8dc34b9f2afcb9462a68189423aed9f60b0941db6cfc310", size = 708811, upload-time = "2025-11-04T03:11:36.214Z" }, - { url = "https://files.pythonhosted.org/packages/a4/77/5aee819c00844fb333fa802507182aa19445b347840103a14bd27ed944c4/xgrammar-0.1.27-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e248488c7c8a8ba175c7d1c5b55a2dd705661bbaa87755a749f9fdda146cbe1e", size = 636084, upload-time = "2025-11-04T03:11:38.192Z" }, - { url = "https://files.pythonhosted.org/packages/23/c2/cd15c44bd6db4411fc733303e0b85033772f3389b32210e6f0ae08f5a2c1/xgrammar-0.1.27-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ac7a307d7a739962c422969cb486aa3994e200bfa6191d9519fdca5224760f0", size = 8870005, upload-time = "2025-11-04T03:11:40.039Z" }, - { url = "https://files.pythonhosted.org/packages/be/45/d3d3dc97c05159d9336fb4b947b22bd074ca259bd291be523c00e5696d24/xgrammar-0.1.27-cp313-cp313-win_amd64.whl", hash = "sha256:37936e04974bcb4c02a69ab734ff530669a43b03b2910c4013233dd074896ac9", size = 708726, upload-time = "2025-11-04T03:11:42.064Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/02/a3/70dbe3ffd331a1e7e1ad5a95690a4086e6c7cdb8089f5c7eda712219ccec/xgrammar-0.1.29.tar.gz", hash = "sha256:cf195afa81b489eebf35d4c6f37f27136d05420739ab4a6f7f065c938d7e4baa", size = 2321317, upload-time = "2025-12-19T08:23:54.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/6d/6384619408da47411c71b2baed3d4bc509a4a9aa0a63d738709b516869b5/xgrammar-0.1.29-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fdc66e834b915cf956168ac086bd577f138261644b944e73d73f07085682a4d8", size = 16008147, upload-time = "2025-12-19T08:22:59.54Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2d/6ead6206bda4582620b176f02840254183c61682e20041a2d950d6f1ee7a/xgrammar-0.1.29-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48c5a5c60c5ca5ab09ff5ef9f6b382384a04b153bae5908006cd4f7d80d71e07", size = 17914539, upload-time = "2025-12-19T08:23:02.011Z" }, + { url = "https://files.pythonhosted.org/packages/04/75/5305fe75823489c160dec8ee2a95a631e44a690eacec765469e513aca738/xgrammar-0.1.29-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cea3e65d60f8e55568dbb1457e6c4da6d381262a9b1211fe023630630b733d8", size = 34702454, upload-time = "2025-12-19T08:23:05.143Z" }, + { url = "https://files.pythonhosted.org/packages/af/3c/7426aadf64a4ecfc1a1966babc57e4694235bf50392e96c506f930a4cdbe/xgrammar-0.1.29-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:866882b58ac654a1d1cd5e0c1ac67824b730aff8a40f9f19f0e8938a107dcd8a", size = 34903300, upload-time = "2025-12-19T08:23:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/05/f5/17ebcb575bd105cbcb5fee3c69906cee2423dbfdd73a18a60e205a619244/xgrammar-0.1.29-cp310-cp310-win_amd64.whl", hash = "sha256:8551dae4d38bd20c36a12c90a2954c3832bb6397211fc3aeba0b0d7920a1ea4b", size = 5928622, upload-time = "2025-12-19T08:23:10.485Z" }, + { url = "https://files.pythonhosted.org/packages/c6/de/88832fac40962fd0d4703bd4ba84598b06b8408bdc4a6722744f363f68a6/xgrammar-0.1.29-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:d2a7eef1b75b8d31b868d5c79855622aad203275ff267fc0e0ef77dd91906cfe", size = 16008004, upload-time = "2025-12-19T08:23:11.998Z" }, + { url = "https://files.pythonhosted.org/packages/76/f6/4d22eec5305657430955442077306bc6ed85becc564116165d4b3a7049ad/xgrammar-0.1.29-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4af7f6ce2b2c6295b936b7cbda09f78e33f2c492a139cd64560f5d8d0fe967ed", size = 17914326, upload-time = "2025-12-19T08:23:14.43Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/b5e5c99ce13a9d378a940cda07c5a08b50cc7efb66936c6ac8fa8232a0d5/xgrammar-0.1.29-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51bcfd63bd48a0b26209ffd2143a42067518559355ec9e4e574cef2ae74fac7c", size = 34699408, upload-time = "2025-12-19T08:23:16.906Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/4ebc1b3f5af79a3f73d0566034758f3fbcd9c64174646314a9a6f7cc1d27/xgrammar-0.1.29-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e27b50cf8c565845295a8263a4a0790c00a7c1fd783e76222fc0f575654d6f56", size = 34903461, upload-time = "2025-12-19T08:23:19.556Z" }, + { url = "https://files.pythonhosted.org/packages/77/21/f6b3978dc9761bbfbbb153d33441206ce2253efa271d8e2d8b6b210d2bd7/xgrammar-0.1.29-cp311-cp311-win_amd64.whl", hash = "sha256:c9f8ea76bcf41b48168974b509b1546d2bee289ff1b20c68bc97434c1ea6e49a", size = 5928633, upload-time = "2025-12-19T08:23:21.67Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d8/fb282fc78be6e9bbefb5cb389f66b22e4efd6ae14f06234f599651620da5/xgrammar-0.1.29-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:d992a3cee7594bbdaa64ae59f90da5ce21c5fe654719df3816014289ada6f04d", size = 16007376, upload-time = "2025-12-19T08:23:23.634Z" }, + { url = "https://files.pythonhosted.org/packages/82/a7/2c9767620ee50f2f40f1eb95e55a3a29e1a0670f087ee6dc1bc1c887b906/xgrammar-0.1.29-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1bbdf02e45cfa8614218ba01ca7952d375f8bc1c13884e3d04daa4b54180cbc2", size = 17913535, upload-time = "2025-12-19T08:23:26.02Z" }, + { url = "https://files.pythonhosted.org/packages/57/94/18793c64bf0368075a34c06e196bf002f1e6ab0aee332268f44e8d356d5a/xgrammar-0.1.29-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eb370a16b27a683e5f2b9e429ab41440c69977d4a504849ed61831b94cc704c", size = 34705239, upload-time = "2025-12-19T08:23:28.369Z" }, + { url = "https://files.pythonhosted.org/packages/3e/da/4c14e3e00be698009b52700f15326a23272b4b00475939b6acc86b151188/xgrammar-0.1.29-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79e6e4f5cd33be77418cf91efc482f2b3d773d309891224383bc8a4948ad7b07", size = 34906135, upload-time = "2025-12-19T08:23:30.838Z" }, + { url = "https://files.pythonhosted.org/packages/22/d8/34423997f48627cef3b74cc894d9dfcaacae02941c06237ac5f3196406a7/xgrammar-0.1.29-cp312-cp312-win_amd64.whl", hash = "sha256:39bdfadedbce34599835486164fa80ba00248c6c75ad91f3843db90ef37e037f", size = 5928381, upload-time = "2025-12-19T08:23:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ef/8a4b4cb10fc996c0a25c9bf5613aaf5a86114291a9a4003e43605cab42bf/xgrammar-0.1.29-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fedf21e447ef646f23a6e2d11877c0812d55965dcf8c0aa9b0f32590c9b6e22a", size = 17913609, upload-time = "2025-12-19T08:23:36.06Z" }, + { url = "https://files.pythonhosted.org/packages/e9/c5/e4965c9921e7bb6061f246ae7f8c7b9b1dfc21262248100c2f9b398b361e/xgrammar-0.1.29-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb22aea775971f7d8c4d0e193257ebeb71b68acd9d36af3331ca5fd4d9a46991", size = 34904126, upload-time = "2025-12-19T08:23:38.335Z" }, + { url = "https://files.pythonhosted.org/packages/09/26/641d7ee1a59e526aa94be980c485f899088d09dd1af517a2e1d0e85853bc/xgrammar-0.1.29-cp313-cp313-win_amd64.whl", hash = "sha256:12e6d63e892e9da8d088569dd629af58a5eafd909dc58788d499c4fd74bcd2a1", size = 5928450, upload-time = "2025-12-19T08:23:40.667Z" }, ] [[package]] @@ -6993,3 +7268,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/b7/503c98092fb3b344a179579f55814b613c1fbb1c23b3ec14a7b008a66a6e/yarl-1.22.0-cp314-cp314t-win_arm64.whl", hash = "sha256:9f6d73c1436b934e3f01df1e1b21ff765cd1d28c77dfb9ace207f746d4610ee1", size = 85171, upload-time = "2025-10-06T14:12:16.935Z" }, { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, ] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] From 518335fa3d29ed36d6d09ccaae9e97d9cd77000e Mon Sep 17 00:00:00 2001 From: zhouhy Date: Wed, 18 Mar 2026 17:03:36 +0800 Subject: [PATCH 3/4] handle transformers>=5.0 issues --- .../sft/oss260312/step_sft_data_config0311.py | 5 +- playground/rlvr/qwen3_1p5b_rlvr_math.py | 4 +- steptronoss/tokenizer/hf_compat_tokenizer.py | 105 ++++++++++++++- tests/test_hf_compat_tokenizer.py | 125 ++++++++++++++++++ 4 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 tests/test_hf_compat_tokenizer.py diff --git a/playground/data/sft/oss260312/step_sft_data_config0311.py b/playground/data/sft/oss260312/step_sft_data_config0311.py index db4fdd88..ab7ffc79 100644 --- a/playground/data/sft/oss260312/step_sft_data_config0311.py +++ b/playground/data/sft/oss260312/step_sft_data_config0311.py @@ -175,11 +175,10 @@ def get_dataset(self, filelist, template): return StepChatJsonDataset(filelist=filelist, template=template) def get_template(self): - from transformers import AutoTokenizer - from steptronoss.data.chat_templates.text_template import HuggingFaceTemplate + from steptronoss.tokenizer.hf_compat_tokenizer import load_hf_tokenizer - tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_path) + tokenizer = load_hf_tokenizer(self.tokenizer_path) return HuggingFaceTemplate(tokenizer=tokenizer) diff --git a/playground/rlvr/qwen3_1p5b_rlvr_math.py b/playground/rlvr/qwen3_1p5b_rlvr_math.py index 405b26f3..3da95cd0 100644 --- a/playground/rlvr/qwen3_1p5b_rlvr_math.py +++ b/playground/rlvr/qwen3_1p5b_rlvr_math.py @@ -107,9 +107,9 @@ class Qwen3TokenizerConfig(TokenizerConfig): tokenizer_path: str = "/oss/opensources_model/Qwen3-1.7B/" def build_tokenizer(self): - from transformers import AutoTokenizer + from steptronoss.tokenizer.hf_compat_tokenizer import load_hf_tokenizer - return AutoTokenizer.from_pretrained(self.tokenizer_path, trust_remote_code=True) + return load_hf_tokenizer(self.tokenizer_path, trust_remote_code=True) class TinyRLVRResourceConfig(ResourceConfig): diff --git a/steptronoss/tokenizer/hf_compat_tokenizer.py b/steptronoss/tokenizer/hf_compat_tokenizer.py index e88cd60c..3bc86b18 100644 --- a/steptronoss/tokenizer/hf_compat_tokenizer.py +++ b/steptronoss/tokenizer/hf_compat_tokenizer.py @@ -1,3 +1,9 @@ +from __future__ import annotations + +import json +import os +from typing import Any + from transformers.tokenization_utils_base import PreTrainedTokenizerBase @@ -28,13 +34,23 @@ class HFCompatTokenizer: hf_tokenizer: PreTrainedTokenizerBase - def __init__(self): - import os + def __init__( + self, + hf_path: str | None = None, + hf_tokenizer: PreTrainedTokenizerBase | None = None, + **kwargs, + ): + os.environ["TOKENIZERS_PARALLELISM"] = "false" + if hf_tokenizer is not None: + self.hf_tokenizer = hf_tokenizer + if hf_path is not None: + self.hf_path = hf_path + return - from transformers import AutoTokenizer + if hf_path is not None: + self.hf_path = hf_path - os.environ["TOKENIZERS_PARALLELISM"] = "false" - self.hf_tokenizer = AutoTokenizer.from_pretrained(self.hf_path) + self.hf_tokenizer = _load_raw_hf_tokenizer(self.hf_path, **kwargs) def encode( self, @@ -63,7 +79,10 @@ def decode( ) def apply_chat_template(self, *args, **kwargs): - return self.hf_tokenizer.apply_chat_template(*args, **kwargs) + tokenized = self.hf_tokenizer.apply_chat_template(*args, **kwargs) + if kwargs.get("tokenize", True): + return self._normalize_tokenized_input_ids(tokenized) + return tokenized def __getattr__(self, key: str): return getattr(self.hf_tokenizer, key) @@ -80,3 +99,77 @@ def __setstate__(self, state): # Restore the state from the pickled dictionary. # This is a direct assignment, which won't trigger __getattr__. self.hf_tokenizer = state["hf_tokenizer"] + + @staticmethod + def _normalize_tokenized_input_ids(tokenized: Any) -> list[int]: + if hasattr(tokenized, "input_ids"): + tokenized = tokenized.input_ids + elif isinstance(tokenized, dict) and "input_ids" in tokenized: + tokenized = tokenized["input_ids"] + + if hasattr(tokenized, "tolist"): + tokenized = tokenized.tolist() + + if isinstance(tokenized, tuple): + tokenized = list(tokenized) + + if isinstance(tokenized, list) and tokenized and isinstance(tokenized[0], list): + tokenized = tokenized[0] + + if not isinstance(tokenized, list): + raise TypeError(f"Unsupported tokenized output type: {type(tokenized).__name__}") + + return tokenized + + +def _should_use_generic_fast_tokenizer(hf_path: str) -> bool: + if not os.path.isdir(hf_path): + return False + + tokenizer_json = os.path.join(hf_path, "tokenizer.json") + tokenizer_model = os.path.join(hf_path, "tokenizer.model") + tokenizer_config = os.path.join(hf_path, "tokenizer_config.json") + if not os.path.isfile(tokenizer_json) or os.path.isfile(tokenizer_model): + return False + if not os.path.isfile(tokenizer_config): + return False + + try: + with open(tokenizer_config, encoding="utf-8") as fh: + config = json.load(fh) + except (OSError, json.JSONDecodeError): + return False + + # Transformers 5 aliases LlamaTokenizerFast to a generic tokenizers-backed + # LlamaTokenizer implementation. For byte-level BPE tokenizers without a + # SentencePiece model, loading through the generic fast class preserves the + # tokenizer.json pre-tokenizer/decoder pipeline exactly. + return config.get("tokenizer_class") == "LlamaTokenizerFast" + + +def _load_raw_hf_tokenizer(hf_path: str, **kwargs) -> PreTrainedTokenizerBase: + from transformers import AutoTokenizer, PreTrainedTokenizerFast + + if _should_use_generic_fast_tokenizer(hf_path): + return PreTrainedTokenizerFast.from_pretrained(hf_path, **kwargs) + return AutoTokenizer.from_pretrained(hf_path, **kwargs) + + +def load_hf_tokenizer(hf_path: str, **kwargs) -> HFCompatTokenizer: + """Load a tokenizer with the pre-HF-5 `AutoTokenizer.from_pretrained` contract. + + Call sites historically treated this as equivalent to + `AutoTokenizer.from_pretrained(...)`: local tokenizer config decides the + implementation, and `apply_chat_template(tokenize=True)` yields token IDs + directly. The Transformers 5.0 upgrade broke that assumption for some + local byte-level BPE tokenizers (for example Step3.5Flash SFT), where the + auto loader no longer preserves the `tokenizer.json` behavior and changes + tokenization results. + + This helper stabilizes the interface by: + - selecting `PreTrainedTokenizerFast.from_pretrained(...)` for the known + LlamaTokenizerFast/byte-level local tokenizer shape that regressed in HF5 + - wrapping the result in `HFCompatTokenizer`, which normalizes + `apply_chat_template(tokenize=True)` back to `list[int]` + """ + return HFCompatTokenizer(hf_path=hf_path, **kwargs) diff --git a/tests/test_hf_compat_tokenizer.py b/tests/test_hf_compat_tokenizer.py new file mode 100644 index 00000000..e01494b0 --- /dev/null +++ b/tests/test_hf_compat_tokenizer.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json + +import pytest + +from steptronoss.tokenizer.hf_compat_tokenizer import HFCompatTokenizer, load_hf_tokenizer + +pytestmark = pytest.mark.cpu + + +class _RawBatchTokenizer: + def apply_chat_template(self, *args, **kwargs): + del args, kwargs + return { + "input_ids": [7, 8, 9], + "attention_mask": [1, 1, 1], + } + + def encode(self, text: str, **kwargs): + del kwargs + return [ord(ch) for ch in text] + + def decode(self, token_ids, **kwargs): + del kwargs + return "".join(chr(token_id) for token_id in token_ids) + + +def test_hf_compat_tokenizer_normalizes_apply_chat_template_token_ids(): + tokenizer = HFCompatTokenizer(hf_tokenizer=_RawBatchTokenizer()) + + assert tokenizer.apply_chat_template([], tokenize=True, add_generation_prompt=True) == [7, 8, 9] + + +def test_hf_compat_tokenizer_keeps_non_tokenized_chat_template_output(): + class _RawStringTokenizer(_RawBatchTokenizer): + def apply_chat_template(self, *args, **kwargs): + del args, kwargs + return "prompt" + + tokenizer = HFCompatTokenizer(hf_tokenizer=_RawStringTokenizer()) + + assert tokenizer.apply_chat_template([], tokenize=False, add_generation_prompt=True) == "prompt" + + +def test_load_hf_tokenizer_prefers_generic_fast_loader_for_byte_level_llama(tmp_path, monkeypatch): + (tmp_path / "tokenizer.json").write_text("{}", encoding="utf-8") + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"tokenizer_class": "LlamaTokenizerFast"}), + encoding="utf-8", + ) + + calls: list[tuple[str, str]] = [] + + class _SentinelTokenizer: + def encode(self, text: str, **kwargs): + del text, kwargs + return [] + + def decode(self, token_ids, **kwargs): + del token_ids, kwargs + return "" + + def apply_chat_template(self, *args, **kwargs): + del args, kwargs + return [] + + def _auto_loader(path, **kwargs): + del kwargs + calls.append(("auto", str(path))) + return _SentinelTokenizer() + + def _fast_loader(path, **kwargs): + del kwargs + calls.append(("fast", str(path))) + return _SentinelTokenizer() + + monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _auto_loader) + monkeypatch.setattr("transformers.PreTrainedTokenizerFast.from_pretrained", _fast_loader) + + tokenizer = load_hf_tokenizer(str(tmp_path)) + + assert isinstance(tokenizer, HFCompatTokenizer) + assert calls == [("fast", str(tmp_path))] + + +def test_load_hf_tokenizer_uses_auto_loader_for_regular_tokenizer(tmp_path, monkeypatch): + (tmp_path / "tokenizer.json").write_text("{}", encoding="utf-8") + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"tokenizer_class": "Qwen2TokenizerFast"}), + encoding="utf-8", + ) + + calls: list[tuple[str, str]] = [] + + class _SentinelTokenizer: + def encode(self, text: str, **kwargs): + del text, kwargs + return [] + + def decode(self, token_ids, **kwargs): + del token_ids, kwargs + return "" + + def apply_chat_template(self, *args, **kwargs): + del args, kwargs + return [] + + def _auto_loader(path, **kwargs): + del kwargs + calls.append(("auto", str(path))) + return _SentinelTokenizer() + + def _fast_loader(path, **kwargs): + del kwargs + calls.append(("fast", str(path))) + return _SentinelTokenizer() + + monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _auto_loader) + monkeypatch.setattr("transformers.PreTrainedTokenizerFast.from_pretrained", _fast_loader) + + tokenizer = load_hf_tokenizer(str(tmp_path)) + + assert isinstance(tokenizer, HFCompatTokenizer) + assert calls == [("auto", str(tmp_path))] From 30f012aa94fa968f001b526f471e8669d774175c Mon Sep 17 00:00:00 2001 From: zhouhy Date: Wed, 18 Mar 2026 17:17:11 +0800 Subject: [PATCH 4/4] modify according to review --- .../eval/benchmarks/AIME25/benchmark.py | 24 ++++++++------- .../eval/benchmarks/GPQADiamond/benchmark.py | 19 ++++++------ .../eval/benchmarks/HMMT25/benchmark.py | 1 - .../eval/benchmarks/IFBench/benchmark.py | 9 +----- .../eval/benchmarks/MMLUPro/benchmark.py | 8 ++--- playground/eval/eval_sets/simple_eval.py | 8 +++++ .../qwen3_1p7b_eval_simple_benchmarks.py | 4 +-- .../step3p5/step3p5_eval_simple_benchmarks.py | 6 ++-- tests/benchmarks/test_ifbench_benchmark.py | 20 +++++++++++++ tests/benchmarks/test_math_benchmarks.py | 16 +++++----- tests/test_simple_eval_cache.py | 29 +++++++++++++++++++ 11 files changed, 98 insertions(+), 46 deletions(-) diff --git a/playground/eval/benchmarks/AIME25/benchmark.py b/playground/eval/benchmarks/AIME25/benchmark.py index 31248bc7..e588e734 100644 --- a/playground/eval/benchmarks/AIME25/benchmark.py +++ b/playground/eval/benchmarks/AIME25/benchmark.py @@ -85,25 +85,27 @@ def _extract_answer(cls, response: str) -> str: return response.strip() @staticmethod - def _is_correct(result: Generated, answer: str) -> bool: + def _gold_answer(result: Generated) -> str: + answer = result.case.benchmark.context.get("answer") + return answer.strip() if isinstance(answer, str) else str(answer).strip() + + @classmethod + def _is_correct(cls, result: Generated) -> bool: if result.error: return False - predicted_raw = AIME25Benchmark._extract_answer(result.response) - predicted = AIME25Benchmark._normalize_answer_text(predicted_raw) - normalized_answer = AIME25Benchmark._normalize_answer_text(answer) + answer = cls._gold_answer(result) + predicted_raw = cls._extract_answer(result.response) + predicted = cls._normalize_answer_text(predicted_raw) + normalized_answer = cls._normalize_answer_text(answer) if predicted == normalized_answer: return True - return AIME25Benchmark._math_verify_equal(predicted_raw, answer) + return cls._math_verify_equal(predicted_raw, answer) def evaluate(self, results: list[Generated]) -> BaseMetric: - def _gold_answer(result: Generated) -> str: - answer = result.case.benchmark.context.get("answer") - return answer.strip() if isinstance(answer, str) else str(answer).strip() - - sample_values = [1.0 if self._is_correct(result, _gold_answer(result)) else 0.0 for result in results] + sample_values = [1.0 if self._is_correct(result) else 0.0 for result in results] return self._build_metric( results=results, sample_values=sample_values, sample_per_prompt=self.sample_per_prompt, - is_success_fn=lambda result: self._is_correct(result, _gold_answer(result)), + is_success_fn=self._is_correct, ) diff --git a/playground/eval/benchmarks/GPQADiamond/benchmark.py b/playground/eval/benchmarks/GPQADiamond/benchmark.py index 87f2fb91..976e79aa 100644 --- a/playground/eval/benchmarks/GPQADiamond/benchmark.py +++ b/playground/eval/benchmarks/GPQADiamond/benchmark.py @@ -28,21 +28,22 @@ def _extract_choice(cls, response: str) -> str: return "" @staticmethod - def _is_correct(result: Generated, answer: str) -> bool: + def _gold_answer(result: Generated) -> str: + answer = result.case.benchmark.context.get("answer") + return answer.strip() if isinstance(answer, str) else str(answer).strip() + + @classmethod + def _is_correct(cls, result: Generated) -> bool: if result.error: return False - predicted = GPQADiamondBenchmark._extract_choice(result.response) - return predicted == answer.strip().upper() + predicted = cls._extract_choice(result.response) + return predicted == cls._gold_answer(result).upper() def evaluate(self, results: list[Generated]) -> BaseMetric: - def _gold_answer(result: Generated) -> str: - answer = result.case.benchmark.context.get("answer") - return answer.strip() if isinstance(answer, str) else str(answer).strip() - - sample_values = [1.0 if self._is_correct(result, _gold_answer(result)) else 0.0 for result in results] + sample_values = [1.0 if self._is_correct(result) else 0.0 for result in results] return self._build_metric( results=results, sample_values=sample_values, sample_per_prompt=self.sample_per_prompt, - is_success_fn=lambda result: self._is_correct(result, _gold_answer(result)), + is_success_fn=self._is_correct, ) diff --git a/playground/eval/benchmarks/HMMT25/benchmark.py b/playground/eval/benchmarks/HMMT25/benchmark.py index 6f85ab10..ef3b1822 100644 --- a/playground/eval/benchmarks/HMMT25/benchmark.py +++ b/playground/eval/benchmarks/HMMT25/benchmark.py @@ -3,4 +3,3 @@ class HMMT25Benchmark(AIME25Benchmark): dataset_name = "HMMT25" - pass diff --git a/playground/eval/benchmarks/IFBench/benchmark.py b/playground/eval/benchmarks/IFBench/benchmark.py index da69ce5f..e8f283cd 100644 --- a/playground/eval/benchmarks/IFBench/benchmark.py +++ b/playground/eval/benchmarks/IFBench/benchmark.py @@ -2,7 +2,6 @@ import copy import json -import random import re from dataclasses import dataclass, field from pathlib import Path @@ -160,13 +159,7 @@ def _load_records(self) -> list[tuple[str, list[dict[str, str]], str, JsonObject context, )) self._records_cache = records - records = list(self._records_cache) - if self.shuffle_prompts: - rng = random.Random(1234) - rng.shuffle(records) - if self.down_sample_to is not None: - records = records[: self.down_sample_to] - return records + return super()._load_records() def get_cases(self): cases = super().get_cases() diff --git a/playground/eval/benchmarks/MMLUPro/benchmark.py b/playground/eval/benchmarks/MMLUPro/benchmark.py index c64a70e2..71b7a410 100644 --- a/playground/eval/benchmarks/MMLUPro/benchmark.py +++ b/playground/eval/benchmarks/MMLUPro/benchmark.py @@ -9,9 +9,9 @@ class MMLUProBenchmark(GPQADiamondBenchmark): _OPTION_PATTERN = re.compile(r"(? bool: + @classmethod + def _is_correct(cls, result: Generated) -> bool: if result.error: return False - predicted = MMLUProBenchmark._extract_choice(result.response) - return predicted == answer.strip().upper() + predicted = cls._extract_choice(result.response) + return predicted == cls._gold_answer(result).upper() diff --git a/playground/eval/eval_sets/simple_eval.py b/playground/eval/eval_sets/simple_eval.py index 6f234855..72426829 100644 --- a/playground/eval/eval_sets/simple_eval.py +++ b/playground/eval/eval_sets/simple_eval.py @@ -69,6 +69,14 @@ def __init__( resolved_max_tokens = remaining_context if sampling_params.max_tokens is not None: resolved_max_tokens = min(sampling_params.max_tokens, remaining_context) + if resolved_max_tokens < sampling_params.max_tokens: + logger.warning( + "Clamped generation budget for " + f"{self.case.benchmark.benchmark_name}:{self.case.benchmark.item_id} " + f"from max_tokens={sampling_params.max_tokens} to {resolved_max_tokens} " + f"because prompt_token_count={prompt.prompt_token_count} leaves only " + f"{remaining_context} tokens under max_model_len={max_model_len}." + ) resolved_sampling_params = SamplingParams( temperature=sampling_params.temperature, top_p=sampling_params.top_p, diff --git a/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py b/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py index 3684c5e5..cdb4b14b 100644 --- a/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py +++ b/playground/eval/qwen3/qwen3_1p7b_eval_simple_benchmarks.py @@ -17,9 +17,9 @@ class Qwen3TokenizerConfig(TokenizerConfig): """Tokenizer directory for Qwen3-1.7B.""" def build_tokenizer(self) -> ChatTokenizer: - from transformers import AutoTokenizer + from steptronoss.tokenizer.hf_compat_tokenizer import load_hf_tokenizer - return AutoTokenizer.from_pretrained(self.tokenizer_path, trust_remote_code=True) + return load_hf_tokenizer(self.tokenizer_path, trust_remote_code=True) class Qwen3SimpleEvalResourceConfig(ResourceConfig): diff --git a/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py b/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py index 1e57a543..5b66e89d 100644 --- a/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py +++ b/playground/eval/step3p5/step3p5_eval_simple_benchmarks.py @@ -36,9 +36,9 @@ class Step3p5TokenizerConfig(TokenizerConfig): """Tokenizer directory for the target Step3.5 model family.""" def build_tokenizer(self) -> ChatTokenizer: - from transformers import AutoTokenizer + from steptronoss.tokenizer.hf_compat_tokenizer import load_hf_tokenizer - return AutoTokenizer.from_pretrained(self.tokenizer_path, trust_remote_code=True) + return load_hf_tokenizer(self.tokenizer_path, trust_remote_code=True) class Step3p5SimpleEvalResourceConfig(ResourceConfig): @@ -85,7 +85,7 @@ class Step3p5SimpleEvalVLLMDeployConfig(VLLMDeployConfig): def __init__(self): super().__init__() self.model_config_path = "/oss/checkpoints/step3_flash_sft_step3_data_muon/it4716/hf_vllm/" - self.tokenizer_path = "/oss/tokenizers/step3p5_flash_sft/" + self.tokenizer_path = "/oss/tokenizers/Step3.5Flash-SFT-Tokenizer/" self.reasoning_parser = "step3p5" self.max_seq_len = 128 * 1024 self.vllm_gpu_memory_utilization = 0.9 diff --git a/tests/benchmarks/test_ifbench_benchmark.py b/tests/benchmarks/test_ifbench_benchmark.py index e31c9836..1dd0a699 100644 --- a/tests/benchmarks/test_ifbench_benchmark.py +++ b/tests/benchmarks/test_ifbench_benchmark.py @@ -175,3 +175,23 @@ def test_ifbench_metric_includes_official_strict_and_loose_reports(tmp_path): assert strict_metric.score_avg == 0.25 assert strict_metric.to_dict()["evaluation_mode"] == "strict" assert strict_metrics == loose_metrics + + +def test_ifbench_load_records_reuses_parent_shuffle_and_downsample(tmp_path): + resource_root = tmp_path / "datasets" / "IFBENCH" + resource_root.mkdir(parents=True) + _write_official_style_prompt_file(resource_root / "IFBench_test.jsonl") + + benchmark = IFBenchBenchmark( + data_path=str(resource_root), + tokenizer=DummyTokenizer(), + sample_per_prompt=1, + shuffle_prompts=True, + down_sample_to=1, + ) + + records = benchmark._load_records() + + assert len(records) == 1 + assert records[0][0] == "IFBENCH" + assert records[0][1][0]["role"] == "user" diff --git a/tests/benchmarks/test_math_benchmarks.py b/tests/benchmarks/test_math_benchmarks.py index e30db379..56f57b79 100644 --- a/tests/benchmarks/test_math_benchmarks.py +++ b/tests/benchmarks/test_math_benchmarks.py @@ -7,7 +7,7 @@ from steptronoss.generation.base_benchmark import BenchmarkMeta, EvaluationCase, EvaluationMeta, Generated, Prompt -def _make_generated(response: str) -> Generated: +def _make_generated(response: str, answer: str = "") -> Generated: return Generated( case=EvaluationCase( prompt=Prompt( @@ -17,7 +17,7 @@ def _make_generated(response: str) -> Generated: benchmark=BenchmarkMeta( benchmark_name="TEST", item_id="test-item", - context={}, + context={"answer": answer}, ), evaluation=EvaluationMeta(prompt_index=0, run_index=0), ), @@ -31,16 +31,16 @@ def test_aime_extract_answer_handles_nested_boxed_braces(): def test_hmmt_accepts_whitespace_and_dfrac_variants(): - result = _make_generated(r"\boxed{\dfrac{1}{576}}") - assert HMMT25Benchmark._is_correct(result, r"\frac{1}{576}") + result = _make_generated(r"\boxed{\dfrac{1}{576}}", answer=r"\frac{1}{576}") + assert HMMT25Benchmark._is_correct(result) def test_hmmt_accepts_boxed_answers_with_nested_braces(): - result = _make_generated(r"\boxed{1 - \frac{2}{\pi}}") - assert HMMT25Benchmark._is_correct(result, r"1-\frac{2}{\pi}") + result = _make_generated(r"\boxed{1 - \frac{2}{\pi}}", answer=r"1-\frac{2}{\pi}") + assert HMMT25Benchmark._is_correct(result) def test_hmmt_accepts_math_verify_equivalent_forms_when_available(): pytest.importorskip("math_verify") - result = _make_generated(r"\boxed{\frac{9}{\sqrt{23}}}") - assert HMMT25Benchmark._is_correct(result, r"\frac{9 \sqrt{23}}{23}") + result = _make_generated(r"\boxed{\frac{9}{\sqrt{23}}}", answer=r"\frac{9 \sqrt{23}}{23}") + assert HMMT25Benchmark._is_correct(result) diff --git a/tests/test_simple_eval_cache.py b/tests/test_simple_eval_cache.py index 6aa6a8f0..23d5b9f3 100644 --- a/tests/test_simple_eval_cache.py +++ b/tests/test_simple_eval_cache.py @@ -410,3 +410,32 @@ def test_generate_rejects_duplicate_fingerprints(tmp_path): with pytest.raises(ValueError, match="Duplicate generation fingerprint detected"): cfg._generate([genable_a, genable_b]) + + +def test_simple_chat_generatable_warns_when_context_budget_is_clamped(monkeypatch): + warning_messages: list[str] = [] + monkeypatch.setattr(simple_eval.logger, "warning", lambda message: warning_messages.append(message)) + + case = EvaluationCase( + prompt=Prompt( + messages=[{"role": "user", "content": "clamped prompt"}], + prompt_token_count=1018, + ), + benchmark=BenchmarkMeta( + benchmark_name="bench", + item_id="item-0", + context={"item_id": "item-0"}, + ), + evaluation=EvaluationMeta(prompt_index=0, run_index=0), + ) + + genable = _CountingSimpleChatGeneratable( + case=case, + response_text="ok", + call_counter={}, + ) + + assert genable.case.prompt.sampling_params is not None + assert genable.case.prompt.sampling_params.max_tokens == 6 + assert len(warning_messages) == 1 + assert "Clamped generation budget" in warning_messages[0]