diff --git a/.gitattributes b/.gitattributes index cbb8c0b0f4..ea625e6fd6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ -benchmark/locomo/dataset/locomo10.json text eol=lf +evaluation/memory/locomo/dataset/locomo10.json text eol=lf e2e/bub/harbor-tasks/** text eol=lf e2e/bub/tasks/*.yaml text eol=lf integrations/dsh/plugins/powercontext/src/operations.generated.ts text eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a8895076db..2e86703832 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -28,7 +28,9 @@ /tests/ @Teingi @PsiACE /e2e/ @PsiACE @AlexStocks /evaluation/ @PsiACE @frf12 -/benchmark/ @Teingi @Alanxtl +/evaluation/memory/ @Teingi @Alanxtl +/evaluation/memory/longmemeval_v2/ @PsiACE @frf12 +/evaluation/performance/ @Teingi @Alanxtl # Documentation, website, and examples. /docs/ @zhanghuidinah @PsiACE diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 50f3dc0df5..71f8f87f94 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -77,13 +77,8 @@ jobs: - name: Set up the environment uses: ./.github/actions/setup-python-env - # `evaluation/` is its own uv project with its own `testpaths`, so the root - # unit-test job never collected it and the benchmark's own tests were only - # ever run by hand. The job starts at the unit subtree because that is the - # subtree this change owns; `evaluation/tests/web` and - # `evaluation/tests/contract` are already red on `master` for reasons - # unrelated to the benchmark, and gating them here would make this check - # red before it could report anything. + # Evaluation has its own uv project. Collect each suite's unit tests through + # its dedicated path; the root project's testpaths do not include them. - name: Run the evaluation project unit tests run: | set -o pipefail diff --git a/.github/workflows/skill-guidance.yml b/.github/workflows/skill-guidance.yml index be1cfc6b2d..0878b7c980 100644 --- a/.github/workflows/skill-guidance.yml +++ b/.github/workflows/skill-guidance.yml @@ -3,14 +3,14 @@ name: Skill guidance validation on: pull_request: paths: - - evaluation/skill-up/** + - evaluation/skills/skill-up/** - openapi/powercontext.yaml - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** - .github/workflows/skill-guidance.yml push: branches: [master] paths: - - evaluation/skill-up/** + - evaluation/skills/skill-up/** - openapi/powercontext.yaml - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** - .github/workflows/skill-guidance.yml @@ -31,7 +31,7 @@ jobs: with: python-version: '3.11' - name: Install validation dependencies - run: python -m pip install -r evaluation/skill-up/requirements-dev.txt + run: python -m pip install -r evaluation/skills/skill-up/requirements-dev.txt - name: Install checksum-pinned skill-up shell: bash run: | @@ -43,7 +43,7 @@ jobs: tar -xzf skill-up.tar.gz echo "$RUNNER_TEMP/skill-up" >> "$GITHUB_PATH" - name: Verify pin, fixtures, reports and native schema - working-directory: evaluation/skill-up + working-directory: evaluation/skills/skill-up run: | python sync_skill.py --check python validate_suite.py diff --git a/.gitignore b/.gitignore index f13511e905..55bea765cd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,10 +27,10 @@ build/ develop-eggs/ dist/ node_modules/ -evaluation/web/node_modules/ -evaluation/web/playwright-report/ -evaluation/web/test-results/ -evaluation/web/*.tsbuildinfo +evaluation/coding/swebench_pro/web/node_modules/ +evaluation/coding/swebench_pro/web/playwright-report/ +evaluation/coding/swebench_pro/web/test-results/ +evaluation/coding/swebench_pro/web/*.tsbuildinfo # Local native-code experiment and validation reports. /docs/*/rfcs/0000-native-git-code-understanding-validation.md /docs/*/rfcs/0000-native-git-code-understanding-experiment.md diff --git a/.licenserc.yaml b/.licenserc.yaml index 7fc9cd170a..5885cd449c 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -50,9 +50,9 @@ header: - '**/*.lock' - '**/*-lock.yaml' - '**/*.lockb' - - 'benchmark/locomo/dataset/**' - - 'benchmark/locomo/results/**' - - 'benchmark/locomo_plus/results/**' + - 'evaluation/memory/locomo/dataset/**' + - 'evaluation/memory/locomo/results/**' + - 'evaluation/memory/locomo_plus/results/**' - 'tox.ini' - 'Makefile' - 'zensical.toml' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b89f03a610..dacd82e9c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,10 +9,10 @@ repos: - id: check-json exclude: ^.devcontainer/devcontainer.json - id: pretty-format-json - exclude: ^(?:\.devcontainer/devcontainer\.json|benchmark/locomo/dataset/locomo10\.json)$ + exclude: ^(?:\.devcontainer/devcontainer\.json|evaluation/memory/locomo/dataset/locomo10\.json)$ args: [--autofix, --no-sort-keys, --no-ensure-ascii] - id: end-of-file-fixer - exclude: ^benchmark/locomo/dataset/locomo10\.json$ + exclude: ^evaluation/memory/locomo/dataset/locomo10\.json$ - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit diff --git a/Makefile b/Makefile index a9002357fb..96c7afe709 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,10 @@ e2e-test: ## Run CLI to Client SDK to Server end-to-end tests. .PHONY: evaluation-unit-test evaluation-unit-test: ## Run the evaluation project's unit tests, the work-continuity benchmark included. @uv sync --project evaluation --frozen - @uv run --project evaluation pytest -c evaluation/pyproject.toml evaluation/tests/unit -m "not live" -q + @uv run --project evaluation pytest -c evaluation/pyproject.toml \ + evaluation/coding/swebench_pro/tests/unit \ + evaluation/coding/work_continuity/tests/unit \ + evaluation/memory/longmemeval_v2/tests -m "not live" -q .PHONY: code-seekdb-test code-seekdb-test: ## Exercise native code indexing against a real embedded seekdb instance. diff --git a/README.md b/README.md index 5ba54fa7a5..47e9241e3f 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,8 @@ make test ``` See [CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow. +Coding and memory evaluations, performance benchmarks, and Skill regressions live under +[`evaluation/`](evaluation/README.md). ## Learn more diff --git a/README_CN.md b/README_CN.md index 7e927956d1..6b76b830f5 100644 --- a/README_CN.md +++ b/README_CN.md @@ -116,6 +116,7 @@ make test ``` 完整开发流程请阅读 [CONTRIBUTING.md](CONTRIBUTING.md)。 +编程能力评测、记忆质量评测、性能压测和 Skill 回归统一放在 [`evaluation/`](evaluation/README.md)，按用途选择对应目录。 ## 进一步了解 diff --git a/README_JP.md b/README_JP.md index dc0eccb45a..5e5d43a161 100644 --- a/README_JP.md +++ b/README_JP.md @@ -102,6 +102,7 @@ make test ``` 開発ワークフロー全体については [CONTRIBUTING.md](CONTRIBUTING.md) を参照してください。 +コーディング評価、メモリ品質評価、性能ベンチマーク、Skill 回帰テストは [`evaluation/`](evaluation/README.md) にまとめています。 ## さらに詳しく diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index c846288590..0000000000 --- a/benchmark/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# PowerContext benchmarks - -Benchmarks in this directory exercise public PowerContext behavior against fixed datasets. They are kept outside -`tests/` because they use real databases and model providers, create durable benchmark namespaces, and may take a -long time or incur inference cost. - -- [`locomo/`](locomo/README.md): conversation-memory retrieval and end-to-end question-answer accuracy. -- [`locomo_plus/`](locomo_plus/README.md): pinned LoCoMo-Plus factual and cognitive memory evaluation, with a - four-case smoke profile, a bundled ten-case dataset with complete histories, and an explicit full profile. -- [`memory_capacity/`](memory_capacity/README.md): manifest capacity, append cost, and tombstone compaction without model calls. diff --git a/docs/en/development/integration-guidance-evaluation.md b/docs/en/development/integration-guidance-evaluation.md index 9592de65e0..77137d4a7f 100644 --- a/docs/en/development/integration-guidance-evaluation.md +++ b/docs/en/development/integration-guidance-evaluation.md @@ -5,7 +5,7 @@ description: Measured routing, execution evidence, and remaining model limitatio # Agent guidance evaluation record -The complementary [skill-up regression suite](https://github.com/oceanbase/powercontext/blob/master/evaluation/skill-up/README.md) pins the packaged Claude Code +The complementary [skill-up regression suite](https://github.com/oceanbase/powercontext/blob/master/evaluation/skills/skill-up/README.md) pins the packaged Claude Code Skill and checks tool selection and authorization boundaries with rule-based assertions, controlled MCP replies, and a with/without-Skill comparison. It covers Claude Code + MCP only, with hooks disabled and permissions bypassed; it does not qualify real host approval, bounded recall, automatic Capture/Flush, persistence or memory quality. diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index aecc4f5da4..2521ddd436 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -95,7 +95,7 @@ PowerContext release number. | WorkBuddy integration | Its hooks and transport scripts carry User-Agent strings; there is no plugin manifest version to align with the Server. | | Skill Receiver | `RECEIVER_VERSION` in `src/powercontext/client/skill_receiver.py` supplies the default receiver identity version reported during enrollment, reconciliation, and receipts. It is independent of the Server version. | | Python integrations | `integrations/{bub,langchain,langgraph,opendal,pydantic-ai}/pyproject.toml` contain separate distribution versions. Dependency constraints express compatibility, not the current main release. | -| Evaluation and harness packages | `evaluation/pyproject.toml`, `evaluation/web/package.json`, and `e2e/bub/pyproject.toml` have their own package versions. | +| Evaluation and harness packages | `evaluation/pyproject.toml`, `evaluation/coding/swebench_pro/web/package.json`, and `e2e/bub/pyproject.toml` have their own package versions. | | Protocols and dependencies | Agent Plugins schema versions, persisted format versions, OpenAPI specification version, API paths, host minimum versions, and third-party dependencies change only with their own contracts. | Installing an Agent integration from the matching `powercontext-v…` repository tag selects the matching source diff --git a/docs/en/rfcs/1718_memory_capacity_contract.md b/docs/en/rfcs/1718_memory_capacity_contract.md index bf06c19acd..9ee0b1bc81 100644 --- a/docs/en/rfcs/1718_memory_capacity_contract.md +++ b/docs/en/rfcs/1718_memory_capacity_contract.md @@ -238,7 +238,7 @@ the byte ceiling even at the default identifier widths. The 5,000 / 10,000 / 4 MiB defaults remain configurable growth limits. Calibrate deployment budgets with isolated, representative measurements on the selected backend, including retained-history cost. Benchmark methodology and -measurement summaries belong in `benchmark/memory_capacity/README.md`; raw run results accompany acceptance evidence. +measurement summaries belong in `evaluation/performance/memory_capacity/README.md`; raw run results accompany acceptance evidence. ## Configuration @@ -501,8 +501,8 @@ projection work, so any change in their statement counts means the implementatio ### Scale benchmark -A new `benchmark/memory_capacity/` module, alongside the existing `locomo` benchmarks and outside `tests/` for the -reasons `benchmark/README.md` gives, records at entry counts 200, 1,000, and 5,000, and across a compaction cycle: +The `evaluation/performance/memory_capacity/` module sits outside `tests/` for the reasons +`evaluation/README.md` gives. It records at entry counts 200, 1,000, and 5,000, and across a compaction cycle: entry count, manifest bytes, database bytes, mean append latency, mean final-window append latency, projection row writes per append, and search recall behavior before and after compaction. @@ -524,7 +524,7 @@ Each step is independently reviewable and leaves the tree green. addition, `make api-generate`, `make contract-test`. 5. **Read bounding.** The `revisions()` cap and its capability error. 6. **Public read endpoint.** `POST /v1/memory/capacity`, OpenAPI, contract test. -7. **Benchmark.** `benchmark/memory_capacity/` and the recorded SQLite and OceanBase results. +7. **Benchmark.** `evaluation/performance/memory_capacity/` and the recorded SQLite and OceanBase results. Steps 1 through 3 alone close the "no observable ceiling" half of #1718 and are worth landing before compaction. diff --git a/docs/zh/development/integration-guidance-evaluation.md b/docs/zh/development/integration-guidance-evaluation.md index a2a45c1205..93d9fb0501 100644 --- a/docs/zh/development/integration-guidance-evaluation.md +++ b/docs/zh/development/integration-guidance-evaluation.md @@ -5,7 +5,7 @@ description: tracking issue 1450 D 的工具选择测量、执行证据及模型 # Agent 指引验证记录 -配套的 [skill-up 回归套件](https://github.com/oceanbase/powercontext/blob/master/evaluation/skill-up/README.md) 固定 Claude Code 打包 Skill 的版本， +配套的 [skill-up 回归套件](https://github.com/oceanbase/powercontext/blob/master/evaluation/skills/skill-up/README.md) 固定 Claude Code 打包 Skill 的版本， 使用规则断言、受控 MCP 响应及加载／不加载 Skill 的对比，检查工具选择与授权边界。 该套件仅覆盖 Claude Code + MCP；运行器关闭 hooks 并绕过权限提示，因此不证明真实宿主审批、 有界召回、自动 Capture/Flush、持久化或记忆质量。声明实测覆盖时，应同时保留真实模型的原始记录与输入快照。 diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index 21be90fa68..7462010019 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -91,7 +91,7 @@ OpenAPI 版本描述待发布的 API 契约，不覆盖 Hatch VCS。打 tag 前 | WorkBuddy 集成 | Hook 和传输脚本包含 User-Agent 字符串，没有需要与 Server 对齐的插件 manifest 版本。 | | Skill Receiver | `src/powercontext/client/skill_receiver.py` 中的 `RECEIVER_VERSION` 提供注册、协调及回执上报时默认使用的 Receiver 身份版本，独立于 Server 版本。 | | Python 集成包 | `integrations/{bub,langchain,langgraph,opendal,pydantic-ai}/pyproject.toml` 有各自的发行版本；依赖范围表示兼容性，不是当前主发布版本。 | -| 评测及 harness 包 | `evaluation/pyproject.toml`、`evaluation/web/package.json`、`e2e/bub/pyproject.toml` 有各自的包版本。 | +| 评测及 harness 包 | `evaluation/pyproject.toml`、`evaluation/coding/swebench_pro/web/package.json`、`e2e/bub/pyproject.toml` 有各自的包版本。 | | 协议及依赖 | Agent Plugins schema 版本、持久化格式版本、OpenAPI 规范版本、API 路径、宿主最低版本及第三方依赖仅随自身契约更新。 | 通过匹配的 `powercontext-v…` 仓库 tag 安装 Agent 集成，可以选中配套的源码修订，不要求每个插件 manifest 的 diff --git a/docs/zh/rfcs/1718_memory_capacity_contract.md b/docs/zh/rfcs/1718_memory_capacity_contract.md index 3e0a31ee75..21f8aa38bd 100644 --- a/docs/zh/rfcs/1718_memory_capacity_contract.md +++ b/docs/zh/rfcs/1718_memory_capacity_contract.md @@ -217,7 +217,7 @@ class MemoryCapacity(BaseModel): 标识符宽度，大批量写入也可能先触发字节上限。 保留可配置的 5,000 / 10,000 / 4 MiB 增长上限。部署预算应通过对应后端上隔离、具有代表性的测量校准，并计入保留 -历史的成本。基准方法与测量摘要集中在 `benchmark/memory_capacity/README.md`，原始运行结果随验收证据保存。 +历史的成本。基准方法与测量摘要集中在 `evaluation/performance/memory_capacity/README.md`，原始运行结果随验收证据保存。 ## 配置 @@ -457,8 +457,8 @@ PR #1709 的 `test_memory_append_projection_writes_do_not_grow_with_entry_histor ### 规模基准 -新增 `benchmark/memory_capacity/` 模块，与现有 `locomo` 基准并列、并按 `benchmark/README.md` 给出的理由置于 -`tests/` 之外，在 entry 数 200、1,000、5,000 以及一个完整压缩周期上记录： +`evaluation/performance/memory_capacity/` 模块按 `evaluation/README.md` 给出的理由置于 `tests/` 之外， +在 entry 数 200、1,000、5,000 以及一个完整压缩周期上记录： entry 数、manifest 字节数、数据库字节数、平均 append 延迟、末窗平均 append 延迟、每次 append 的投影行写入数， 以及压缩前后的搜索召回行为。 @@ -479,7 +479,7 @@ entry 数、manifest 字节数、数据库字节数、平均 append 延迟、末 `make contract-test`。 5. **读取约束。** `revisions()` 的上限及其 capability 错误。 6. **公开读取端点。** `POST /v1/memory/capacity`、OpenAPI、契约测试。 -7. **基准。** `benchmark/memory_capacity/` 以及记录的 SQLite 与 OceanBase 结果。 +7. **基准。** `evaluation/performance/memory_capacity/` 以及记录的 SQLite 与 OceanBase 结果。 仅第 1 至 3 步就能闭合 #1718 中"没有可观测上限"的那一半，值得在压缩之前先合入。 diff --git a/e2e/bub/tasks/locomo-multihop-football.yaml b/e2e/bub/tasks/locomo-multihop-football.yaml index 0e54cce07e..b17a811a85 100644 --- a/e2e/bub/tasks/locomo-multihop-football.yaml +++ b/e2e/bub/tasks/locomo-multihop-football.yaml @@ -19,7 +19,7 @@ categories: - sample - batch:locomo provenance: - source: benchmark/locomo/dataset/locomo10.json + source: evaluation/memory/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 selection: category-3-first-two-explicit-named-facts/v1 case_ids: diff --git a/e2e/bub/tasks/locomo-open-pastries.yaml b/e2e/bub/tasks/locomo-open-pastries.yaml index 1c48cab1a8..8e0561b1bc 100644 --- a/e2e/bub/tasks/locomo-open-pastries.yaml +++ b/e2e/bub/tasks/locomo-open-pastries.yaml @@ -19,7 +19,7 @@ categories: - sample - batch:locomo provenance: - source: benchmark/locomo/dataset/locomo10.json + source: evaluation/memory/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 selection: category-4-first-three-item-list/v1 case_ids: diff --git a/e2e/bub/tasks/locomo-support-group.yaml b/e2e/bub/tasks/locomo-support-group.yaml index 3ad7f23f0c..48ecdd0b03 100644 --- a/e2e/bub/tasks/locomo-support-group.yaml +++ b/e2e/bub/tasks/locomo-support-group.yaml @@ -19,7 +19,7 @@ categories: - sample - batch:locomo provenance: - source: benchmark/locomo/dataset/locomo10.json + source: evaluation/memory/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 selection: first-conversation-first-question/v1 case_ids: diff --git a/e2e/bub/tasks/locomo-temporal-banker.yaml b/e2e/bub/tasks/locomo-temporal-banker.yaml index 98b46dd690..9d9ff60023 100644 --- a/e2e/bub/tasks/locomo-temporal-banker.yaml +++ b/e2e/bub/tasks/locomo-temporal-banker.yaml @@ -19,7 +19,7 @@ categories: - sample - batch:locomo provenance: - source: benchmark/locomo/dataset/locomo10.json + source: evaluation/memory/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 selection: category-2-first-short-answer-after-conv-26/v1 case_ids: diff --git a/e2e/bub/tests/test_harbor_job_config.py b/e2e/bub/tests/test_harbor_job_config.py index 293e50a66e..298c263663 100644 --- a/e2e/bub/tests/test_harbor_job_config.py +++ b/e2e/bub/tests/test_harbor_job_config.py @@ -693,7 +693,7 @@ def test_agent_container_cannot_read_workload_answers(monkeypatch, tmp_path: Pat monkeypatch.setenv(plugin_host.server_url, "http://host-gateway:8000") task = load_tasks(_REPOSITORY / "e2e" / "bub" / manifest)[0] protected = [_REPOSITORY / "e2e" / "bub" / name for name in ("harbor-tasks", "paired-tasks", "tasks")] - protected.append(_REPOSITORY / "benchmark") + protected.append(_REPOSITORY / "evaluation") sources = [Path(mount["source"]) for mount in _config(task, tmp_path, host=host_adapter(host)).environment.mounts] diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 615e1aaeaf..4f91f46722 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1685,6 +1685,9 @@ requires-dist = [ { name = "tree-sitter-typescript", marker = "extra == 'code'", specifier = "==0.23.2" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, + { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, ] provides-extras = ["builtin", "cli", "client", "code", "seekdb", "server", "tracing-otlp"] diff --git a/evaluation/README.md b/evaluation/README.md index 3f085a84af..04518904d0 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -1,658 +1,105 @@ -# PowerContext evaluation console +# PowerContext evaluation -For Claude Code + MCP Skill instruction regressions, see the separate [skill-up guidance suite](skill-up/README.md). -It grades tool routing and authorization boundaries against a pinned Skill with controlled MCP replies and a no-Skill -baseline. Its scores complement the integration-guidance record and must not be combined with SWE-bench Pro task -outcomes or LoCoMo memory-quality measurements. +Evaluation suites are grouped by what they measure. Each suite owns its dataset inputs, execution, grading, +reports, tests, command entry point, and any console or deployment tools it needs. This directory provides the shared +Python environment and build configuration. -This directory contains a self-progressing SWE-bench Pro evaluation service. A batch can run the paired OFF/ON -experiment or only one Arm to reduce cost and latency. The web process owns the HTTP API and report UI; the worker -owns task execution, retries, resource cleanup, and durable recovery. - -The service is intentionally deployment-neutral. Host names, operators, filesystem roots, optional proxy endpoints, Docker -network ranges, credentials, and service locations are supplied by the operator. The repository does not contain a -production environment file or a ready-to-install host-specific systemd unit. - -## Runtime requirements - -- Linux, Git, Python 3.11 or newer, `uv`, and Node.js/npm -- Docker with permission to pull images, create isolated bridge networks, and run evaluation containers -- Codex CLI and a valid Codex `auth.json` -- [`regctl`](https://github.com/regclient/regclient) for importing task images that are not already present -- enough disk space and inodes for the selected parallelism - -TokensFlow telemetry and the credential-free loopback proxy are optional integrations. Both are disabled by default; -the normal open-source path uses native container egress and starts neither a proxy relay nor a TokensFlow -daemon/finalizer. - -Install and validate from the repository root: - -```bash -uv sync --project evaluation --frozen -uv run --project evaluation pytest -c evaluation/pyproject.toml evaluation/tests -m "not live" -q -uv run --project evaluation ruff check evaluation -uv run --project evaluation ruff format --check evaluation -uv run --directory evaluation ty check src -``` - -Build and test the web UI: - -```bash -cd evaluation/web -npm ci -npm test -- --run -npm run build -``` - -## Quick start - -The commands below prepare a single-host development deployment. They intentionally bind the unauthenticated console -to `127.0.0.1`. Put an authenticating reverse proxy in front of the console before allowing access from another -machine; do not expose the HTTP service directly to an untrusted network. - -### 1. Prepare the evaluation root - -Choose an absolute writable directory and create the layout expected by the default configuration. The Web and -Worker processes must be able to read this repository and the protected configuration; the Worker also needs Docker -access and write access to the evaluation root. - -```bash -export REPOSITORY_ROOT="$(pwd -P)" -export EVALUATION_ROOT=/srv/powercontext-eval - -install -d -m 0700 \ - "$EVALUATION_ROOT/bin" \ - "$EVALUATION_ROOT/cache" \ - "$EVALUATION_ROOT/codex-home" \ - "$EVALUATION_ROOT/config" \ - "$EVALUATION_ROOT/source" \ - "$EVALUATION_ROOT/venvs" -``` - -### 2. Install the pinned SWE-bench Pro harness - -The runner accepts exactly harness commit `ca10a60a5fcae51e6948ffe1485d4153d421e6c5`. That commit contains the pinned -731-row dataset; the console verifies its schema, order, row count, and SHA-256 before admitting a batch. - -```bash -git clone https://github.com/scaleapi/SWE-bench_Pro-os.git \ - "$EVALUATION_ROOT/cache/swebench-pro.git" -git -C "$EVALUATION_ROOT/cache/swebench-pro.git" checkout --detach \ - ca10a60a5fcae51e6948ffe1485d4153d421e6c5 -test "$(git -C "$EVALUATION_ROOT/cache/swebench-pro.git" rev-parse HEAD)" = \ - ca10a60a5fcae51e6948ffe1485d4153d421e6c5 -test "$(sha256sum "$EVALUATION_ROOT/cache/swebench-pro.git/helper_code/sweap_eval_full_v2.jsonl" | cut -d' ' -f1)" = \ - b5b2462bfbf5aeb2cb7ba7d215778a1768b85f9d7ad7f748546c7f80a0ad1510 - -uv venv --python 3.11 "$EVALUATION_ROOT/venvs/swebench-pro-ca10a60" -uv pip install \ - --python "$EVALUATION_ROOT/venvs/swebench-pro-ca10a60/bin/python" \ - -r "$EVALUATION_ROOT/cache/swebench-pro.git/requirements.txt" -``` - -### 3. Prepare source, tools, credentials, and frontend - -The source is a bare mirror so every submitted branch, tag, or commit resolves to immutable Git data rather than a -mutable developer working tree. Refresh or replace this mirror deliberately when evaluating newer PowerContext -commits. - -```bash -git clone --mirror "$REPOSITORY_ROOT" "$EVALUATION_ROOT/source/powercontext.git" - -install -m 0755 "$(command -v codex)" "$EVALUATION_ROOT/bin/codex" -install -m 0755 "$(command -v uv)" "$EVALUATION_ROOT/bin/uv" -install -m 0755 "$(command -v regctl)" "$EVALUATION_ROOT/bin/regctl" - -install -m 0600 "${CODEX_HOME:-$HOME/.codex}/auth.json" \ - "$EVALUATION_ROOT/codex-home/auth.json" -``` - -If the Codex account uses a custom provider, also copy its configuration without printing it: - -```bash -install -m 0600 "${CODEX_HOME:-$HOME/.codex}/config.toml" \ - "$EVALUATION_ROOT/codex-home/config.toml" -``` - -Build the frontend from the repository checkout that will run the services: - -```bash -cd "$REPOSITORY_ROOT/evaluation/web" -npm ci -npm test -- --run -npm run build -cd "$REPOSITORY_ROOT" -``` - -### 4. Create and validate the runtime environment - -There is one platform runtime configuration file. Copy the example, edit every path that differs from the layout -above, and keep it private. In particular, set `POWERCONTEXT_EVAL_FRONTEND_DIST` to the absolute -`$REPOSITORY_ROOT/evaluation/web/dist` path. Uncomment `POWERCONTEXT_EVAL_CODEX_CONFIG` only if the file was installed -in the previous step. - -```bash -install -m 0600 evaluation/deploy/powercontext-eval.env.example \ - "$EVALUATION_ROOT/config/evaluation-console.env" -${EDITOR:-vi} "$EVALUATION_ROOT/config/evaluation-console.env" - -set -a -. "$EVALUATION_ROOT/config/evaluation-console.env" -set +a - -uv run --project evaluation python -c \ - 'import os; from powercontext_eval.web.config import WebConfig; WebConfig.from_environment(os.environ); print("configuration valid")' -docker info >/dev/null -``` - -Use `POWERCONTEXT_EVAL_USAGE_MODE=api_key` for API-key credentials. That mode skips subscription-usage probing and -treats quota admission as sufficient; filesystem and Docker admission remain active. Keep `subscription` for a -Codex subscription whose CLI exposes the supported usage probe. - -### 5. Start Web and Worker - -From the repository root, load the same protected environment in two terminals: - -```bash -# Terminal 1 -set -a; . "$EVALUATION_ROOT/config/evaluation-console.env"; set +a -uv run --project evaluation powercontext-eval web -``` - -```bash -# Terminal 2 -set -a; . "$EVALUATION_ROOT/config/evaluation-console.env"; set +a -uv run --project evaluation powercontext-eval worker -``` - -Verify the control plane before submitting work: - -```bash -curl --fail --silent http://127.0.0.1:8787/api/health -``` - -Open `http://127.0.0.1:8787/`, choose `swebench-pro-stability-v1` and an `OFF + ON`, `ON only`, or `OFF only` run. -This 24-task set is the deployment regression suite; use it before the 731-task `swebench-pro-public-v2` set. A -completed aggregate report can freeze any executed Arm as an immutable baseline. Reports may select multiple -compatible baselines for historical comparison without rerunning evaluation, while the baseline library lists the -newest saved baselines first. The same bounded batch can be created from the CLI after Web and Worker are healthy: - -```bash -uv run --project evaluation powercontext-eval swebench-pro create-batch \ - --console-url http://127.0.0.1:8787 \ - --task-set swebench-pro-stability-v1 \ - --treatment-mode on_only \ - --powercontext-ref latest \ - --idempotency-key "stability-$(date -u +%Y%m%dT%H%M%SZ)" -``` - -## LongMemEval-V2 smoke workload - -The LongMemEval-V2 commands run a fixed ten-question smoke subset end to end: preflight -validation of the pinned inputs, real PowerContext HTTP retrieval, prompt preparation, an -optional Reader, upstream scoring, offline score replay, and a one-command orchestration -(`run-smoke`) that writes one run directory with a unified report. A full-tier run has not been -executed; its recorded configuration lives in -[docs/longmemeval-v2-full-run.md](docs/longmemeval-v2-full-run.md). - -Prepare a detached upstream checkout at the pinned harness commit and download -the matching LongMemEval-V2 data root outside this repository. The checked-in -small-tier lock and smoke manifest fix the data revision, three core data-file -hashes, ten source-ordered questions, all five published abilities, and both -published domains. The dataset lock has this shape: - -```json -{ - "schema": "powercontext.longmemeval-v2-dataset-lock.v1", - "upstream": { - "repository": "https://github.com/xiaowu0162/LongMemEval-V2", - "harness_commit": "2cc8c540bdb87fe6761629b585e727e1c4704520" - }, - "dataset_revision": "DATASET_REVISION", - "tier": "small", - "files": { - "questions.jsonl": "SHA256", - "trajectories.jsonl": "SHA256", - "haystacks/lme_v2_small.json": "SHA256" - } -} -``` - -The smoke manifest fixes question IDs, their upstream source order, and coverage -of the published ability categories. Run the preflight against a new output -directory: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 smoke \ - --harness-root /path/to/LongMemEval-V2 \ - --data-root /path/to/longmemeval-v2-data \ - --dataset-lock evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json \ - --smoke-manifest evaluation/locks/longmemeval-v2-small-v1.smoke.json \ - --output-dir /path/to/new-smoke-artifacts -``` - -The command refuses a harness checkout at a different commit, mismatched input -hashes, invalid or incomplete smoke coverage, and an existing output directory. -It writes `manifest.json` and `subset.json`, both labelled as a smoke subset. - -### PowerContext Memory adapter bootstrap - -Run the bootstrap with a Python environment containing the pinned upstream -harness dependencies. It validates the clean harness commit, registers -`memory_type: powercontext` in memory, and then forwards all remaining arguments -to the unchanged upstream harness: - -```bash -python evaluation/scripts/run_longmemeval_v2_harness.py \ - --harness-root /path/to/LongMemEval-V2 \ - -- -``` - -The adapter memory configuration requires a dedicated evaluation Scope and -audit artifact path. Credentials are resolved only from `token_env` at runtime: - -```json -{ - "memory_type": "powercontext", - "memory_params": { - "scope_id": "longmemeval-v2-smoke-run-id", - "audit_path": "/path/to/run/context/powercontext-memory.jsonl", - "base_url": "http://127.0.0.1:8765", - "token_env": "POWERCONTEXT_TOKEN", - "search_mode": "auto", - "search_limit": 10 - } -} -``` - -Each trajectory chunk is captured through the public Content Source endpoint -and one deterministic bounded projection per trajectory is explicitly -remembered through the public Memory endpoint. The projection keeps goal, -outcome, URL, action, thought, and bounded observation evidence without calling -a model. The adapter audit correlates the returned Source references and Memory -citation; it does not claim that this correlation is native Memory lineage. -Queries use the public Memory search endpoint and return upstream-compatible -text context items. Query images are neither read nor sent because the current -Memory search contract is text only. - -With a ready PowerContext Server, run the fixed ten-question retrieval-only -smoke workload without a Reader, Judge, model credential, or scoring step: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 retrieval-smoke \ - --harness-root /path/to/LongMemEval-V2 \ - --data-root /path/to/longmemeval-v2-data \ - --dataset-lock evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json \ - --smoke-manifest evaluation/locks/longmemeval-v2-small-v1.smoke.json \ - --base-url http://127.0.0.1:8000 \ - --run-id retrieval-smoke-001 \ - --powercontext-revision POWERCONTEXT_GIT_SHA \ - --integration-revision INTEGRATION_GIT_SHA \ - --output-dir /path/to/new-retrieval-smoke-artifacts -``` - -The runner verifies the locked input digests while streaming -`trajectories.jsonl`, retains only one trajectory object at a time, and creates -one isolated child Scope for each distinct ordered haystack. Identical -haystacks reuse their ingestion, while different haystacks cannot retrieve each -other's Memory. It writes `retrieval-manifest.json`, -`retrieval-results.jsonl`, `adapter-audit.jsonl`, `failures.jsonl`, and -`summary.json` beside the preflight `manifest.json` and `subset.json`. Accuracy, -Reader, and Judge fields remain null because retrieval-only output is not a -benchmark score. - -### Experiment arms - -Retrieval behaviour is selected through a registered experiment arm, not a free-form search mode: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 retrieval-smoke \ - ... \ - --experiment-arm current-memory-hybrid-v1 -``` - -An arm is a frozen configuration identity: same questions and the same downstream token -budget, with only the declared retrieval/projection knobs allowed to differ. Five arms are -currently registered: - -| Arm ID | Strategy | Notes | +| Directory | Purpose | Suites | | --- | --- | --- | -| `current-memory-fts-v1` | Memory search (`fts`) | Default; identical to the previous `--search-mode fts` behaviour. | -| `current-memory-hybrid-v1` | Memory search (`hybrid`) | Requires a Server whose `/v1/capabilities` advertises `hybrid`. | -| `query-time-compact-v1` | PreparedContext (8,000 bytes) | Uses public `/v1/context/prepare`; does not alter ingestion or persist a new index schema. | -| `write-time-l0-l1-v1` | Memory search (`fts`) | Writes deterministic bounded L0 index and L1 summary entries through public `remember`; no schema fields are added. | -| `task-lensed-selection-v1` | Memory search (`fts`) | Projects the question into a deterministic keyword lens; never reads question type, gold answer, or Judge data. | - -Unregistered arm IDs (currently the unsupported temporal-filter arm) are rejected -before any work runs instead of silently falling back to FTS. Before ingesting anything, the -runner checks `/v1/capabilities`: a Server without the arm's search mode or PreparedContext -schema fails as a capability error. Every explicitly requested search mode must match the -executed mode the Server reports (only `auto` accepts the Server's own choice) — a mismatch is -recorded as an integrity failure rather than a benchmark result. Every manifest, summary, and -unified report records the full arm block, `adapter-audit.jsonl` records the requested and -actual strategy/mode per query, and `ensure_comparable_experiment_runs` refuses to compare two -runs whose dataset lock, question manifest, harness commit, processor, context budget, search -limit, Reader/Judge configuration, or revisions differ beyond the arm. - -Prepare bounded, replayable Reader inputs without calling a Reader. This command -uses the pinned upstream harness to count and truncate Memory context, and -requires an immutable Hugging Face processor revision rather than resolving the -processor from a moving branch: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 prepare-smoke \ - --retrieval-dir /path/to/retrieval-smoke-artifacts \ - --harness-root /path/to/LongMemEval-V2 \ - --harness-python /path/to/longmemeval-harness-python \ - --processor-revision PROCESSOR_GIT_SHA \ - --memory-context-max-tokens 200000 \ - --output-dir /path/to/new-prepared-prompt-artifacts -``` - -It writes `prepare-manifest.json`, `prepared-prompts.jsonl`, -`prepare-failures.jsonl`, and `prepare-summary.json`. Each prepared prompt -records the original and bounded Context token counts, the bounded Context -bytes, final system/user messages, a prompt SHA-256, and prepare latency. This -is still a no-model stage: Reader, Judge, and accuracy remain absent. - -Run an Anthropic-compatible Reader or DeepSeek's direct OpenAI-compatible API -over prepared prompts. Credentials are read only from the named process -environment variable and are never written to the run artifacts: - -```bash -export DEEPSEEK_API_KEY=YOUR_KEY -uv run --project evaluation powercontext-eval longmemeval-v2 reader-smoke \ - --prepared-dir /path/to/prepared-prompt-artifacts \ - --provider deepseek-openai \ - --model deepseek-flash \ - --token-env DEEPSEEK_API_KEY \ - --max-tokens 512 \ - --temperature 0 \ - --output-dir /path/to/new-reader-artifacts -``` - -The Reader writes `reader-manifest.json`, `reader-outputs.jsonl`, -`reader-failures.jsonl`, and `reader-summary.json`. It records provider usage -returned by the API but does not score answers. Keep OpenTelemetry variables -out of this command when prompt telemetry is not approved. - -Score Reader outputs with the pinned deterministic metric functions. The -abstention and gotchas metric types require an LLM Judge and send the question, -reference answer, full Reader response, and parsed answer to the configured -Judge provider: - -```bash -export DEEPSEEK_API_KEY=YOUR_KEY -uv run --project evaluation powercontext-eval longmemeval-v2 score-smoke \ - --reader-dir /path/to/reader-artifacts \ - --data-root /path/to/longmemeval-v2-data \ - --dataset-lock evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json \ - --smoke-manifest evaluation/locks/longmemeval-v2-small-v1.smoke.json \ - --harness-root /path/to/LongMemEval-V2 \ - --judge-model deepseek-flash \ - --judge-token-env DEEPSEEK_API_KEY \ - --judge-max-tokens 256 \ - --judge-temperature 0 \ - --output-dir /path/to/new-score-artifacts -``` - -The score run writes `per-question.jsonl`, `judge-outputs.jsonl`, -`score-failures.jsonl`, and `score-summary.json`. It also writes -`scoring-inputs.local.jsonl`, which contains reference answers for local replay -and must remain outside Git, shared reports, and telemetry. - -Replay the saved deterministic and Judge decisions without a Reader, Judge -provider, token, or network request: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 replay-score \ - --score-dir /path/to/score-artifacts \ - --harness-root /path/to/LongMemEval-V2 \ - --output-dir /path/to/new-score-replay-artifacts -``` - -The replay records source digests and writes `replay-manifest.json`, -`replay-per-question.jsonl`, `replay-failures.jsonl`, and -`replay-summary.json`. It reuses only saved Judge labels for LLM-scored cases. - -### One-command smoke run - -`run-smoke` chains every stage above into one fail-closed run directory. With a -ready PowerContext Server and no model credentials, run the model-free mode -first. The retrieval arm defaults to `current-memory-fts-v1` and can be selected -explicitly with `--experiment-arm`: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 run-smoke \ - --data-root /path/to/longmemeval-v2-data \ - --dataset-lock evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json \ - --smoke-manifest evaluation/locks/longmemeval-v2-small-v1.smoke.json \ - --harness-root /path/to/LongMemEval-V2 \ - --harness-python /path/to/longmemeval-harness-python \ - --processor-revision PROCESSOR_GIT_SHA \ - --powercontext-revision POWERCONTEXT_GIT_SHA \ - --integration-revision INTEGRATION_GIT_SHA \ - --powercontext-base-url http://127.0.0.1:18765 \ - --experiment-arm current-memory-fts-v1 \ - --skip-reader \ - --output-dir /path/to/new-run-artifacts -``` - -The full mode drops `--skip-reader`, requires `DEEPSEEK_API_KEY` in the environment, and also -runs the Reader, Judge scoring, and score replay. `--skip-score` keeps the Reader but skips -Judge scoring and replay. Every mode writes the same layout: +| [`coding/`](coding/README.md) | Coding-task completion and continuation quality | [SWE-bench Pro](coding/swebench_pro/README.md), [Work continuity](coding/work_continuity/README.md) | +| [`memory/`](memory/README.md) | Memory retrieval and answer quality | [LoCoMo](memory/locomo/README.md), [LoCoMo-Plus](memory/locomo_plus/README.md), [LongMemEval-V2](memory/longmemeval_v2/README.md) | +| [`performance/`](performance/README.md) | Capacity, storage growth, latency, and compaction cost | [Memory capacity](performance/memory_capacity/README.md) | +| [`skills/`](skills/README.md) | Agent instruction routing and authorization regressions | [Claude Code skill-up](skills/skill-up/README.md) | ```text -/ - run-manifest.json # exclusive-create; inputs, revisions, providers (no secret values) - run-summary.json # completed/skipped/failed phases; accuracy only when scored - failures.jsonl # one row per failed phase, with an error class - report.json # unified machine-readable summary - report.md # human summary with the fixed boundary banner - 01-inputs/ 02-retrieval/ 03-prepare/ 04-reader/ 05-score/ 06-replay/ -``` - -The command exits non-zero unless the run completed: `--skip-reader` and `--skip-score` runs end -as `partial`, and a failed stage ends as `failed` while keeping the earlier stage artifacts. -Recorded failure summaries are redacted against the configured token values, including -`POWERCONTEXT_TOKEN` in model-free mode. Reference answers are read only by the score stage, -which writes the local replay artifact, and by the replay stage reading that artifact; the -adapter, retrieval, prepare, and reader stages never read them. - -### Cost reporting with an explicit price policy - -Token usage is always recorded from the provider's own response. Model cost is reported only when -an explicit price policy is passed; prices are never hardcoded, and an unconfigured cost stays -`null` with a reason instead of `0`: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 run-smoke \ - ... \ - --price-policy '{"provider":"deepseek-openai","model":"deepseek-flash","currency":"USD","input_cache_hit_price_per_million":0.006,"input_cache_miss_price_per_million":0.3,"output_price_per_million":1.2,"price_policy_revision":"deepseek-public-list-2026-09"}' \ - --judge-price-policy '{"provider":"deepseek-openai","model":"deepseek-flash","currency":"USD","input_cache_hit_price_per_million":0.006,"input_cache_miss_price_per_million":0.3,"output_price_per_million":1.2,"price_policy_revision":"deepseek-public-list-2026-09"}' \ - --output-dir /path/to/new-run-artifacts -``` - -`run-smoke` takes a separate policy per model role because the Reader and the Judge may run -different models; `reader-smoke` takes `--price-policy` and `score-smoke` takes -`--judge-price-policy`. A policy must contain exactly `provider`, `model`, `currency`, -`input_cache_hit_price_per_million`, `input_cache_miss_price_per_million`, -`output_price_per_million`, and `price_policy_revision`. Every field is required, prices must be -finite and non-negative, and `currency` must be `USD` because the recorded amount fields are named -`*_usd`. A policy is applied only when both its `provider` and its `model` match the configured -stage; otherwise the stage records `null` plus the mismatch reason instead of borrowing an -unrelated price. - -Cached and uncached input are priced separately. DeepSeek reports -`prompt_cache_hit_tokens` and `prompt_cache_miss_tokens` alongside `prompt_tokens`, and the -cache-hit rate is roughly fifty times cheaper than the cache-miss rate, so the reader and judge -records keep that split instead of collapsing it into one blended input total. A response that -reports no split cannot be priced at either rate, so its stage records `null` with that reason -rather than an estimate. - -Reader cost comes from the Reader's reported usage, Judge cost from the Judge's reported usage, -and each stage summary and manifest records the policy identity it was priced under. The unified -report adds `usage.reader_cost`, `usage.judge_cost`, `usage.estimated_cost_usd` (the sum, only -when every model stage that actually ran was priced in USD under one policy revision) and -`usage.estimated_cost_note` explaining how the total was computed or why it stays `null`. The -report decides which stages ran from the run manifest `modes` plus the stage summaries, so a -missing, unpriced, mismatched, or differently-revisioned stage cost keeps the total `null` -instead of silently summing the stages that happen to be present. Ingestion is model-free, so the -report records `usage.ingestion_cost` as zero tokens, `cost_usd: 0.0`, and that reason. Without a -policy the report keeps the `null` behavior and explains it. - -Regenerate or inspect a report for any saved run without a model, provider, or server: - -```bash -uv run --project evaluation powercontext-eval longmemeval-v2 report \ - --run-dir /path/to/saved-run \ - --output-dir /path/to/new-report-artifacts -``` - -### LongMemEval-V2 full run (not executed) - -A full-tier run has not been executed and is not approved, and its results must never be -confused with the smoke subset. The recorded prerequisites, configuration template, -cost-estimation and approval gates, output isolation, result labels, and failure-recovery -procedure live in [docs/longmemeval-v2-full-run.md](docs/longmemeval-v2-full-run.md); the -secret-free environment template lives in -[deploy/longmemeval-v2-run-config.example.env](deploy/longmemeval-v2-run-config.example.env). A -full-tier dataset lock and question manifest do not exist yet and must be created and reviewed -before any full run. - -### What the LongMemEval-V2 workload evaluates - -The smoke workload and any future full run measure one pipeline over fixed LongMemEval-V2 -trajectories: whether the PowerContext Memory adapter retrieves citable evidence through public -interfaces under fixed upstream data, fixed questions, and a fixed context budget; the answer -accuracy, latency, context size, failures, and abstention of the configured Reader and Judge; and -whether saved outputs replay the same deterministic scoring inputs without a model. - -They do not evaluate Handoff, cross-host recovery, normal Runtime persistence, or Work -Continuity. A ten-question smoke subset cannot be extrapolated to full benchmark performance, -general model capability, or product leadership, and it does not replace LoCoMo, SWE-bench Pro, -or real user-task acceptance. Scores must never be improved by rewriting gold prompts, reference -answers, or the Memory schema. Every smoke artifact is labelled `smoke-subset`; none of them is -a complete benchmark result. - -The implemented scope and local validation evidence are summarized in -[docs/longmemeval-v2-smoke-delivery.md](docs/longmemeval-v2-smoke-delivery.md). - -## Configuration files - -Only the environment file configures the evaluation platform. The other files either belong to Codex or are -deployment templates: - -| File | Required | Purpose | -| --- | --- | --- | -| `evaluation-console.env` | yes | Web, Worker, storage, benchmark, retries, parallelism, and optional integrations | -| `auth.json` | yes | Codex credentials; referenced by path and copied into isolated task homes | -| `config.toml` | no | Custom Codex provider/model configuration | -| `powercontext-eval.env.example` | no | Safe template; never loaded directly in production | -| `*.service.in` | no | Unrendered systemd templates; they are not additional application configuration | - -## Configuration reference - -Copy `evaluation/deploy/powercontext-eval.env.example` to a protected location, replace every applicable example -value, and set mode `0600`. Only `POWERCONTEXT_EVAL_ROOT` is required by the configuration loader; the runtime tools, -dataset, credentials, and source checkout must exist at their derived or explicitly configured paths before work is -claimed. - -Derived paths are rooted below `POWERCONTEXT_EVAL_ROOT` unless explicitly overridden. The settings fall into these -groups: - -| Group | Variables | -| --- | --- | -| Service and storage | `ROOT`, `HOST`, `PORT`, `DATABASE_PATH`, `RUN_ROOT`, `FRONTEND_DIST` | -| Benchmark inputs | `POWERCONTEXT_SOURCE`, `HARNESS_ROOT`, `HARNESS_PYTHON`, `DATASET_PATH` | -| Executables and credentials | `CODEX_BINARY`, `CODEX_MODELS`, `CODEX_CONFIG`, `UV_BINARY`, `REGISTRY_BINARY`, `AUTH_JSON` | -| Scheduling and recovery | `TASK_PARALLELISM`, `MAX_ATTEMPTS`, `LEASE_SECONDS`, `POLL_SECONDS`, `WORKSPACE_RECLAIM_INTERVAL_SECONDS` | -| Resource admission | `FILESYSTEM_MIN_FREE_BYTES`, `FILESYSTEM_MIN_FREE_INODES`, `DOCKER_NETWORK_POOL` | -| Usage admission | `USAGE_MODE`, `USAGE_PAUSE_PERCENT`, `USAGE_PROBE_SECONDS`, `USAGE_PROBE_TIMEOUT_SECONDS`, `USAGE_SNAPSHOT_MAX_AGE_SECONDS` | -| Optional integrations | `TOKENSFLOW_ENABLED`, TokenFlow settings, `PROXY_URL`, `EXTRA_NO_PROXY_HOSTS` | - -Every name in the table has the `POWERCONTEXT_EVAL_` prefix. Important behavior-changing settings are: - -- `POWERCONTEXT_EVAL_TASK_PARALLELISM` controls concurrent OFF/ON task pairs and defaults to `1`. -- `POWERCONTEXT_EVAL_MAX_ATTEMPTS` bounds durable per-task retries and defaults to `5`. -- `POWERCONTEXT_EVAL_USAGE_MODE=api_key` disables subscription quota probing; API-key mode is treated as having - sufficient quota while still enforcing resource admission. -- `POWERCONTEXT_EVAL_TOKENSFLOW_ENABLED=true` explicitly enables internal TokensFlow telemetry and then requires its - binary, user home, and egress network settings. When false or absent, no TokensFlow profile, daemon, finalizer, - mount, or retained audit artifact is created. -- `POWERCONTEXT_EVAL_PROXY_URL` explicitly enables the loopback proxy relay. When absent, task networks use native - egress and proxy variables are cleared from managed task processes. -- `POWERCONTEXT_EVAL_EXTRA_NO_PROXY_HOSTS` is a comma-separated, validated list appended to loopback-only - `NO_PROXY`. It is valid only with the evaluation proxy and is recorded in each retained report. -- `POWERCONTEXT_EVAL_DOCKER_NETWORK_POOL` selects the private IPv4 pool used for isolated /28 task networks. It must - provide at least 32 subnets and is recorded in each retained report. - -Credential files are referenced by path and copied into isolated task homes. Never place credential values in the -environment example, command line, logs, reports, or repository. - -## Standalone runner - -The systemd templates and local commands must use the repository root as their working directory. A packaging -workflow that starts the service outside a Git checkout must provide the exact 40-character source revision in -`POWERCONTEXT_EVAL_BUILD_REVISION`. - -The standalone runner defaults to native networking with TokensFlow disabled. Other paths derive from the supplied -root and remain individually overridable: - -```bash -uv run --project evaluation powercontext-eval swebench-pro run \ - --root /srv/powercontext-eval \ - --instance-id instance_owner__repository-revision -``` - -An internal deployment can explicitly add `--proxy-url http://127.0.0.1:8081 --tokensflow ---tokensflow-egress-network bridge`; TokensFlow binary and profile paths derive from the root unless overridden. - -## systemd templates - -`evaluation/deploy/*.service.in` are templates, not installable units. Render all placeholders into a staging -directory, inspect the result, and run `systemd-analyze verify` before installation. Required placeholders are: - -- `@EVALUATION_ROOT@`: absolute writable evaluation root -- `@REPOSITORY_ROOT@`: absolute deployment checkout -- `@UV_BINARY@`: absolute `uv` executable -- `@EVALUATION_USER@` and `@EVALUATION_GROUP@`: unprivileged web identity -- `@EVALUATION_WORKER_USER@` and `@EVALUATION_WORKER_GROUP@`: worker identity with the required Docker access - -The web template is sandboxed to the evaluation root. The worker template intentionally does not claim a generic -sandbox because Docker access and host cleanup policy vary by deployment. Do not grant the web role Docker access. - -## Control and recovery model - -Only operator pause or cancel changes durable control intent. Task failures do not pause healthy peers. Retriable -task failures enter durable backoff and another free worker slot may claim the next eligible task. The default five -attempt budget uses 30, 120, 300, and 600 second delays. An exhausted or non-retriable task becomes a retained -failure while the rest of the batch continues. - -The worker performs startup-only orphan recovery under a process lock. A normal claim never steals another slot's -lease. Loss of attempt ownership cancels child processes, after which the lifecycle cleaner removes exact -attempt-owned containers, networks, and scratch workspaces. Retained `/runs` reports and private incident evidence -remain available for audit. When TokensFlow is explicitly enabled, finalizer-owned containers are excluded until -their durable finalization job is terminal. - -Admission pressure is transient: disk, inode, Docker, or usage probe pressure prevents new claims without changing -batch intent. Workspace reclamation runs continuously and removes only terminal attempts after report publication; -it never deletes running, queued, retryable, or finalizer-owned state. - -## Operational acceptance - -Before resuming a real batch: - -1. verify the exact source revision and a clean checkout; -2. validate the protected environment file without displaying it; -3. run backend, lint, format, type, frontend test, and frontend build gates; -4. verify rendered systemd units and confirm web/worker health; -5. run a small OFF/ON batch and inspect Gold, OFF, ON, official evaluator, retry, cleanup, and report evidence; -6. confirm `active_task_pairs` never exceeds configured parallelism; -7. confirm unrelated host services are neither dependencies nor restart targets; -8. perform a secret scan over retained artifacts and logs; -9. document the exact rollback revision before increasing parallelism. - -Rollback is a source checkout and service restart operation. Do not delete the database, `/runs`, incident evidence, -or task workspaces needed by queued and retryable attempts. Do not use global Docker prune as an evaluation cleanup -mechanism. +evaluation/ +├── pyproject.toml # Shared powercontext-eval environment and build +├── uv.lock +├── coding/ +│ ├── swebench_pro/ +│ │ ├── src/powercontext_eval_swebench_pro/ +│ │ ├── tests/ +│ │ ├── web/ +│ │ ├── deploy/ +│ │ └── docs/ +│ └── work_continuity/ +│ ├── src/powercontext_eval_work_continuity/ +│ ├── tests/ +│ ├── locks/ +│ └── docs/ +├── memory/ +│ ├── locomo/ +│ ├── locomo_plus/ +│ └── longmemeval_v2/ +│ ├── src/powercontext_eval_longmemeval_v2/ +│ ├── locks/ +│ ├── config/ +│ ├── docs/ +│ ├── scripts/ +│ └── tests/ +├── performance/ +│ └── memory_capacity/ +└── skills/ + └── skill-up/ +``` + +## Run a suite + +Run these commands from the repository root. LoCoMo, LoCoMo-Plus, and performance modules use the root project's +Python dependencies. SWE-bench Pro, Work continuity, and LongMemEval-V2 share this directory's locked environment and +expose independent `swebench-pro`, `work-continuity`, and `longmemeval-v2` commands. Skill-up has its own validation +dependencies and pinned CLI. + +```bash +# Install the shared environment, then choose a suite command. +uv sync --project evaluation --locked +uv run --project evaluation swebench-pro --help +uv run --project evaluation work-continuity --help +uv run --project evaluation longmemeval-v2 --help + +# Memory quality: inspect a bundled, provider-free smoke plan. +uv run python -m evaluation.memory.locomo --help +uv run python -m evaluation.memory.locomo_plus run \ + --dataset-file evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json \ + --dry-run + +# Capacity and performance: inspect options before choosing a backend and scale. +uv run python -m evaluation.performance.memory_capacity --help + +# Skill regression: validate the pinned Skill and controlled fixtures locally. +uv run python evaluation/skills/skill-up/sync_skill.py --check +uv run python evaluation/skills/skill-up/validate_suite.py +``` + +The [SWE-bench Pro console guide](coding/swebench_pro/docs/console.md) covers its Web and worker deployment. +LongMemEval-V2 uses its own suite CLI. Real dataset runs may call model providers, use databases or Docker, +create durable namespaces, and incur inference cost. Help, fixture validation, and dry-run plans do not establish +real-model or external-backend performance. + +## Development and results + +Run the installed suite checks together: + +```bash +uv run --directory evaluation pytest -m "not live" +uv run --project evaluation ruff check evaluation/coding evaluation/memory/longmemeval_v2 +uv run --project evaluation ruff format --check evaluation/coding evaluation/memory/longmemeval_v2 +uv run --directory evaluation ty check +uv build --project evaluation +``` + +The `powercontext-eval` distribution contains three packages: `powercontext_eval_swebench_pro`, +`powercontext_eval_work_continuity`, and `powercontext_eval_longmemeval_v2`. Editable installs and wheels expose the +independent `swebench-pro`, `work-continuity`, and `longmemeval-v2` commands and their suite modules. +LoCoMo behavior tests remain under [`tests/`](../tests/), including [`tests/evaluation/`](../tests/evaluation/). +Skill-up keeps its own validation tests. The [Bub replay harness](../e2e/bub/README.md) and +[product E2E tests](../tests/e2e/) cover cross-component acceptance. + +Keep fixed datasets, locks, configuration examples, and scoring rules with their owning suite. Store local outputs in +a suite's ignored results directory or an explicit directory such as `.artifacts/`; keep credentials and unredacted +provider traces out of version control. Preserve dataset hashes and result schema identifiers so saved evidence +remains interpretable. + +Add suites under the category describing what they measure. Keep suite-specific execution tools, storage, workers, +and Web interfaces inside that suite. Define each suite's command and errors in its own package. +Report different datasets, judges, hosts, and protocols as separate experiments. diff --git a/evaluation/coding/README.md b/evaluation/coding/README.md new file mode 100644 index 0000000000..531e2f90d0 --- /dev/null +++ b/evaluation/coding/README.md @@ -0,0 +1,13 @@ +# Coding evaluations + +Coding suites measure whether an agent can complete repository tasks and preserve the context needed to continue work. + +- [SWE-bench Pro](swebench_pro/README.md): pinned public tasks, gold-patch validation, PowerContext OFF/ON runs, + official grading, and task reports. +- [Work continuity](work_continuity/README.md): recorded continuation attempts, context assembly, quality scoring, + and comparisons between handoff and compaction methods, exposed through `work-continuity`. + +The shared Python environment lives in [evaluation](../README.md). SWE-bench Pro provides the `swebench-pro` command +and owns its +[Web console, worker, execution tools, and deployment configuration](swebench_pro/docs/console.md). +Keep each coding suite's runtime and results with that suite. diff --git a/evaluation/coding/swebench_pro/README.md b/evaluation/coding/swebench_pro/README.md new file mode 100644 index 0000000000..11ecd18401 --- /dev/null +++ b/evaluation/coding/swebench_pro/README.md @@ -0,0 +1,150 @@ +# SWE-bench Pro coding evaluation + +This suite measures coding-task completion with the pinned SWE-bench Pro public v2 dataset. It validates the gold +patch before running PowerContext OFF/ON treatments, grades each generated patch with the official harness, and +writes task reports and context traces. A batch can also select only the OFF or ON treatment. + +The suite owns its dataset adapter and catalog, prediction format, gold validation, runner, scoring integration, +reports, CLI commands, and tests. Its execution tools, artifact storage, Web console, worker, and deployment +templates also live here. The suite provides the `swebench-pro` command; the shared Python environment lives in +[`evaluation/`](../../README.md). + +```text +swebench_pro/ +├── src/powercontext_eval_swebench_pro/ # Suite, execution, storage, Web API, and worker +├── tests/ # Suite, execution, API, and worker behavior +├── web/ # Console frontend and browser tests +├── deploy/ # Environment and service templates +└── docs/console.md # Console setup and operations +``` + +See [console and worker setup](docs/console.md) for deployment, configuration, batch scheduling, and recovery. + +## Pinned inputs + +- Harness commit: `ca10a60a5fcae51e6948ffe1485d4153d421e6c5` +- Dataset revision: `7ab5114912baf22bb098818e604c02fe7ad2c11f` +- Public dataset: 731 tasks, SHA-256 `b5b2462bfbf5aeb2cb7ba7d215778a1768b85f9d7ad7f748546c7f80a0ad1510` +- Task sets: `swebench-pro-public-v2` (731 tasks) and `swebench-pro-stability-v1` (24 pinned regression tasks) + +The catalog validates the dataset schema, row count, order, and hash before admitting a batch. Gold validation +overrides retain the original task identity and an audit record in the report. + +## Prepare harness + +Choose the same absolute writable runtime root used by the console configuration. The commands below use +`/srv/powercontext-eval`; change this value consistently if your deployment uses a different location. + +```bash +export EVALUATION_ROOT=/srv/powercontext-eval +install -d -m 0700 "$EVALUATION_ROOT/cache" "$EVALUATION_ROOT/venvs" + +git clone https://github.com/scaleapi/SWE-bench_Pro-os.git \ + "$EVALUATION_ROOT/cache/swebench-pro.git" +git -C "$EVALUATION_ROOT/cache/swebench-pro.git" checkout --detach \ + ca10a60a5fcae51e6948ffe1485d4153d421e6c5 +test "$(git -C "$EVALUATION_ROOT/cache/swebench-pro.git" rev-parse HEAD)" = \ + ca10a60a5fcae51e6948ffe1485d4153d421e6c5 +test "$(sha256sum "$EVALUATION_ROOT/cache/swebench-pro.git/helper_code/sweap_eval_full_v2.jsonl" | cut -d' ' -f1)" = \ + b5b2462bfbf5aeb2cb7ba7d215778a1768b85f9d7ad7f748546c7f80a0ad1510 + +uv venv --python 3.11 "$EVALUATION_ROOT/venvs/swebench-pro-ca10a60" +uv pip install \ + --python "$EVALUATION_ROOT/venvs/swebench-pro-ca10a60/bin/python" \ + -r "$EVALUATION_ROOT/cache/swebench-pro.git/requirements.txt" +``` + +Both revision and dataset-hash checks must succeed before running evaluations. The console's default harness +Python path uses this pinned virtual environment; the standalone command below selects the same interpreter +explicitly with `--harness-python`. + +## Run + +Run these commands from the repository root. The shared environment is configured by `evaluation/pyproject.toml`: + +```bash +uv sync --project evaluation --frozen +uv run --project evaluation swebench-pro --help +``` + +A real run requires Linux, Docker, the pinned harness and task images, Codex credentials, and a PowerContext Git +source. Prepare the harness above, then follow the [console setup instructions](docs/console.md#quick-start) +to configure the source, tools, credentials, and Web and Worker services. + +Run one task with the standalone runner: + +```bash +uv run --project evaluation swebench-pro run \ + --root "$EVALUATION_ROOT" \ + --harness-python "$EVALUATION_ROOT/venvs/swebench-pro-ca10a60/bin/python" \ + --instance-id instance_owner__repository-revision +``` + +The instance ID must come from the pinned catalog. Paths derive from `--root` and can be overridden individually. +Native networking is the default; TokensFlow telemetry and proxy integration are optional. + +To test against OceanBase, pass `--database-config /absolute/private/database.json`. This private JSON object must +contain exactly `off` and `on`, each using the `settings.database` shape (`kind`, `url`, and optional profile +settings). Create a separate evaluation database for each arm: + +```json +{ + "off": { + "kind": "oceanbase", + "url": "mysql+aoceanbase://eval:REPLACE_WITH_PASSWORD@db.example:2881/swe_eval_off?charset=utf8mb4" + }, + "on": { + "kind": "oceanbase", + "url": "mysql+aoceanbase://eval:REPLACE_WITH_PASSWORD@db.example:2881/swe_eval_on?charset=utf8mb4" + } +} +``` + +OFF disables inference while ON enables its configured processing capabilities. PowerContext records that +processing configuration in a database-wide manifest, so separate Scopes in the same OceanBase database do not +isolate these configurations. The runner rejects OFF/ON configurations that point to the same OceanBase host, +port, and database, even when they use different credentials. + +OceanBase URLs must put credentials and the host/port in the authority and the database name in the path. +Query options such as `charset=utf8mb4` remain supported. The runner rejects query parameters that override +connection identity (`host`, `port`, `user`, `username`, `password`, `passwd`, `db`, `database`, `unix_socket`), +load external connection defaults (`read_default_file`, `read_default_group`), or run initialization SQL +(`init_command`, `sql_mode`). Query keys are checked after URL decoding, regardless of case. Percent-encode +`@` inside passwords and any `#`, tab, carriage return, or newline characters because URL parsers interpret +their raw forms differently. Usernames such as `user@tenant` remain supported. + +When deriving the configurations from `.env`, serialize each actual URL, not the masked `SecretStr` +representation, and replace its database name with the corresponding newly created evaluation database. Keep the +file private (mode `0600`) and delete it after the run. The runner does not create or drop external databases. +Never point this configuration at a business database. + +Each arm uses its own explicit database configuration and registers its own Scope. Other `container_env` +settings continue to apply only to ON. Without an explicit database configuration, each arm uses its local SQLite default. +Treatment evidence queries the configured backend and records its kind and a target fingerprint without the +connection URL or password. Containers install the locked Server/CLI runtime dependencies with `--no-dev`. +The PowerContext source revision must include `tzdata` in its builtin runtime dependencies for images without +system timezone data. OceanBase credentials appear only in each arm's +private `runtime/container.env`; remove these files with the disposable runtime when cleaning up a run. + +After the Web and Worker services are healthy, create a bounded batch through the console API: + +```bash +uv run --project evaluation swebench-pro create-batch \ + --console-url http://127.0.0.1:8787 \ + --task-set swebench-pro-stability-v1 \ + --treatment-mode on_only \ + --powercontext-ref latest \ + --idempotency-key "stability-$(date -u +%Y%m%dT%H%M%SZ)" +``` + +Choose `off_on`, `on_only`, or `off_only` for the treatment mode. Start with the stability task set before a full +public-v2 run. The console retains completed reports and supports compatible historical baseline comparisons. + +## Validate + +```bash +uv run --project evaluation pytest -c evaluation/pyproject.toml evaluation/coding/swebench_pro/tests -m "not live" -q +``` + +These local tests verify the pinned contracts, runner phases, report semantics, and command wiring using controlled +inputs and processes. They do not execute a real model or establish the outcome of a full SWE-bench Pro run. diff --git a/evaluation/deploy/powercontext-eval-web.service.in b/evaluation/coding/swebench_pro/deploy/powercontext-eval-web.service.in similarity index 80% rename from evaluation/deploy/powercontext-eval-web.service.in rename to evaluation/coding/swebench_pro/deploy/powercontext-eval-web.service.in index 1479e8a6ca..c3d13b4769 100644 --- a/evaluation/deploy/powercontext-eval-web.service.in +++ b/evaluation/coding/swebench_pro/deploy/powercontext-eval-web.service.in @@ -1,5 +1,5 @@ [Unit] -Description=PowerContext evaluation console web service +Description=PowerContext SWE-bench Pro console web service After=network.target [Service] @@ -9,7 +9,7 @@ Group=@EVALUATION_GROUP@ WorkingDirectory=@REPOSITORY_ROOT@ EnvironmentFile=@EVALUATION_ROOT@/config/evaluation-console.env Environment=UV_CACHE_DIR=@EVALUATION_ROOT@/cache/uv -ExecStart=@UV_BINARY@ run --project evaluation powercontext-eval web +ExecStart=@UV_BINARY@ run --project evaluation swebench-pro web KillMode=mixed SuccessExitStatus=143 Restart=on-failure diff --git a/evaluation/deploy/powercontext-eval-worker.service.in b/evaluation/coding/swebench_pro/deploy/powercontext-eval-worker.service.in similarity index 75% rename from evaluation/deploy/powercontext-eval-worker.service.in rename to evaluation/coding/swebench_pro/deploy/powercontext-eval-worker.service.in index 22041f0325..6d2fb5a3bf 100644 --- a/evaluation/deploy/powercontext-eval-worker.service.in +++ b/evaluation/coding/swebench_pro/deploy/powercontext-eval-worker.service.in @@ -1,5 +1,5 @@ [Unit] -Description=PowerContext evaluation console worker +Description=PowerContext SWE-bench Pro console worker After=network.target powercontext-eval-web.service [Service] @@ -9,7 +9,7 @@ Group=@EVALUATION_WORKER_GROUP@ WorkingDirectory=@REPOSITORY_ROOT@ EnvironmentFile=@EVALUATION_ROOT@/config/evaluation-console.env Environment=UV_CACHE_DIR=@EVALUATION_ROOT@/cache/uv -ExecStart=@UV_BINARY@ run --project evaluation powercontext-eval worker +ExecStart=@UV_BINARY@ run --project evaluation swebench-pro worker KillMode=mixed Restart=on-failure RestartSec=5s diff --git a/evaluation/deploy/powercontext-eval.env.example b/evaluation/coding/swebench_pro/deploy/powercontext-eval.env.example similarity index 98% rename from evaluation/deploy/powercontext-eval.env.example rename to evaluation/coding/swebench_pro/deploy/powercontext-eval.env.example index 7966fa2b89..4ed539473d 100644 --- a/evaluation/deploy/powercontext-eval.env.example +++ b/evaluation/coding/swebench_pro/deploy/powercontext-eval.env.example @@ -45,7 +45,7 @@ POWERCONTEXT_EVAL_AUTH_JSON=/srv/powercontext-eval/codex-home/auth.json # POWERCONTEXT_EVAL_EXTRA_NO_PROXY_HOSTS=mirror.example.test # Private IPv4 pool with at least 32 /28 subnets; change it if it overlaps host networks. POWERCONTEXT_EVAL_DOCKER_NETWORK_POOL=172.30.0.0/15 -POWERCONTEXT_EVAL_FRONTEND_DIST=/srv/powercontext-eval/deploy/powercontext/evaluation/web/dist +POWERCONTEXT_EVAL_FRONTEND_DIST=/srv/powercontext-eval/deploy/powercontext/evaluation/coding/swebench_pro/web/dist POWERCONTEXT_EVAL_LEASE_SECONDS=60 POWERCONTEXT_EVAL_POLL_SECONDS=1 POWERCONTEXT_EVAL_USAGE_PAUSE_PERCENT=80 diff --git a/evaluation/coding/swebench_pro/docs/console.md b/evaluation/coding/swebench_pro/docs/console.md new file mode 100644 index 0000000000..b27a2fbfb2 --- /dev/null +++ b/evaluation/coding/swebench_pro/docs/console.md @@ -0,0 +1,269 @@ +# SWE-bench Pro console and worker + +The [SWE-bench Pro suite](../README.md) owns this Web console, worker, process execution, and artifact storage. +Its Python package, `powercontext_eval_swebench_pro`, is built with the other suite packages by +[`evaluation/pyproject.toml`](../../../pyproject.toml). The Web process owns the HTTP API and report UI; the worker +owns execution, retries, resource cleanup, and durable recovery. The console schedules SWE-bench Pro batches with +paired OFF/ON or single-arm runs. + +The service is intentionally deployment-neutral. Host names, operators, filesystem roots, optional proxy endpoints, Docker +network ranges, credentials, and service locations are supplied by the operator. The repository does not contain a +production environment file or a ready-to-install host-specific systemd unit. + +## Runtime requirements + +- Linux, Git, Python 3.11 or newer, `uv`, and Node.js/npm +- Docker with permission to pull images, create isolated bridge networks, and run evaluation containers +- Codex CLI and a valid Codex `auth.json` +- [`regctl`](https://github.com/regclient/regclient) for importing task images that are not already present +- enough disk space and inodes for the selected parallelism + +TokensFlow telemetry and the credential-free loopback proxy are optional integrations. Both are disabled by default; +the normal open-source path uses native container egress and starts neither a proxy relay nor a TokensFlow +daemon/finalizer. + +Install and validate from the repository root: + +```bash +uv sync --project evaluation --frozen +uv run --project evaluation pytest -c evaluation/pyproject.toml evaluation/coding/swebench_pro/tests -m "not live" -q +uv run --project evaluation ruff check evaluation/coding/swebench_pro evaluation/memory/longmemeval_v2 +uv run --project evaluation ruff format --check evaluation/coding/swebench_pro evaluation/memory/longmemeval_v2 +uv run --directory evaluation ty check +``` + +Build and test the web UI: + +```bash +cd evaluation/coding/swebench_pro/web +npm ci +npm test -- --run +npm run build +``` + +## Quick start + +The commands below prepare a single-host development deployment. They intentionally bind the unauthenticated console +to `127.0.0.1`. Put an authenticating reverse proxy in front of the console before allowing access from another +machine; do not expose the HTTP service directly to an untrusted network. + +### 1. Prepare the evaluation root + +Choose an absolute writable directory and create the layout expected by the default configuration. The Web and +Worker processes must be able to read this repository and the protected configuration; the Worker also needs Docker +access and write access to the evaluation root. + +```bash +export REPOSITORY_ROOT="$(pwd -P)" +export EVALUATION_ROOT=/srv/powercontext-eval + +install -d -m 0700 \ + "$EVALUATION_ROOT/bin" \ + "$EVALUATION_ROOT/cache" \ + "$EVALUATION_ROOT/codex-home" \ + "$EVALUATION_ROOT/config" \ + "$EVALUATION_ROOT/source" \ + "$EVALUATION_ROOT/venvs" +``` + +### 2. Prepare the task harness + +Follow the [SWE-bench Pro suite setup](../README.md#prepare-harness) to install its pinned harness, dataset, and +Python environment under `$EVALUATION_ROOT`. The Web worker validates that task input before admitting a batch. + +### 3. Prepare source, tools, credentials, and frontend + +The source is a bare mirror so every submitted branch, tag, or commit resolves to immutable Git data rather than a +mutable developer working tree. Refresh or replace this mirror deliberately when evaluating newer PowerContext +commits. + +```bash +git clone --mirror "$REPOSITORY_ROOT" "$EVALUATION_ROOT/source/powercontext.git" + +install -m 0755 "$(command -v codex)" "$EVALUATION_ROOT/bin/codex" +install -m 0755 "$(command -v uv)" "$EVALUATION_ROOT/bin/uv" +install -m 0755 "$(command -v regctl)" "$EVALUATION_ROOT/bin/regctl" + +install -m 0600 "${CODEX_HOME:-$HOME/.codex}/auth.json" \ + "$EVALUATION_ROOT/codex-home/auth.json" +``` + +If the Codex account uses a custom provider, also copy its configuration without printing it: + +```bash +install -m 0600 "${CODEX_HOME:-$HOME/.codex}/config.toml" \ + "$EVALUATION_ROOT/codex-home/config.toml" +``` + +Build the frontend from the repository checkout that will run the services: + +```bash +cd "$REPOSITORY_ROOT/evaluation/coding/swebench_pro/web" +npm ci +npm test -- --run +npm run build +cd "$REPOSITORY_ROOT" +``` + +### 4. Create and validate the runtime environment + +There is one SWE-bench Pro console runtime configuration file. Copy the example, edit every path that differs from the layout +above, and keep it private. In particular, set `POWERCONTEXT_EVAL_FRONTEND_DIST` to the absolute +`$REPOSITORY_ROOT/evaluation/coding/swebench_pro/web/dist` path. Uncomment `POWERCONTEXT_EVAL_CODEX_CONFIG` only if the file was installed +in the previous step. + +```bash +install -m 0600 evaluation/coding/swebench_pro/deploy/powercontext-eval.env.example \ + "$EVALUATION_ROOT/config/evaluation-console.env" +${EDITOR:-vi} "$EVALUATION_ROOT/config/evaluation-console.env" + +set -a +. "$EVALUATION_ROOT/config/evaluation-console.env" +set +a + +uv run --project evaluation python -c \ + 'import os; from powercontext_eval_swebench_pro.web.config import WebConfig; WebConfig.from_environment(os.environ); print("configuration valid")' +docker info >/dev/null +``` + +Use `POWERCONTEXT_EVAL_USAGE_MODE=api_key` for API-key credentials. That mode skips subscription-usage probing and +treats quota admission as sufficient; filesystem and Docker admission remain active. Keep `subscription` for a +Codex subscription whose CLI exposes the supported usage probe. + +### 5. Start Web and Worker + +From the repository root, load the same protected environment in two terminals: + +```bash +# Terminal 1 +set -a; . "$EVALUATION_ROOT/config/evaluation-console.env"; set +a +uv run --project evaluation swebench-pro web +``` + +```bash +# Terminal 2 +set -a; . "$EVALUATION_ROOT/config/evaluation-console.env"; set +a +uv run --project evaluation swebench-pro worker +``` + +Verify the control plane before submitting work: + +```bash +curl --fail --silent http://127.0.0.1:8787/api/health +``` + +Open `http://127.0.0.1:8787/`, choose `swebench-pro-stability-v1` and an `OFF + ON`, `ON only`, or `OFF only` run. +This 24-task set is the deployment regression suite; use it before the 731-task `swebench-pro-public-v2` set. A +completed aggregate report can freeze any executed Arm as an immutable baseline. Reports may select multiple +compatible baselines for historical comparison without rerunning evaluation, while the baseline library lists the +newest saved baselines first. The same bounded batch can be created from the CLI after Web and Worker are healthy: + +```bash +uv run --project evaluation swebench-pro create-batch \ + --console-url http://127.0.0.1:8787 \ + --task-set swebench-pro-stability-v1 \ + --treatment-mode on_only \ + --powercontext-ref latest \ + --idempotency-key "stability-$(date -u +%Y%m%dT%H%M%SZ)" +``` + +## Configuration files + +Only the environment file configures the SWE-bench Pro console and worker. The other files either belong to Codex or are +deployment templates: + +| File | Required | Purpose | +| --- | --- | --- | +| `evaluation-console.env` | yes | Web, Worker, storage, benchmark, retries, parallelism, and optional integrations | +| `auth.json` | yes | Codex credentials; referenced by path and copied into isolated task homes | +| `config.toml` | no | Custom Codex provider/model configuration | +| `powercontext-eval.env.example` | no | Safe template; never loaded directly in production | +| `*.service.in` | no | Unrendered systemd templates; they are not additional application configuration | + +## Configuration reference + +Copy `evaluation/coding/swebench_pro/deploy/powercontext-eval.env.example` to a protected location, replace every applicable example +value, and set mode `0600`. Only `POWERCONTEXT_EVAL_ROOT` is required by the configuration loader; the runtime tools, +dataset, credentials, and source checkout must exist at their derived or explicitly configured paths before work is +claimed. + +Derived paths are rooted below `POWERCONTEXT_EVAL_ROOT` unless explicitly overridden. The settings fall into these +groups: + +| Group | Variables | +| --- | --- | +| Service and storage | `ROOT`, `HOST`, `PORT`, `DATABASE_PATH`, `RUN_ROOT`, `FRONTEND_DIST` | +| Benchmark inputs | `POWERCONTEXT_SOURCE`, `HARNESS_ROOT`, `HARNESS_PYTHON`, `DATASET_PATH` | +| Executables and credentials | `CODEX_BINARY`, `CODEX_MODELS`, `CODEX_CONFIG`, `UV_BINARY`, `REGISTRY_BINARY`, `AUTH_JSON` | +| Scheduling and recovery | `TASK_PARALLELISM`, `MAX_ATTEMPTS`, `LEASE_SECONDS`, `POLL_SECONDS`, `WORKSPACE_RECLAIM_INTERVAL_SECONDS` | +| Resource admission | `FILESYSTEM_MIN_FREE_BYTES`, `FILESYSTEM_MIN_FREE_INODES`, `DOCKER_NETWORK_POOL` | +| Usage admission | `USAGE_MODE`, `USAGE_PAUSE_PERCENT`, `USAGE_PROBE_SECONDS`, `USAGE_PROBE_TIMEOUT_SECONDS`, `USAGE_SNAPSHOT_MAX_AGE_SECONDS` | +| Optional integrations | `TOKENSFLOW_ENABLED`, TokenFlow settings, `PROXY_URL`, `EXTRA_NO_PROXY_HOSTS` | + +Every name in the table has the `POWERCONTEXT_EVAL_` prefix. Important behavior-changing settings are: + +- `POWERCONTEXT_EVAL_TASK_PARALLELISM` controls concurrent OFF/ON task pairs and defaults to `1`. +- `POWERCONTEXT_EVAL_MAX_ATTEMPTS` bounds durable per-task retries and defaults to `5`. +- `POWERCONTEXT_EVAL_USAGE_MODE=api_key` disables subscription quota probing; API-key mode is treated as having + sufficient quota while still enforcing resource admission. +- `POWERCONTEXT_EVAL_TOKENSFLOW_ENABLED=true` explicitly enables internal TokensFlow telemetry and then requires its + binary, user home, and egress network settings. When false or absent, no TokensFlow profile, daemon, finalizer, + mount, or retained audit artifact is created. +- `POWERCONTEXT_EVAL_PROXY_URL` explicitly enables the loopback proxy relay. When absent, task networks use native + egress and proxy variables are cleared from managed task processes. +- `POWERCONTEXT_EVAL_EXTRA_NO_PROXY_HOSTS` is a comma-separated, validated list appended to loopback-only + `NO_PROXY`. It is valid only with the evaluation proxy and is recorded in each retained report. +- `POWERCONTEXT_EVAL_DOCKER_NETWORK_POOL` selects the private IPv4 pool used for isolated /28 task networks. It must + provide at least 32 subnets and is recorded in each retained report. + +Credential files are referenced by path and copied into isolated task homes. Never place credential values in the +environment example, command line, logs, reports, or repository. + +## systemd templates + +`evaluation/coding/swebench_pro/deploy/*.service.in` are templates, not installable units. Render all placeholders into a staging +directory, inspect the result, and run `systemd-analyze verify` before installation. Required placeholders are: + +- `@EVALUATION_ROOT@`: absolute writable evaluation root +- `@REPOSITORY_ROOT@`: absolute deployment checkout +- `@UV_BINARY@`: absolute `uv` executable +- `@EVALUATION_USER@` and `@EVALUATION_GROUP@`: unprivileged web identity +- `@EVALUATION_WORKER_USER@` and `@EVALUATION_WORKER_GROUP@`: worker identity with the required Docker access + +The web template is sandboxed to the evaluation root. The worker template intentionally does not claim a generic +sandbox because Docker access and host cleanup policy vary by deployment. Do not grant the web role Docker access. + +## Control and recovery model + +Only operator pause or cancel changes durable control intent. Task failures do not pause healthy peers. Retriable +task failures enter durable backoff and another free worker slot may claim the next eligible task. The default five +attempt budget uses 30, 120, 300, and 600 second delays. An exhausted or non-retriable task becomes a retained +failure while the rest of the batch continues. + +The worker performs startup-only orphan recovery under a process lock. A normal claim never steals another slot's +lease. Loss of attempt ownership cancels child processes, after which the lifecycle cleaner removes exact +attempt-owned containers, networks, and scratch workspaces. Retained `/runs` reports and private incident evidence +remain available for audit. When TokensFlow is explicitly enabled, finalizer-owned containers are excluded until +their durable finalization job is terminal. + +Admission pressure is transient: disk, inode, Docker, or usage probe pressure prevents new claims without changing +batch intent. Workspace reclamation runs continuously and removes only terminal attempts after report publication; +it never deletes running, queued, retryable, or finalizer-owned state. + +## Operational acceptance + +Before resuming a real batch: + +1. verify the exact source revision and a clean checkout; +2. validate the protected environment file without displaying it; +3. run backend, lint, format, type, frontend test, and frontend build gates; +4. verify rendered systemd units and confirm web/worker health; +5. run a small OFF/ON batch and inspect Gold, OFF, ON, official evaluator, retry, cleanup, and report evidence; +6. confirm `active_task_pairs` never exceeds configured parallelism; +7. confirm unrelated host services are neither dependencies nor restart targets; +8. perform a secret scan over retained artifacts and logs; +9. document the exact rollback revision before increasing parallelism. + +Rollback is a source checkout and service restart operation. Do not delete the database, `/runs`, incident evidence, +or task workspaces needed by queued and retryable attempts. Do not use global Docker prune as an evaluation cleanup +mechanism. diff --git a/benchmark/__init__.py b/evaluation/coding/swebench_pro/ruff.toml similarity index 92% rename from benchmark/__init__.py rename to evaluation/coding/swebench_pro/ruff.toml index d9d44fcbe5..2c6e969e7e 100644 --- a/benchmark/__init__.py +++ b/evaluation/coding/swebench_pro/ruff.toml @@ -12,4 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Reproducible PowerContext benchmarks.""" + +target-version = "py311" +line-length = 120 diff --git a/evaluation/src/powercontext_eval/__init__.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/__init__.py similarity index 76% rename from evaluation/src/powercontext_eval/__init__.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/__init__.py index 0a6a0b13ec..875bbcf515 100644 --- a/evaluation/src/powercontext_eval/__init__.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/__init__.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Independent evaluation runner for PowerContext.""" +"""Pinned SWE-bench Pro evaluation and execution services.""" -from powercontext_eval.models import Arm, PowerContextRef -from powercontext_eval.paths import EvaluationPaths +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef +from powercontext_eval_swebench_pro.paths import EvaluationPaths __all__ = ["Arm", "EvaluationPaths", "PowerContextRef"] diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/adapter.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/adapter.py similarity index 98% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/adapter.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/adapter.py index 97f7c7522f..8f4e5c5903 100644 --- a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/adapter.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/adapter.py @@ -23,7 +23,7 @@ from types import MappingProxyType from typing import Any -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_swebench_pro.errors import SweBenchProError HARNESS_COMMIT = "ca10a60a5fcae51e6948ffe1485d4153d421e6c5" DATASET_REVISION = "7ab5114912baf22bb098818e604c02fe7ad2c11f" @@ -82,7 +82,7 @@ _DOCKER_TAG_LIMIT = 128 -class DatasetSchemaError(PowerContextEvalError): +class DatasetSchemaError(SweBenchProError): """A pinned dataset row does not match the expected public schema.""" diff --git a/evaluation/src/powercontext_eval/artifacts.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/artifacts.py similarity index 99% rename from evaluation/src/powercontext_eval/artifacts.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/artifacts.py index 2a52b6e4b6..6a5b5bb6a7 100644 --- a/evaluation/src/powercontext_eval/artifacts.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/artifacts.py @@ -30,7 +30,7 @@ from types import MappingProxyType from typing import Any, BinaryIO -from powercontext_eval.models import Arm +from powercontext_eval_swebench_pro.models import Arm class ArtifactError(Exception): diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/catalog.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/catalog.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/catalog.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/catalog.py index 5f4e542bf3..403ff23c90 100644 --- a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/catalog.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/catalog.py @@ -24,8 +24,8 @@ from types import MappingProxyType from typing import Literal, Self, TypeAlias -from powercontext_eval.benchmarks.swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance +from powercontext_eval_swebench_pro.errors import SweBenchProError PUBLIC_V2_COUNT = 731 PUBLIC_V2_SHA256 = "b5b2462bfbf5aeb2cb7ba7d215778a1768b85f9d7ad7f748546c7f80a0ad1510" @@ -98,7 +98,7 @@ STABILITY_V1_COUNT = len(STABILITY_V1_CASES) -class CatalogError(PowerContextEvalError): +class CatalogError(SweBenchProError): """The pinned task-set file cannot be trusted or indexed.""" diff --git a/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/cli.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/cli.py new file mode 100644 index 0000000000..38f9508f80 --- /dev/null +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/cli.py @@ -0,0 +1,230 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SWE-bench Pro command-line commands.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Annotated, Any, cast +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +import typer +from pydantic import ValidationError + +from powercontext_eval_swebench_pro.catalog import PUBLIC_V2_TASK_SET, SweBenchProCatalog, TaskSet +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT +from powercontext_eval_swebench_pro.models import Arm, TreatmentMode +from powercontext_eval_swebench_pro.service_cli import codex_contract_smoke, web, worker + +app = typer.Typer(no_args_is_help=True, help="Pinned SWE-bench Pro evaluation.") +app.command("web")(web) +app.command("worker")(worker) +app.command("codex-contract-smoke")(codex_contract_smoke) +DEFAULT_DOCKER_NETWORK_POOL = "172.30.0.0/15" + + +def run_swebench_pro_instance(*args: Any, **kwargs: Any) -> Any: + """Load the SWE-bench Pro runner only when its command is used.""" + + from powercontext_eval_swebench_pro.runner import run_swebench_pro_instance as implementation + + return implementation(*args, **kwargs) + + +@app.command("run") +def swebench_pro_run( + root_path: str = typer.Option(..., "--root"), + powercontext_source: str | None = typer.Option(None), + powercontext_ref: str = typer.Option("latest"), + harness_root: str | None = typer.Option(None), + harness_python: str | None = typer.Option(None), + dataset_path: str | None = typer.Option(None), + instance_id: str = typer.Option(...), + codex_bin: str | None = typer.Option(None), + tokensflow_enabled: bool = typer.Option(False, "--tokensflow/--no-tokensflow"), + tokensflow_bin: str | None = typer.Option(None), + tokensflow_user_home: str | None = typer.Option(None), + tokensflow_egress_network: str | None = typer.Option(None), + uv_bin: str | None = typer.Option(None), + registry_bin: str | None = typer.Option(None), + auth_json: str | None = typer.Option(None), + proxy_url: str | None = typer.Option(None), + docker_network_pool: str = typer.Option(DEFAULT_DOCKER_NETWORK_POOL), + extra_no_proxy_hosts: str = typer.Option(""), + model: str = typer.Option(DEFAULT_CODEX_MODEL, "--model"), + reasoning_effort: str = typer.Option(DEFAULT_REASONING_EFFORT, "--reasoning-effort"), + run_id: str | None = typer.Option(None), + database_config: Annotated[ + Path | None, + typer.Option( + help="Private JSON object with off/on settings.database configurations; use separate test databases." + ), + ] = None, +) -> None: + """Run Gold, PowerContext OFF/ON, official grading, and report generation.""" + + from powercontext_eval_swebench_pro.powercontext_sut import UnsafeSutConfiguration, validated_database_configs + from powercontext_eval_swebench_pro.runner import RunConfig + + root = Path(root_path) + harness = Path(harness_root) if harness_root is not None else root / "cache" / "swebench-pro.git" + dataset = Path(dataset_path) if dataset_path is not None else harness / "helper_code" / "sweap_eval_full_v2.jsonl" + binaries = root / "bin" + databases = None + if database_config is not None: + try: + payload = json.loads(database_config.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or set(payload) != {arm.value for arm in Arm}: + raise TypeError + databases = validated_database_configs({Arm(key): value for key, value in payload.items()}) + except (OSError, UnicodeError, ValueError, TypeError, UnsafeSutConfiguration): + raise typer.BadParameter( + "Database configuration must be a readable JSON object with valid off/on settings for separate databases.", + param_hint="--database-config", + ) from None + + catalog = SweBenchProCatalog.load(dataset) + result = run_swebench_pro_instance( + RunConfig( + root=root, + powercontext_source=( + Path(powercontext_source) if powercontext_source is not None else root / "source" / "powercontext.git" + ), + powercontext_ref=powercontext_ref, + harness_root=harness, + harness_python=( + Path(harness_python) + if harness_python is not None + else root / "venvs" / "swebench-pro" / "bin" / "python" + ), + codex_binary=Path(codex_bin) if codex_bin is not None else binaries / "codex", + tokensflow_enabled=tokensflow_enabled, + tokensflow_binary=( + Path(tokensflow_bin) + if tokensflow_bin is not None + else (binaries / "tokensflow" if tokensflow_enabled else None) + ), + tokensflow_user_home=( + Path(tokensflow_user_home) + if tokensflow_user_home is not None + else (root / "tokensflow-home" if tokensflow_enabled else None) + ), + tokensflow_egress_network=tokensflow_egress_network, + uv_binary=Path(uv_bin) if uv_bin is not None else binaries / "uv", + registry_binary=Path(registry_bin) if registry_bin is not None else binaries / "regctl", + auth_json=Path(auth_json) if auth_json is not None else root / "codex-home" / "auth.json", + proxy_url=proxy_url, + docker_network_pool=docker_network_pool, + extra_no_proxy_hosts=tuple(host for host in extra_no_proxy_hosts.split(",") if host), + run_id=run_id or datetime.now(UTC).strftime("run-%Y%m%d-%H%M%S"), + model=model, + reasoning_effort=reasoning_effort, + database_configs=databases, + ), + instance=catalog.require(instance_id), + ) + typer.echo( + json.dumps( + { + "run_id": result.run_id, + "report": str(result.report_path), + "off_resolved": result.off_resolved, + "on_resolved": result.on_resolved, + }, + sort_keys=True, + ) + ) + + +@app.command("create-batch") +def swebench_pro_create_batch( + idempotency_key: str = typer.Option(..., "--idempotency-key"), + console_url: str = typer.Option("http://127.0.0.1:8787", "--console-url"), + powercontext_ref: str = typer.Option("latest", "--powercontext-ref"), + task_set: str = typer.Option(PUBLIC_V2_TASK_SET, "--task-set"), + model: str = typer.Option(DEFAULT_CODEX_MODEL, "--model"), + treatment_mode: Annotated[TreatmentMode, typer.Option("--treatment-mode")] = TreatmentMode.OFF_ON, + usage_pause_percent: int = typer.Option(80, "--usage-pause-percent", min=1, max=100), + start_paused: bool = typer.Option(False, "--start-paused/--start-running"), +) -> None: + """Create one full batch through the console API, optionally atomically paused.""" + + from powercontext_eval_swebench_pro.web.batches import BatchCreate + + endpoint = _batch_api_endpoint(console_url) + try: + batch = BatchCreate( + powercontext_ref=powercontext_ref, + benchmark="swebench-pro", + task_set=cast(TaskSet, task_set), + model=model, + reasoning_effort=DEFAULT_REASONING_EFFORT, + treatment_mode=treatment_mode, + idempotency_key=idempotency_key, + usage_pause_percent=usage_pause_percent, + initial_control_intent="pause" if start_paused else "run", + ) + except ValidationError: + raise typer.BadParameter("Invalid batch configuration.") from None + request = Request( + endpoint, + data=batch.model_dump_json().encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=30) as response: + payload = json.loads(response.read()) + except HTTPError as error: + if error.code == 422: + raise typer.BadParameter( + "Codex model is not enabled for new evaluation work.", + param_hint="--model", + ) from None + raise typer.Exit(code=1) from None + except (URLError, OSError, ValueError, UnicodeDecodeError): + raise typer.Exit(code=1) from None + if not isinstance(payload, dict) or not isinstance(payload.get("batch_id"), str): + raise typer.Exit(code=1) + typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + + +def _batch_api_endpoint(console_url: str) -> str: + parsed = urlsplit(console_url) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise typer.BadParameter("Invalid console URL.", param_hint="--console-url") + return console_url.rstrip("/") + "/api/batches" + + +def main() -> None: + """Run the SWE-bench Pro command-line application.""" + + app() + + +if __name__ == "__main__": + main() diff --git a/evaluation/src/powercontext_eval/codex.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/codex.py similarity index 97% rename from evaluation/src/powercontext_eval/codex.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/codex.py index a223c4cef7..c06c3cfddf 100644 --- a/evaluation/src/powercontext_eval/codex.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/codex.py @@ -25,10 +25,10 @@ from types import MappingProxyType from typing import Any, BinaryIO, Protocol -from powercontext_eval.artifacts import ArtifactStore -from powercontext_eval.errors import CommandError, PowerContextEvalError -from powercontext_eval.models import Arm -from powercontext_eval.process import CommandResult +from powercontext_eval_swebench_pro.artifacts import ArtifactStore +from powercontext_eval_swebench_pro.errors import CommandError, SweBenchProError +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.process import CommandResult EXPECTED_CODEX_VERSION = "0.145.0" DEFAULT_CODEX_MODEL = "gpt-5.6-sol" @@ -44,11 +44,11 @@ def is_safe_codex_model(value: str) -> bool: return _SAFE_CODEX_MODEL.fullmatch(value) is not None -class UnsafeCodexInvocation(PowerContextEvalError): +class UnsafeCodexInvocation(SweBenchProError): """Dangerous Codex flags were requested outside an isolated task container.""" -class CodexInfrastructureError(PowerContextEvalError): +class CodexInfrastructureError(SweBenchProError): """Codex infrastructure failed before a benchmark patch could be evaluated.""" diff --git a/evaluation/src/powercontext_eval/context_trace.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/context_trace.py similarity index 97% rename from evaluation/src/powercontext_eval/context_trace.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/context_trace.py index 6e125def67..9fa3d0f1c1 100644 --- a/evaluation/src/powercontext_eval/context_trace.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/context_trace.py @@ -24,9 +24,9 @@ from pathlib import Path from typing import Any -from powercontext_eval.artifacts import ArtifactStore -from powercontext_eval.benchmarks.swebench_pro.evaluator import OfficialEvaluation, TestGroupResult -from powercontext_eval.models import Arm +from powercontext_eval_swebench_pro.artifacts import ArtifactStore +from powercontext_eval_swebench_pro.evaluator import OfficialEvaluation, TestGroupResult +from powercontext_eval_swebench_pro.models import Arm class ContextTraceError(ValueError): diff --git a/evaluation/src/powercontext_eval/docker_pressure.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/docker_pressure.py similarity index 100% rename from evaluation/src/powercontext_eval/docker_pressure.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/docker_pressure.py diff --git a/evaluation/src/powercontext_eval/errors.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/errors.py similarity index 82% rename from evaluation/src/powercontext_eval/errors.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/errors.py index 29721205b5..58e01a7e16 100644 --- a/evaluation/src/powercontext_eval/errors.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/errors.py @@ -12,21 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Evaluation runner errors.""" +"""SWE-bench Pro execution errors.""" from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: - from powercontext_eval.process import CommandResult + from powercontext_eval_swebench_pro.process import CommandResult -class PowerContextEvalError(Exception): - """Base error for evaluation runner failures.""" +class SweBenchProError(Exception): + """Base error for SWE-bench Pro evaluation failures.""" -class CommandError(PowerContextEvalError): +class CommandError(SweBenchProError): """Base error for a failed child process.""" def __init__(self, message: str, result: CommandResult) -> None: @@ -50,5 +50,5 @@ class CommandFailed(CommandError): """The child process exited unsuccessfully.""" -class GitSourceError(PowerContextEvalError): +class GitSourceError(SweBenchProError): """A Git source could not be normalized, resolved, or materialized.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/evaluator.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/evaluator.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/evaluator.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/evaluator.py index 358b79d6f4..12df3c5298 100644 --- a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/evaluator.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/evaluator.py @@ -24,9 +24,9 @@ from dataclasses import dataclass, field from pathlib import Path -from powercontext_eval import docker_pressure -from powercontext_eval.errors import PowerContextEvalError -from powercontext_eval.powercontext_sut import ( +from powercontext_eval_swebench_pro import docker_pressure +from powercontext_eval_swebench_pro.errors import SweBenchProError +from powercontext_eval_swebench_pro.powercontext_sut import ( LOOPBACK_NO_PROXY, ProxyRelay, ProxyRelayConfig, @@ -34,14 +34,14 @@ default_docker_bridge_gateway, direct_egress_environment, ) -from powercontext_eval.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner _SAFE_PREFIX = re.compile(r"[A-Za-z0-9._-]+") _TEST_STATUSES = frozenset({"PASSED", "FAILED", "SKIPPED", "ERROR"}) _LOG_EXCERPT_LIMIT = 4_000 -class OfficialResultError(PowerContextEvalError): +class OfficialResultError(SweBenchProError): """Official evaluator output is absent or ambiguous.""" diff --git a/evaluation/src/powercontext_eval/git_source.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/git_source.py similarity index 99% rename from evaluation/src/powercontext_eval/git_source.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/git_source.py index a18c87584d..6395511996 100644 --- a/evaluation/src/powercontext_eval/git_source.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/git_source.py @@ -35,9 +35,9 @@ from pathlib import Path from urllib.parse import parse_qsl, unquote, urlsplit, urlunsplit -from powercontext_eval.errors import CommandError, GitSourceError -from powercontext_eval.models import PowerContextRef -from powercontext_eval.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.errors import CommandError, GitSourceError +from powercontext_eval_swebench_pro.models import PowerContextRef +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner _FULL_LOWERCASE_SHA = re.compile(r"[0-9a-f]{40}") _SUPPORTED_URL_SCHEMES = frozenset({"http", "https", "ssh"}) diff --git a/evaluation/src/powercontext_eval/benchmarks/base.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/gold.py similarity index 88% rename from evaluation/src/powercontext_eval/benchmarks/base.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/gold.py index c994c92399..4778a3f161 100644 --- a/evaluation/src/powercontext_eval/benchmarks/base.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/gold.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared benchmark gates.""" +"""SWE-bench Pro gold-patch validation gate.""" from __future__ import annotations @@ -20,12 +20,12 @@ from dataclasses import dataclass from typing import TypeVar -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_swebench_pro.errors import SweBenchProError ResultT = TypeVar("ResultT") -class GoldCheckFailed(PowerContextEvalError): +class GoldCheckFailed(SweBenchProError): """The official evaluator rejected its own gold patch.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/gold_overrides.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/gold_overrides.py similarity index 100% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/gold_overrides.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/gold_overrides.py diff --git a/evaluation/src/powercontext_eval/models.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/models.py similarity index 100% rename from evaluation/src/powercontext_eval/models.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/models.py diff --git a/evaluation/src/powercontext_eval/paths.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/paths.py similarity index 96% rename from evaluation/src/powercontext_eval/paths.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/paths.py index a4c901fc71..e1c0d2547c 100644 --- a/evaluation/src/powercontext_eval/paths.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/paths.py @@ -18,7 +18,7 @@ from pathlib import Path from re import fullmatch -from powercontext_eval.models import Arm +from powercontext_eval_swebench_pro.models import Arm @dataclass(frozen=True) diff --git a/evaluation/src/powercontext_eval/powercontext_sut.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/powercontext_sut.py similarity index 91% rename from evaluation/src/powercontext_eval/powercontext_sut.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/powercontext_sut.py index 3047045e99..0aa3ed43dd 100644 --- a/evaluation/src/powercontext_eval/powercontext_sut.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/powercontext_sut.py @@ -52,11 +52,11 @@ from pathlib import Path from types import MappingProxyType from typing import Any, Protocol -from urllib.parse import quote, quote_plus, urlsplit +from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlsplit -from powercontext_eval import docker_pressure -from powercontext_eval.artifacts import ArtifactStore -from powercontext_eval.codex import ( +from powercontext_eval_swebench_pro import docker_pressure +from powercontext_eval_swebench_pro.artifacts import ArtifactStore +from powercontext_eval_swebench_pro.codex import ( DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, EXPECTED_CODEX_VERSION, @@ -66,10 +66,10 @@ CodexRunner, is_safe_codex_model, ) -from powercontext_eval.errors import CommandError, CommandFailed, CommandTimedOut, PowerContextEvalError -from powercontext_eval.models import Arm, arm_scope_key -from powercontext_eval.process import CommandResult, ProcessRunner -from powercontext_eval.tokensflow import ( +from powercontext_eval_swebench_pro.errors import CommandError, CommandFailed, CommandTimedOut, SweBenchProError +from powercontext_eval_swebench_pro.models import Arm, arm_scope_key +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.tokensflow import ( DrainDeadline, TokensFlowDaemonHandle, TokensFlowEvidence, @@ -89,6 +89,23 @@ PLUGIN_ID = "powercontext@powercontext" _SAFE_RUN_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") _SAFE_DOCKER_NETWORK = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") +_OCEANBASE_QUERY_OVERRIDES = frozenset( + { + "host", + "port", + "user", + "username", + "password", + "passwd", + "db", + "database", + "unix_socket", + "read_default_file", + "read_default_group", + "init_command", + "sql_mode", + } +) _SHA = re.compile(r"[0-9a-f]{40}") _INVALID_DOCKER_COPY_SYMLINK = re.compile(r'invalid symlink "[^"\r\n]+" -> "[^"\r\n]+"') _DOCKER_NETWORK_CONTROL_LOCK = threading.Lock() @@ -110,7 +127,7 @@ _CONTAINER_UV = "/tools/uv-dir/uv" _CONTAINER_RECORDER = "/evaluation/record_codex_jsonl.py" _CONTAINER_UV_PYTHON_INSTALL_DIR = "/runtime/uv-python" -_DEFAULT_RECORDER_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "record_codex_jsonl.py" +_DEFAULT_RECORDER_SCRIPT = Path(__file__).with_name("record_codex_jsonl.py") _TIMED_OUT_CONTAINER_REMOVAL_SETTLE_SECONDS = 90.0 _TIMED_OUT_CONTAINER_REMOVAL_POLL_SECONDS = 0.25 _READINESS_BUDGET_SECONDS = 120.0 @@ -162,6 +179,53 @@ with urlopen(request, timeout=10) as response: print(json.dumps({"scope_id": json.load(response)["scope_id"]})) """.strip() +_DATABASE_EVIDENCE_SCRIPT = """ +import asyncio +import hashlib +import json +import sys +from pathlib import Path +from sqlalchemy import text +from sqlalchemy.dialects import registry +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import create_async_engine +from powercontext.server.settings import ServerSettings + +async def main(): + config = ServerSettings().database + if config.kind == 'oceanbase': + registry.register('mysql.aoceanbase', 'pyobvector', 'AsyncOceanBaseDialect') + url = make_url(config.url.get_secret_value()) + elif config.kind == 'sqlite': + url = make_url(config.url) + else: + raise ValueError('Unsupported evaluation database backend') + target = { + 'kind': config.kind, 'driver': url.drivername, 'host': url.host, + 'port': url.port, 'username': url.username, 'database': url.database, + 'query': dict(url.query), + } + if config.kind == 'sqlite': + if not url.database or url.database == ':memory:': + raise ValueError('Evaluation evidence requires a persistent database') + target['database'] = str(Path(url.database).resolve()) + fingerprint = hashlib.sha256(json.dumps(target, sort_keys=True).encode()).hexdigest() + engine = create_async_engine(url, hide_parameters=True) + try: + async with engine.connect() as connection: + count = await connection.scalar( + text('SELECT COUNT(*) FROM pc_sources WHERE scope_id = :scope'), + {'scope': sys.argv[1]}, + ) + print(json.dumps({ + 'prompt_sources': count, 'database_kind': config.kind, + 'database_fingerprint': fingerprint, + })) + finally: + await engine.dispose() + +asyncio.run(main()) +""".strip() LOOPBACK_NO_PROXY = "127.0.0.1,localhost,::1" _TOKENSFLOW_RETRY_ATTEMPTS = 10 _PLUGIN_RELATIVE = Path("integrations/codex/plugins/powercontext") @@ -191,7 +255,7 @@ """ -class InvalidTreatment(PowerContextEvalError): +class InvalidTreatment(SweBenchProError): """Observed evidence does not prove the requested treatment.""" @@ -263,7 +327,7 @@ def safe_summary(self) -> str: return _READINESS_FAILURE_SUMMARIES[self.reason] -class UnsafeSutConfiguration(PowerContextEvalError): +class UnsafeSutConfiguration(SweBenchProError): """A SUT value could escape its owned resource boundary.""" @@ -420,6 +484,8 @@ class TreatmentEvidence: # Identifies the run and arm that registered ``scope_id``; absent in evidence # recorded before arms registered their own Scope. scope_key: str | None = None + database_kind: str | None = None + database_fingerprint: str | None = None def __post_init__(self) -> None: if type(self.plugin_installed) is not bool or type(self.server_ready) is not bool: @@ -437,6 +503,12 @@ def __post_init__(self) -> None: raise ValueError("Treatment counters must be non-negative integers") if not self.scope_id or self.scope_key == "": raise ValueError("Treatment scope must not be empty") + if self.database_kind is not None and self.database_kind not in {"sqlite", "oceanbase"}: + raise ValueError("Treatment database backend is invalid") + if (self.database_kind is None) != (self.database_fingerprint is None) or ( + self.database_fingerprint is not None and re.fullmatch(r"[0-9a-f]{64}", self.database_fingerprint) is None + ): + raise ValueError("Treatment database identity is invalid") def as_dict(self) -> dict[str, object]: return { @@ -449,6 +521,8 @@ def as_dict(self) -> dict[str, object]: "mcp_requests": self.mcp_requests, "scope_id": self.scope_id, "scope_key": self.scope_key, + "database_kind": self.database_kind, + "database_fingerprint": self.database_fingerprint, } @classmethod @@ -550,8 +624,10 @@ class SutConfig: codex_timeout: float = 3600 finalization_registrar: TokensFlowFinalizationRegistrar | None = None container_env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + database_configs: Mapping[Arm, Mapping[str, object]] | None = field(default=None, repr=False) def __post_init__(self) -> None: + object.__setattr__(self, "database_configs", validated_database_configs(self.database_configs)) if _SAFE_RUN_ID.fullmatch(self.run_id) is None: raise UnsafeSutConfiguration("Run id is unsafe") if ( @@ -1102,7 +1178,11 @@ def _execute_arm( tokensflow_secret_variants(paths.tokensflow_home / ".tokensflow/credentials.json") + tokensflow_environment_values ) - credential_variants = auth_secret_variants(paths.auth_source) + tokensflow_command_secrets + credential_variants = ( + auth_secret_variants(paths.auth_source) + + tokensflow_command_secrets + + database_configs_secret_variants(config.database_configs) + ) tokensflow: TokensFlowEvidence | None = None tokensflow_daemon: TokensFlowDaemonHandle | None = None try: @@ -2294,7 +2374,6 @@ def _prewarm_with_docker( network: str, relay_url: str | None, ) -> None: - del arm common_environment = { **runtime_proxy_environment(relay_url, config.extra_no_proxy_hosts), "UV_CACHE_DIR": "/runtime/uv-cache", @@ -2326,6 +2405,7 @@ def _prewarm_with_docker( config.task_image, "sync", "--frozen", + "--no-dev", "--project", "/source", "--extra", @@ -2680,12 +2760,6 @@ def _evidence( plugin: tuple[str, str], scope: str, ) -> TreatmentEvidence: - query = ( - "import json,sqlite3,sys;" - "db=sqlite3.connect('/runtime/pc-home/powercontext.db');" - "count=db.execute('SELECT COUNT(*) FROM pc_sources WHERE scope_id = ?', (sys.argv[1],)).fetchone()[0];" - "print(json.dumps({'prompt_sources':count}))" - ) result = self._docker.run( ( "docker", @@ -2693,20 +2767,29 @@ def _evidence( container, "/runtime/pc-env/bin/python", "-c", - query, + _DATABASE_EVIDENCE_SCRIPT, scope, "evidence", ), cwd=paths.runtime, timeout=60, + secrets=database_configs_secret_variants(config.database_configs), ) try: raw = json.loads(result.stdout) prompt_sources = raw["prompt_sources"] if isinstance(prompt_sources, bool) or not isinstance(prompt_sources, int): raise TypeError + database_kind = raw["database_kind"] + database_fingerprint = raw["database_fingerprint"] + if database_kind not in {"sqlite", "oceanbase"} or not isinstance(database_fingerprint, str): + raise TypeError + if re.fullmatch(r"[0-9a-f]{64}", database_fingerprint) is None: + raise TypeError + if config.database_configs is not None and database_kind != config.database_configs[arm]["kind"]: + raise TypeError except (json.JSONDecodeError, KeyError, TypeError) as error: - raise InvalidTreatment("PowerContext SQLite evidence is malformed") from error + raise InvalidTreatment("PowerContext database evidence is malformed") from error logs = self._docker.run( ("docker", "logs", container), cwd=paths.runtime, @@ -2724,6 +2807,8 @@ def _evidence( mcp_requests=mcp_requests, scope_id=scope, scope_key=arm_scope_key(config.run_id, arm), + database_kind=database_kind, + database_fingerprint=database_fingerprint, ) @@ -2804,24 +2889,131 @@ def _container_env_file_args( ) -> tuple[str, ...]: """Write user-supplied env to a file and return the --env-file docker flag. - Only the ON arm receives user env; the OFF arm stays pristine. Docker reads - --env-file line by line without shell expansion, avoiding escaping issues. + Each arm receives its own explicit database; other user env applies only to ON. + Docker reads --env-file without shell expansion. """ - if arm is not Arm.ON or not config.container_env: - return () reserved = {"NO_PROXY", "no_proxy", "HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"} - safe_env = {k: v for k, v in config.container_env.items() if k not in reserved} + safe_env = {k: v for k, v in config.container_env.items() if k not in reserved} if arm is Arm.ON else {} + if config.database_configs is not None: + safe_env = {k: v for k, v in safe_env.items() if not k.upper().startswith("POWERCONTEXT_SERVER_DATABASE")} + safe_env["POWERCONTEXT_SERVER_DATABASE"] = json.dumps(dict(config.database_configs[arm]), allow_nan=False) if not safe_env: return () + if any( + re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None or any(c in value for c in "\r\n\0") + for key, value in safe_env.items() + ): + raise UnsafeSutConfiguration("Container environment is invalid") env_file = paths.runtime / "container.env" - env_file.write_text( - "".join(f"{key}={value}\n" for key, value in safe_env.items()), - encoding="utf-8", - ) + descriptor = os.open(env_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + os.fchmod(stream.fileno(), 0o600) + stream.write("".join(f"{key}={value}\n" for key, value in safe_env.items())) return ("--env-file", str(env_file)) +def database_secret_variants(config: Mapping[str, object] | None) -> tuple[str, ...]: + """Keep external database credentials out of command output and artifacts.""" + + if config is None or config.get("kind") != "oceanbase": + return () + url = str(config["url"]) + password = urlsplit(url).password + values = (url, unquote(password)) if password else (url,) + variants: set[str] = set() + for value in values: + if not value: + continue + variants.update((value, quote(value, safe=""), quote_plus(value), base64.b64encode(value.encode()).decode())) + escaped = {value} + # The environment file embeds database JSON; Codex wraps command output in JSONL. + for _ in range(2): + escaped = { + json.dumps(item, ensure_ascii=ascii_only)[1:-1] for item in escaped for ascii_only in (False, True) + } + variants.update(escaped) + return tuple(sorted(variants, key=lambda item: (-len(item), item))) + + +def database_configs_secret_variants(configs: Mapping[Arm, Mapping[str, object]] | None) -> tuple[str, ...]: + """Protect credentials for both arms wherever evaluation output is retained.""" + + variants = {secret for config in (configs or {}).values() for secret in database_secret_variants(config)} + return tuple(sorted(variants, key=lambda item: (-len(item), item))) + + +def validated_database_config(config: Mapping[str, object] | None) -> Mapping[str, object] | None: + """Freeze a serializable explicit backend before starting evaluation work.""" + + if config is None: + return None + if not isinstance(config, Mapping): + raise UnsafeSutConfiguration("Evaluation database configuration is invalid") + kind = config.get("kind") + url = config.get("url") + prefix = {"sqlite": "sqlite+aiosqlite:///", "oceanbase": "mysql+aoceanbase://"}.get(str(kind)) + if prefix is None or not isinstance(url, str) or not url.startswith(prefix): + raise UnsafeSutConfiguration("Evaluation database configuration is invalid") + try: + json.dumps(dict(config), allow_nan=False) + if kind == "oceanbase": + parsed = urlsplit(url) + # Keep urllib's authority/query interpretation aligned with the + # database driver's URL parser; encoded password bytes are safe. + if ( + not parsed.hostname + or not parsed.path.strip("/") + or any(character in url for character in "#\r\n\t") + or "@" in (parsed.password or "") + ): + raise ValueError + # Query overrides, option files and initial SQL can change the + # credentials or target beyond the authority/path we audit. + if any( + key.casefold() in _OCEANBASE_QUERY_OVERRIDES + for key, _ in parse_qsl(parsed.query, keep_blank_values=True) + ): + raise ValueError + except (TypeError, ValueError): + raise UnsafeSutConfiguration("Evaluation database configuration is invalid") from None + return MappingProxyType(dict(config)) + + +def validated_database_configs( + configs: Mapping[Arm, Mapping[str, object]] | None, +) -> Mapping[Arm, Mapping[str, object]] | None: + """Require an explicit backend for each arm and separate OceanBase databases.""" + + if configs is None: + return None + if not isinstance(configs, Mapping) or set(configs) != {Arm.OFF, Arm.ON}: + raise UnsafeSutConfiguration("Evaluation database configuration must contain exactly off and on") + validated: dict[Arm, Mapping[str, object]] = {} + identities: list[tuple[str, int, str]] = [] + for arm in (Arm.OFF, Arm.ON): + config = validated_database_config(configs[arm]) + if config is None: + raise UnsafeSutConfiguration("Each evaluation arm requires a database configuration") + validated[arm] = config + if config["kind"] == "oceanbase": + try: + url = urlsplit(str(config["url"])) + host = (url.hostname or "").rstrip(".").lower() + try: + host = str(ipaddress.ip_address(host)) + except ValueError: + pass + identities.append((host, url.port or 3306, url.path.removeprefix("/"))) + except ValueError: + raise UnsafeSutConfiguration("Evaluation database configuration is invalid") from None + if len(identities) == 2 and identities[0] == identities[1]: + raise UnsafeSutConfiguration( + "OFF and ON require separate OceanBase databases because their processing configurations differ" + ) + return MappingProxyType(validated) + + def _directory_entry_matches(parent_fd: int, name: str, opened_fd: int) -> bool: """Confirm a directory name still identifies the descriptor-opened object.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/prediction.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/prediction.py similarity index 92% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/prediction.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/prediction.py index 29f531a231..e4e16fc73c 100644 --- a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/prediction.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/prediction.py @@ -18,10 +18,10 @@ import json -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_swebench_pro.errors import SweBenchProError -class BinaryPatchError(PowerContextEvalError): +class BinaryPatchError(SweBenchProError): """The MVP does not accept binary Git patches.""" diff --git a/evaluation/src/powercontext_eval/process.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/process.py similarity index 99% rename from evaluation/src/powercontext_eval/process.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/process.py index 2a8abadab9..6b11518421 100644 --- a/evaluation/src/powercontext_eval/process.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/process.py @@ -27,7 +27,7 @@ from typing import BinaryIO from urllib.parse import unquote, urlsplit -from powercontext_eval.errors import CommandCancelled, CommandFailed, CommandNotFound, CommandTimedOut +from powercontext_eval_swebench_pro.errors import CommandCancelled, CommandFailed, CommandNotFound, CommandTimedOut _INHERITED_ENVIRONMENT_KEYS = frozenset( { diff --git a/evaluation/scripts/record_codex_jsonl.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/record_codex_jsonl.py similarity index 100% rename from evaluation/scripts/record_codex_jsonl.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/record_codex_jsonl.py diff --git a/evaluation/src/powercontext_eval/report.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/report.py similarity index 98% rename from evaluation/src/powercontext_eval/report.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/report.py index 09f3a6d42e..baf3a6d426 100644 --- a/evaluation/src/powercontext_eval/report.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/report.py @@ -25,8 +25,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic_core import PydanticSerializationError -from powercontext_eval.artifacts import ArmState -from powercontext_eval.benchmarks.swebench_pro.gold_overrides import ( +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.gold_overrides import ( OFFICIAL_EVALUATION_DIRECT, OFFICIAL_EVALUATION_DOCKER_PROXY, OFFICIAL_EVALUATION_PROXY_BYPASSED, @@ -39,7 +39,7 @@ SOURCE595_DATASET_PATCH_SHA256, SOURCE595_INSTANCE_ID, ) -from powercontext_eval.models import Arm, TreatmentMode +from powercontext_eval_swebench_pro.models import Arm, TreatmentMode class MetricSet(BaseModel): diff --git a/evaluation/src/powercontext_eval/runner.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/runner.py similarity index 94% rename from evaluation/src/powercontext_eval/runner.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/runner.py index 0d939f2760..5a6a3f2bf6 100644 --- a/evaluation/src/powercontext_eval/runner.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/runner.py @@ -30,28 +30,27 @@ from pathlib import Path from types import MappingProxyType -from powercontext_eval.artifacts import ArmState, ArtifactStore -from powercontext_eval.benchmarks.base import GoldResult, run_after_gold -from powercontext_eval.benchmarks.swebench_pro.adapter import ( +from powercontext_eval_swebench_pro.adapter import ( DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance, ) -from powercontext_eval.benchmarks.swebench_pro.evaluator import OfficialEvaluation, OfficialEvaluator -from powercontext_eval.benchmarks.swebench_pro.gold_overrides import ( +from powercontext_eval_swebench_pro.artifacts import ArmState, ArtifactStore +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model +from powercontext_eval_swebench_pro.context_trace import write_context_trace +from powercontext_eval_swebench_pro.errors import CommandFailed +from powercontext_eval_swebench_pro.evaluator import OfficialEvaluation, OfficialEvaluator +from powercontext_eval_swebench_pro.git_source import GitSource +from powercontext_eval_swebench_pro.gold import GoldResult, run_after_gold +from powercontext_eval_swebench_pro.gold_overrides import ( OFFICIAL_EVALUATION_DIRECT, OFFICIAL_EVALUATION_DOCKER_PROXY, SOURCE595_INSTANCE_ID, select_gold_validation, ) -from powercontext_eval.benchmarks.swebench_pro.prediction import encode_predictions -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model -from powercontext_eval.context_trace import write_context_trace -from powercontext_eval.errors import CommandFailed -from powercontext_eval.git_source import GitSource -from powercontext_eval.models import Arm, PowerContextRef, TreatmentMode -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.powercontext_sut import ( +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef, TreatmentMode +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.powercontext_sut import ( DEFAULT_DOCKER_NETWORK_POOL, ArmPaths, DockerSut, @@ -59,11 +58,14 @@ SutConfig, SutOutcome, auth_secret_variants, + database_configs_secret_variants, direct_egress_environment, loopback_proxy_environment, + validated_database_configs, ) -from powercontext_eval.process import ProcessRunner -from powercontext_eval.report import ( +from powercontext_eval_swebench_pro.prediction import encode_predictions +from powercontext_eval_swebench_pro.process import ProcessRunner +from powercontext_eval_swebench_pro.report import ( ArmReport, GoldValidationAudit, MetricSet, @@ -71,7 +73,7 @@ TestGroupReport, render_report, ) -from powercontext_eval.tokensflow import ( +from powercontext_eval_swebench_pro.tokensflow import ( TokensFlowFinalizationRegistrar, TokensFlowInfrastructureError, UnsafeTokensFlowConfiguration, @@ -165,9 +167,11 @@ class RunConfig: reasoning_effort: str = DEFAULT_REASONING_EFFORT finalization_registrar: TokensFlowFinalizationRegistrar | None = None container_env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + database_configs: Mapping[Arm, Mapping[str, object]] | None = field(default=None, repr=False) cancel_event: threading.Event | None = field(default=None, repr=False, compare=False) def __post_init__(self) -> None: + object.__setattr__(self, "database_configs", validated_database_configs(self.database_configs)) if isinstance(self.treatment_mode, str): object.__setattr__(self, "treatment_mode", TreatmentMode(self.treatment_mode)) if not is_safe_codex_model(self.model): @@ -358,7 +362,9 @@ def _run_swebench_pro_instance( selected_arms = config.treatment_mode.arms def arms() -> tuple[dict[Arm, OfficialEvaluation], Mapping[Arm, SutOutcome], dict[Arm, int]]: - codex_secrets = auth_secret_variants(config.auth_json) + codex_secrets = auth_secret_variants(config.auth_json) + database_configs_secret_variants( + config.database_configs + ) arm_paths: dict[Arm, ArmPaths] = {} stores: dict[Arm, ArtifactStore] = {} for arm in selected_arms: @@ -409,6 +415,7 @@ def arms() -> tuple[dict[Arm, OfficialEvaluation], Mapping[Arm, SutOutcome], dic reasoning_effort=config.reasoning_effort, finalization_registrar=config.finalization_registrar, container_env=config.container_env, + database_configs=config.database_configs, ) sut = DockerSut(process) if config.treatment_mode is TreatmentMode.OFF_ON: diff --git a/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/service_cli.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/service_cli.py new file mode 100644 index 0000000000..f120e7a81e --- /dev/null +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/service_cli.py @@ -0,0 +1,156 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""SWE-bench Pro web, worker, and runtime contract commands.""" + +from __future__ import annotations + +import json +import os +import signal +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import timedelta +from pathlib import Path +from types import FrameType +from typing import TYPE_CHECKING, Annotated, Any, Protocol + +import typer +from pydantic import ValidationError + +if TYPE_CHECKING: + from powercontext_eval_swebench_pro.web.config import WebConfig + + +class _Stoppable(Protocol): + def stop(self) -> None: ... + + +def run_codex_contract_smoke(**kwargs: Any) -> Any: + """Load the SWE-bench Pro contract runner only when its command is used.""" + + from powercontext_eval_swebench_pro.powercontext_sut import run_codex_contract_smoke as implementation + + return implementation(**kwargs) + + +def _request_worker_stop(worker: _Stoppable, _signum: int, _frame: FrameType | None) -> None: + """Request that a worker exit after its current task finishes.""" + worker.stop() + + +def _web_config(root_path: Path | None) -> WebConfig: + from powercontext_eval_swebench_pro.web.config import WebConfig + + try: + environ = dict(os.environ) + if root_path is not None: + environ["POWERCONTEXT_EVAL_ROOT"] = os.fspath(root_path) + return WebConfig.from_environment(environ) + except (KeyError, TypeError, ValueError, ValidationError): + raise typer.BadParameter("Invalid evaluation configuration.", param_hint="--root") from None + + +@contextmanager +def _worker_signal_handlers(worker: _Stoppable) -> Iterator[None]: + previous: dict[signal.Signals, Any] = {} + stop_requested = False + + def handler(signum: int, frame: FrameType | None) -> None: + nonlocal stop_requested + if stop_requested: + return + stop_requested = True + _request_worker_stop(worker, signum, frame) + + try: + for signum in (signal.SIGTERM, signal.SIGINT): + previous[signum] = signal.getsignal(signum) + signal.signal(signum, handler) + yield + finally: + for signum, prior in previous.items(): + signal.signal(signum, prior) + + +def web(root_path: Annotated[Path | None, typer.Option("--root")] = None) -> None: + """Serve the evaluation console API and frontend.""" + import uvicorn + + from powercontext_eval_swebench_pro.web.api import create_app + + config = _web_config(root_path) + uvicorn.run(create_app(config), host=config.host, port=config.port) + + +def worker(root_path: Annotated[Path | None, typer.Option("--root")] = None) -> None: + """Run queued task pairs at configured parallelism until shutdown is requested.""" + from powercontext_eval_swebench_pro.web.store import TaskStore + from powercontext_eval_swebench_pro.web.usage import CodexUsageProbe + from powercontext_eval_swebench_pro.web.worker import EvaluationWorker + + config = _web_config(root_path) + store = TaskStore( + config.database_path, + lease_duration=timedelta(seconds=config.lease_seconds), + max_attempts=config.max_attempts, + ) + store.initialize() + service = EvaluationWorker( + config, + store, + usage_probe=CodexUsageProbe( + codex_binary=config.codex_binary, + auth_json=config.auth_json, + codex_config=config.codex_config, + proxy_url=config.proxy_url, + timeout_seconds=config.usage_probe_timeout_seconds, + ), + ) + with _worker_signal_handlers(service): + service.run_forever() + + +def codex_contract_smoke( + run_root: str = typer.Option(...), + task_image: str = typer.Option(...), + codex_bin: str = typer.Option(...), + tokensflow_bin: str = typer.Option(...), + tokensflow_user_home: str = typer.Option(...), + tokensflow_egress_network: str = typer.Option(...), + uv_bin: str = typer.Option(...), + powercontext_source: str = typer.Option(...), + powercontext_sha: str = typer.Option(...), + auth_json: str = typer.Option(...), + proxy_url: str = typer.Option(...), + prompt: str = typer.Option("Reply with exactly OK."), +) -> None: + """Run OFF/ON identity, daemon, bounded-drain, and Codex contract checks.""" + + outcome = run_codex_contract_smoke( + run_root=run_root, + task_image=task_image, + codex_bin=codex_bin, + tokensflow_bin=tokensflow_bin, + tokensflow_user_home=tokensflow_user_home, + tokensflow_egress_network=tokensflow_egress_network, + uv_bin=uv_bin, + powercontext_source=powercontext_source, + powercontext_sha=powercontext_sha, + auth_json=auth_json, + proxy_url=proxy_url, + prompt=prompt, + ) + typer.echo(json.dumps(outcome, ensure_ascii=False, sort_keys=True)) diff --git a/evaluation/src/powercontext_eval/tokensflow.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/tokensflow.py similarity index 98% rename from evaluation/src/powercontext_eval/tokensflow.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/tokensflow.py index a12778e5aa..95b0a99a6d 100644 --- a/evaluation/src/powercontext_eval/tokensflow.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/tokensflow.py @@ -30,15 +30,15 @@ from pathlib import Path from urllib.parse import quote, quote_plus -from powercontext_eval.errors import PowerContextEvalError -from powercontext_eval.models import Arm +from powercontext_eval_swebench_pro.errors import SweBenchProError +from powercontext_eval_swebench_pro.models import Arm -class UnsafeTokensFlowConfiguration(PowerContextEvalError): +class UnsafeTokensFlowConfiguration(SweBenchProError): """A TokensFlow profile or destination is missing or unsafe.""" -class TokensFlowInfrastructureError(PowerContextEvalError): +class TokensFlowInfrastructureError(SweBenchProError): """TokensFlow could not prove a safe pre-inference runtime.""" diff --git a/evaluation/src/powercontext_eval/web/__init__.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/__init__.py similarity index 90% rename from evaluation/src/powercontext_eval/web/__init__.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/__init__.py index da5cbf933c..ea6ecc4e9f 100644 --- a/evaluation/src/powercontext_eval/web/__init__.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/__init__.py @@ -14,8 +14,8 @@ """Public domain types for the evaluation console.""" -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.models import ( Capabilities, FailureCategory, HealthResponse, diff --git a/evaluation/src/powercontext_eval/web/api.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/api.py similarity index 97% rename from evaluation/src/powercontext_eval/web/api.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/api.py index 4b331cb9d4..69c3b4bfd3 100644 --- a/evaluation/src/powercontext_eval/web/api.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/api.py @@ -34,23 +34,23 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, PlainTextResponse, Response, StreamingResponse -from powercontext_eval.benchmarks.swebench_pro.catalog import ( +from powercontext_eval_swebench_pro.catalog import ( CatalogError, SweBenchProCatalog, instance_ids_for_task_set, ) -from powercontext_eval.codex import DEFAULT_REASONING_EFFORT -from powercontext_eval.errors import GitSourceError -from powercontext_eval.git_source import GitSource -from powercontext_eval.models import Arm, PowerContextRef -from powercontext_eval.web.baselines import ( +from powercontext_eval_swebench_pro.codex import DEFAULT_REASONING_EFFORT +from powercontext_eval_swebench_pro.errors import GitSourceError +from powercontext_eval_swebench_pro.git_source import GitSource +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef +from powercontext_eval_swebench_pro.web.baselines import ( BaselineCandidate, BaselineComparisonResponse, BaselineCreate, BaselineSelectionUpdate, CompatibilityStatus, ) -from powercontext_eval.web.batches import ( +from powercontext_eval_swebench_pro.web.batches import ( BatchCreate, BatchPreviewResponse, BatchRecord, @@ -60,11 +60,18 @@ PairCategory, TaskRetryRequest, ) -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.controls import BatchControlPatch, BatchPauseReason, BatchPreviewRequest -from powercontext_eval.web.estimation import BatchEstimate, EstimateBasis, estimate_batch -from powercontext_eval.web.models import Capabilities, HealthResponse, TaskCreate, TaskRecord, TaskStatus, TaskSummary -from powercontext_eval.web.reporting import ( +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.controls import BatchControlPatch, BatchPauseReason, BatchPreviewRequest +from powercontext_eval_swebench_pro.web.estimation import BatchEstimate, EstimateBasis, estimate_batch +from powercontext_eval_swebench_pro.web.models import ( + Capabilities, + HealthResponse, + TaskCreate, + TaskRecord, + TaskStatus, + TaskSummary, +) +from powercontext_eval_swebench_pro.web.reporting import ( BenchmarkCatalog, InvalidReportArtifact, ReportingError, @@ -83,9 +90,9 @@ load_report, task_run_dir, ) -from powercontext_eval.web.resources import FilesystemResourceProbe, ResourceProbe, ResourceUnavailable -from powercontext_eval.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision -from powercontext_eval.web.store import ( +from powercontext_eval_swebench_pro.web.resources import FilesystemResourceProbe, ResourceProbe, ResourceUnavailable +from powercontext_eval_swebench_pro.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision +from powercontext_eval_swebench_pro.web.store import ( BaselineNotFound, BatchNotFound, TaskAdmissionRejected, @@ -93,7 +100,7 @@ TaskNotFound, TaskStore, ) -from powercontext_eval.web.usage import AccountUsage, UsageSnapshot, is_fresh +from powercontext_eval_swebench_pro.web.usage import AccountUsage, UsageSnapshot, is_fresh _TERMINAL = {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.INTERRUPTED, TaskStatus.CANCELLED} _NO_STORE = {"Cache-Control": "no-store"} diff --git a/evaluation/src/powercontext_eval/web/baselines.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/baselines.py similarity index 97% rename from evaluation/src/powercontext_eval/web/baselines.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/baselines.py index 8e3b336bba..57dc2564a3 100644 --- a/evaluation/src/powercontext_eval/web/baselines.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/baselines.py @@ -22,9 +22,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from powercontext_eval.benchmarks.swebench_pro.catalog import TaskSet -from powercontext_eval.models import Arm -from powercontext_eval.web.models import TaskStatus +from powercontext_eval_swebench_pro.catalog import TaskSet +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.web.models import TaskStatus class _FrozenModel(BaseModel): diff --git a/evaluation/src/powercontext_eval/web/batches.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/batches.py similarity index 95% rename from evaluation/src/powercontext_eval/web/batches.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/batches.py index 1ab24e00e1..80280d6cb1 100644 --- a/evaluation/src/powercontext_eval/web/batches.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/batches.py @@ -22,13 +22,13 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator -from powercontext_eval.benchmarks.swebench_pro.catalog import TaskSet -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model -from powercontext_eval.models import PowerContextRef, TreatmentMode -from powercontext_eval.web.controls import BatchControlState -from powercontext_eval.web.estimation import BatchEstimate -from powercontext_eval.web.models import FailureCategory, FailureCode, TaskPhase, TaskStatus -from powercontext_eval.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.catalog import TaskSet +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model +from powercontext_eval_swebench_pro.models import PowerContextRef, TreatmentMode +from powercontext_eval_swebench_pro.web.controls import BatchControlState +from powercontext_eval_swebench_pro.web.estimation import BatchEstimate +from powercontext_eval_swebench_pro.web.models import FailureCategory, FailureCode, TaskPhase, TaskStatus +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot class _FrozenModel(BaseModel): diff --git a/evaluation/src/powercontext_eval/web/claiming.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/claiming.py similarity index 95% rename from evaluation/src/powercontext_eval/web/claiming.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/claiming.py index 8dda18b3fb..fe9ead1625 100644 --- a/evaluation/src/powercontext_eval/web/claiming.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/claiming.py @@ -21,9 +21,9 @@ from datetime import datetime, timedelta from typing import Protocol -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.models import TaskRecord -from powercontext_eval.web.resources import ( +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.models import TaskRecord +from powercontext_eval_swebench_pro.web.resources import ( DependencyProbe, DockerDependencyProbe, FilesystemCapacity, @@ -31,8 +31,8 @@ ResourceProbe, ResourceUnavailable, ) -from powercontext_eval.web.store import TaskStore -from powercontext_eval.web.usage import UsageSnapshot, UsageUnavailable, is_fresh +from powercontext_eval_swebench_pro.web.store import TaskStore +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot, UsageUnavailable, is_fresh class UsageProbe(Protocol): diff --git a/evaluation/src/powercontext_eval/web/config.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/config.py similarity index 98% rename from evaluation/src/powercontext_eval/web/config.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/config.py index 778d8d9877..a5e3b9c49c 100644 --- a/evaluation/src/powercontext_eval/web/config.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/config.py @@ -24,8 +24,12 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, is_safe_codex_model -from powercontext_eval.powercontext_sut import DEFAULT_DOCKER_NETWORK_POOL, ProxyRelayConfig, UnsafeSutConfiguration +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, is_safe_codex_model +from powercontext_eval_swebench_pro.powercontext_sut import ( + DEFAULT_DOCKER_NETWORK_POOL, + ProxyRelayConfig, + UnsafeSutConfiguration, +) _SAFE_DOCKER_NETWORK = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") _SAFE_NO_PROXY_HOST = re.compile(r"(?:[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?|\[[0-9A-Fa-f:]+\])") @@ -271,7 +275,8 @@ def for_root( root=root, database_path=database_path or root / "web" / "tasks.sqlite3", run_root=run_root or root, - frontend_dist=frontend_dist or root / "deploy" / "powercontext" / "evaluation" / "web" / "dist", + frontend_dist=frontend_dist + or root / "deploy" / "powercontext" / "evaluation" / "coding" / "swebench_pro" / "web" / "dist", powercontext_source=powercontext_source or root / "source" / "powercontext.git", harness_root=harness_root or root / "cache" / "swebench-pro.git", harness_python=harness_python or root / "venvs" / "swebench-pro-ca10a60" / "bin" / "python", diff --git a/evaluation/src/powercontext_eval/web/controls.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/controls.py similarity index 91% rename from evaluation/src/powercontext_eval/web/controls.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/controls.py index 3f8a8e1f1f..9c9efce089 100644 --- a/evaluation/src/powercontext_eval/web/controls.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/controls.py @@ -22,13 +22,13 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator -from powercontext_eval.benchmarks.swebench_pro.catalog import PUBLIC_V2_TASK_SET, TaskSet -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, is_safe_codex_model -from powercontext_eval.models import PowerContextRef, TreatmentMode -from powercontext_eval.web.models import TaskStatus +from powercontext_eval_swebench_pro.catalog import PUBLIC_V2_TASK_SET, TaskSet +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, is_safe_codex_model +from powercontext_eval_swebench_pro.models import PowerContextRef, TreatmentMode +from powercontext_eval_swebench_pro.web.models import TaskStatus if TYPE_CHECKING: - from powercontext_eval.web.batches import BatchStatus + from powercontext_eval_swebench_pro.web.batches import BatchStatus class _FrozenModel(BaseModel): @@ -116,7 +116,7 @@ def derive_controlled_batch_status( ) -> BatchStatus: """Derive the visible batch lifecycle from operator intent and child tasks.""" - from powercontext_eval.web.batches import BatchStatus + from powercontext_eval_swebench_pro.web.batches import BatchStatus if not task_statuses: raise ValueError("A batch must contain at least one task") diff --git a/evaluation/src/powercontext_eval/web/estimation.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/estimation.py similarity index 100% rename from evaluation/src/powercontext_eval/web/estimation.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/estimation.py diff --git a/evaluation/src/powercontext_eval/web/finalization.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/finalization.py similarity index 98% rename from evaluation/src/powercontext_eval/web/finalization.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/finalization.py index 9d897edd5c..5adc370d65 100644 --- a/evaluation/src/powercontext_eval/web/finalization.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/finalization.py @@ -30,12 +30,12 @@ from typing import Protocol, cast from uuid import uuid4 -from powercontext_eval.errors import CommandError -from powercontext_eval.powercontext_sut import ArmPaths, DockerSut, UnsafeSutConfiguration -from powercontext_eval.process import CommandResult, ProcessRunner -from powercontext_eval.tokensflow import tokensflow_queue_caught_up -from powercontext_eval.web.config import MAX_TASK_PARALLELISM -from powercontext_eval.web.store import ( +from powercontext_eval_swebench_pro.errors import CommandError +from powercontext_eval_swebench_pro.powercontext_sut import ArmPaths, DockerSut, UnsafeSutConfiguration +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.tokensflow import tokensflow_queue_caught_up +from powercontext_eval_swebench_pro.web.config import MAX_TASK_PARALLELISM +from powercontext_eval_swebench_pro.web.store import ( FinalizationState, TaskOwnershipError, TaskStore, diff --git a/evaluation/src/powercontext_eval/web/models.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/models.py similarity index 95% rename from evaluation/src/powercontext_eval/web/models.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/models.py index f1f25c3407..ab69d51196 100644 --- a/evaluation/src/powercontext_eval/web/models.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/models.py @@ -22,11 +22,11 @@ from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator -from powercontext_eval.artifacts import ArmState -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model -from powercontext_eval.models import Arm, PowerContextRef, TreatmentMode -from powercontext_eval.report import GoldValidationAudit -from powercontext_eval.runner import INSTANCE_ID +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT, is_safe_codex_model +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef, TreatmentMode +from powercontext_eval_swebench_pro.report import GoldValidationAudit +from powercontext_eval_swebench_pro.runner import INSTANCE_ID class StrictModel(BaseModel): @@ -404,6 +404,14 @@ class TreatmentEvidence(FrozenModel): scope_id: str scope_key: Annotated[str, Field(min_length=1)] | None = None server_ready: bool + database_kind: Literal["sqlite", "oceanbase"] | None = None + database_fingerprint: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$", max_length=64)] | None = None + + @model_validator(mode="after") + def require_complete_database_identity(self) -> Self: + if (self.database_kind is None) != (self.database_fingerprint is None): + raise ValueError("Treatment database backend and fingerprint must be recorded together") + return self class EvidenceResponse(FrozenModel): diff --git a/evaluation/src/powercontext_eval/web/reporting.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/reporting.py similarity index 98% rename from evaluation/src/powercontext_eval/web/reporting.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/reporting.py index 46de72e64c..f636840ff3 100644 --- a/evaluation/src/powercontext_eval/web/reporting.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/reporting.py @@ -28,11 +28,11 @@ from pydantic import ValidationError -from powercontext_eval.artifacts import ArmState -from powercontext_eval.benchmarks.swebench_pro.adapter import DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance -from powercontext_eval.models import Arm, TreatmentMode, arm_scope_key -from powercontext_eval.report import ArmReport, ReportBundle, TestGroupReport -from powercontext_eval.web.baselines import ( +from powercontext_eval_swebench_pro.adapter import DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.models import Arm, TreatmentMode, arm_scope_key +from powercontext_eval_swebench_pro.report import ArmReport, ReportBundle, TestGroupReport +from powercontext_eval_swebench_pro.web.baselines import ( BaselineComparison, BaselineCompatibility, BaselineItemRecord, @@ -43,7 +43,7 @@ HistoricalResolutionComparison, HistoricalTokenComparison, ) -from powercontext_eval.web.batches import ( +from powercontext_eval_swebench_pro.web.batches import ( BatchRecord, BatchReportResponse, BatchStatus, @@ -64,8 +64,8 @@ TokensFlowFinalizationPair, TokensFlowFinalizationSummary, ) -from powercontext_eval.web.estimation import BatchEstimate, EstimateBasis, EstimateSample, estimate_batch -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.web.estimation import BatchEstimate, EstimateBasis, EstimateSample, estimate_batch +from powercontext_eval_swebench_pro.web.models import ( ArmResponse, ComparisonResponse, EvidenceResponse, @@ -75,8 +75,8 @@ TaskStatus, TreatmentEvidence, ) -from powercontext_eval.web.store import FinalizationState, TokensFlowFinalizationRecord -from powercontext_eval.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.web.store import FinalizationState, TokensFlowFinalizationRecord +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot _REPORT_JSON_LIMIT = 1024 * 1024 _REPORT_MARKDOWN_LIMIT = 4 * 1024 * 1024 diff --git a/evaluation/src/powercontext_eval/web/resources.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/resources.py similarity index 97% rename from evaluation/src/powercontext_eval/web/resources.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/resources.py index 4f6220df49..0ca78b1e98 100644 --- a/evaluation/src/powercontext_eval/web/resources.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/resources.py @@ -29,14 +29,14 @@ from pathlib import Path from typing import Any, Protocol -from powercontext_eval.artifacts import ArtifactStore -from powercontext_eval.errors import CommandError -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.process import CommandResult, ProcessRunner -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.models import TaskRecord -from powercontext_eval.web.reporting import load_report -from powercontext_eval.web.store import AttemptCleanupCandidate, TaskStore +from powercontext_eval_swebench_pro.artifacts import ArtifactStore +from powercontext_eval_swebench_pro.errors import CommandError +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.models import TaskRecord +from powercontext_eval_swebench_pro.web.reporting import load_report +from powercontext_eval_swebench_pro.web.store import AttemptCleanupCandidate, TaskStore _LOGGER = logging.getLogger(__name__) _ATTEMPT_CLEANUP_RETRY_SECONDS = 30 diff --git a/evaluation/src/powercontext_eval/web/revision.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/revision.py similarity index 100% rename from evaluation/src/powercontext_eval/web/revision.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/revision.py diff --git a/evaluation/src/powercontext_eval/web/store.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/store.py similarity index 99% rename from evaluation/src/powercontext_eval/web/store.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/store.py index 5594734ace..e146108b7f 100644 --- a/evaluation/src/powercontext_eval/web/store.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/store.py @@ -28,35 +28,35 @@ from re import fullmatch from typing import Any, Literal, TypedDict, cast -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.web.baselines import ( +from powercontext_eval_swebench_pro.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.web.baselines import ( BaselineCreate, BaselineItemRecord, BaselineRecord, BaselineSelection, BaselineSnapshot, ) -from powercontext_eval.web.batches import ( +from powercontext_eval_swebench_pro.web.batches import ( BatchControlEvent, BatchControlEventType, BatchCreate, BatchRecord, BatchStatus, ) -from powercontext_eval.web.config import ( +from powercontext_eval_swebench_pro.web.config import ( DEFAULT_MAX_ATTEMPTS, MAX_ATTEMPTS_LIMIT, MAX_TASK_PARALLELISM, MAX_TOKENSFLOW_FINALIZER_TIMEOUT_SECONDS, ) -from powercontext_eval.web.controls import ( +from powercontext_eval_swebench_pro.web.controls import ( BatchControlIntent, BatchControlState, BatchPauseReason, derive_controlled_batch_status, ) -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.web.models import ( FailureCategory, FailureCode, RetryDisposition, @@ -69,7 +69,7 @@ TaskStatus, TaskSummary, ) -from powercontext_eval.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot _RETRY_BACKOFF_SECONDS = (30, 120, 300, 600) # Some Gold suites contain wall-clock tests with a deterministic two-hour bad window. diff --git a/evaluation/src/powercontext_eval/web/usage.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/usage.py similarity index 99% rename from evaluation/src/powercontext_eval/web/usage.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/usage.py index 56c1995dda..c239f76af4 100644 --- a/evaluation/src/powercontext_eval/web/usage.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/usage.py @@ -32,7 +32,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator -from powercontext_eval.process import build_process_environment +from powercontext_eval_swebench_pro.process import build_process_environment CLIENT_INFO = { "name": "powercontext_eval", diff --git a/evaluation/src/powercontext_eval/web/worker.py b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/worker.py similarity index 94% rename from evaluation/src/powercontext_eval/web/worker.py rename to evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/worker.py index 694f5f08eb..65b0d19af2 100644 --- a/evaluation/src/powercontext_eval/web/worker.py +++ b/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/web/worker.py @@ -29,26 +29,26 @@ from typing import Protocol from uuid import uuid4 -from powercontext_eval.artifacts import ArtifactError -from powercontext_eval.benchmarks.base import GoldCheckFailed -from powercontext_eval.benchmarks.swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance -from powercontext_eval.benchmarks.swebench_pro.catalog import CatalogError, SweBenchProCatalog -from powercontext_eval.benchmarks.swebench_pro.evaluator import OfficialResultError -from powercontext_eval.benchmarks.swebench_pro.prediction import BinaryPatchError -from powercontext_eval.codex import CodexCapacityError, CodexInfrastructureError, UnsafeCodexInvocation -from powercontext_eval.errors import CommandCancelled, CommandError, GitSourceError, PowerContextEvalError -from powercontext_eval.git_source import GitSource -from powercontext_eval.models import Arm, PowerContextRef -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.powercontext_sut import ( +from powercontext_eval_swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance +from powercontext_eval_swebench_pro.artifacts import ArtifactError +from powercontext_eval_swebench_pro.catalog import CatalogError, SweBenchProCatalog +from powercontext_eval_swebench_pro.codex import CodexCapacityError, CodexInfrastructureError, UnsafeCodexInvocation +from powercontext_eval_swebench_pro.errors import CommandCancelled, CommandError, GitSourceError, SweBenchProError +from powercontext_eval_swebench_pro.evaluator import OfficialResultError +from powercontext_eval_swebench_pro.git_source import GitSource +from powercontext_eval_swebench_pro.gold import GoldCheckFailed +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.powercontext_sut import ( InvalidTreatment, PluginInspectionFailure, ReadinessFailure, UnsafeSutConfiguration, ) -from powercontext_eval.process import ProcessRunner -from powercontext_eval.report import InvalidReportBundle -from powercontext_eval.runner import ( +from powercontext_eval_swebench_pro.prediction import BinaryPatchError +from powercontext_eval_swebench_pro.process import ProcessRunner +from powercontext_eval_swebench_pro.report import InvalidReportBundle +from powercontext_eval_swebench_pro.runner import ( MinimalRunConfig, MinimalRunResult, RunConfig, @@ -56,11 +56,11 @@ run_minimal_swebench_pro, run_swebench_pro_instance, ) -from powercontext_eval.tokensflow import TokensFlowFinalizationDescriptor, TokensFlowFinalizationRegistrar -from powercontext_eval.web.claiming import ClaimCoordinator, PeriodicUsageRefresher -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.finalization import DockerFinalizationRuntime, TokensFlowFinalizer -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.tokensflow import TokensFlowFinalizationDescriptor, TokensFlowFinalizationRegistrar +from powercontext_eval_swebench_pro.web.claiming import ClaimCoordinator, PeriodicUsageRefresher +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.finalization import DockerFinalizationRuntime, TokensFlowFinalizer +from powercontext_eval_swebench_pro.web.models import ( FailureCategory, FailureCode, RetryDisposition, @@ -69,16 +69,20 @@ TaskRecord, TaskResult, ) -from powercontext_eval.web.reporting import ReportingError, load_report -from powercontext_eval.web.resources import AttemptLifecycleCleaner, ResourceProbe, default_workspace_reclaimer -from powercontext_eval.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision -from powercontext_eval.web.store import ( +from powercontext_eval_swebench_pro.web.reporting import ReportingError, load_report +from powercontext_eval_swebench_pro.web.resources import ( + AttemptLifecycleCleaner, + ResourceProbe, + default_workspace_reclaimer, +) +from powercontext_eval_swebench_pro.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision +from powercontext_eval_swebench_pro.web.store import ( TaskConflict, TaskOwnershipError, TaskStore, TokensFlowFinalizationCreate, ) -from powercontext_eval.web.usage import CodexUsageProbe, UsageSnapshot +from powercontext_eval_swebench_pro.web.usage import CodexUsageProbe, UsageSnapshot _INTERNAL_SUMMARY = "The evaluation worker failed unexpectedly. Inspect the retained worker logs." _REPORT_SUMMARY = "Evaluation report validation failed." @@ -882,7 +886,7 @@ def _safe_failure(error: Exception, phase: TaskPhase | None) -> SafeFailure: _REPORT_SUMMARY, RetryDisposition.RETRY, ) - elif isinstance(error, (CommandError, PowerContextEvalError)): + elif isinstance(error, (CommandError, SweBenchProError)): category, summary = _phase_failure(phase) fixed = category, _phase_failure_code(phase), summary, RetryDisposition.RETRY else: diff --git a/evaluation/tests/contract/__init__.py b/evaluation/coding/swebench_pro/tests/contract/__init__.py similarity index 100% rename from evaluation/tests/contract/__init__.py rename to evaluation/coding/swebench_pro/tests/contract/__init__.py diff --git a/evaluation/tests/contract/fixtures/fake_codex.py b/evaluation/coding/swebench_pro/tests/contract/fixtures/fake_codex.py similarity index 100% rename from evaluation/tests/contract/fixtures/fake_codex.py rename to evaluation/coding/swebench_pro/tests/contract/fixtures/fake_codex.py diff --git a/evaluation/tests/contract/fixtures/fake_evaluator.py b/evaluation/coding/swebench_pro/tests/contract/fixtures/fake_evaluator.py similarity index 100% rename from evaluation/tests/contract/fixtures/fake_evaluator.py rename to evaluation/coding/swebench_pro/tests/contract/fixtures/fake_evaluator.py diff --git a/evaluation/tests/contract/fixtures/swebench_pro_public_v2.jsonl b/evaluation/coding/swebench_pro/tests/contract/fixtures/swebench_pro_public_v2.jsonl similarity index 100% rename from evaluation/tests/contract/fixtures/swebench_pro_public_v2.jsonl rename to evaluation/coding/swebench_pro/tests/contract/fixtures/swebench_pro_public_v2.jsonl diff --git a/evaluation/tests/contract/helpers.py b/evaluation/coding/swebench_pro/tests/contract/helpers.py similarity index 100% rename from evaluation/tests/contract/helpers.py rename to evaluation/coding/swebench_pro/tests/contract/helpers.py diff --git a/evaluation/tests/contract/test_codex_contract.py b/evaluation/coding/swebench_pro/tests/contract/test_codex_contract.py similarity index 93% rename from evaluation/tests/contract/test_codex_contract.py rename to evaluation/coding/swebench_pro/tests/contract/test_codex_contract.py index 6b56f275b7..11de5615dd 100644 --- a/evaluation/tests/contract/test_codex_contract.py +++ b/evaluation/coding/swebench_pro/tests/contract/test_codex_contract.py @@ -36,18 +36,18 @@ import pytest -from powercontext_eval import powercontext_sut -from powercontext_eval.artifacts import ArtifactStore, SecretDetected -from powercontext_eval.codex import ( +from powercontext_eval_swebench_pro import powercontext_sut +from powercontext_eval_swebench_pro.artifacts import ArtifactStore, SecretDetected +from powercontext_eval_swebench_pro.codex import ( CodexCapacityError, CodexInfrastructureError, CodexInvocation, CodexRunner, UnsafeCodexInvocation, ) -from powercontext_eval.errors import CommandError -from powercontext_eval.models import Arm -from powercontext_eval.powercontext_sut import ( +from powercontext_eval_swebench_pro.errors import CommandError +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.powercontext_sut import ( LOOPBACK_NO_PROXY, ArmPaths, ContainerLimits, @@ -68,8 +68,8 @@ run_codex_contract_smoke, validate_treatment, ) -from powercontext_eval.process import CommandFailed, CommandResult, CommandTimedOut, ProcessRunner -from powercontext_eval.tokensflow import ( +from powercontext_eval_swebench_pro.process import CommandFailed, CommandResult, CommandTimedOut, ProcessRunner +from powercontext_eval_swebench_pro.tokensflow import ( DrainDeadline, TokensFlowDaemonHandle, TokensFlowFinalizationDescriptor, @@ -110,13 +110,13 @@ def test_timestamp_recorder_wraps_but_does_not_change_the_codex_command() -> Non arm=Arm.ON, inside_disposable_container=True, recorder_python="/runtime/pc-env/bin/python", - recorder_script="/source/evaluation/scripts/record_codex_jsonl.py", + recorder_script="/source/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/record_codex_jsonl.py", recorder_sidecar="/runtime/pc-home/codex-observed.jsonl", ).argv() assert wrapped[:5] == ( "/runtime/pc-env/bin/python", - "/source/evaluation/scripts/record_codex_jsonl.py", + "/source/evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/record_codex_jsonl.py", "--sidecar", "/runtime/pc-home/codex-observed.jsonl", "--", @@ -344,6 +344,18 @@ def test_treatment_evidence_accepts_valid_on_and_off() -> None: ) +def test_treatment_evidence_preserves_database_identity_and_reads_legacy_records() -> None: + legacy = evidence().as_dict() + legacy.pop("database_kind") + legacy.pop("database_fingerprint") + restored = TreatmentEvidence.from_json(json.dumps(legacy)) + assert restored.database_kind is None + assert restored.database_fingerprint is None + + current = evidence(database_kind="oceanbase", database_fingerprint="d" * 64) + assert TreatmentEvidence.from_json(json.dumps(current.as_dict())) == current + + @pytest.mark.parametrize( ("arm", "changes"), [ @@ -411,6 +423,8 @@ def __init__( container_identity: bytes | None = None, host_tokensflow_version: bytes = b"tokensflow 1.0.16\n", container_tokensflow_version: bytes | None = None, + database_kind: str = "sqlite", + database_fingerprint: str = "d" * 64, ) -> None: self.commands: list[tuple[str, ...]] = [] self.fail_at = fail_at @@ -422,6 +436,8 @@ def __init__( container_tokensflow_version if container_tokensflow_version is not None else host_tokensflow_version ) self.container_networks: dict[str, set[str]] = {} + self.database_kind = database_kind + self.database_fingerprint = database_fingerprint @staticmethod def _output(payload: bytes, kwargs: dict[str, object], *, returncode: int = 0) -> CommandResult: @@ -501,7 +517,15 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: return command_result(json.dumps({"scope_id": f"scp_fixture_{key.rsplit(':', 1)[-1]}"})) if "evidence" in argv: scope = argv[argv.index("evidence") - 1] - return command_result(json.dumps({"prompt_sources": 0 if scope.endswith("_off") else 1})) + return command_result( + json.dumps( + { + "prompt_sources": 0 if scope.endswith("_off") else 1, + "database_kind": self.database_kind, + "database_fingerprint": self.database_fingerprint, + } + ) + ) if any(part.endswith("/codex") or part == "codex" for part in argv) and "exec" in argv: result = command_result( '{"type":"agent_message","message":"done"}\n' @@ -1046,7 +1070,8 @@ def test_sut_transcript_has_hardening_mount_allowlist_shared_network_and_scope(t prewarm_index = next( index for index, command in enumerate(transcript) - if command[-8:] == ("sync", "--frozen", "--project", "/source", "--extra", "server", "--extra", "cli") + if command[-9:] + == ("sync", "--frozen", "--no-dev", "--project", "/source", "--extra", "server", "--extra", "cli") ) assert not any("/bin/chown" in command for command in transcript[:prewarm_index]) @@ -1097,6 +1122,175 @@ def _is_codex_inference(command: tuple[str, ...]) -> bool: ) +@pytest.mark.parametrize("password", ['test-pass"marker', "test-pass\\marker", "test-pass-中文"]) +def test_database_credentials_in_environment_output_are_rejected_from_codex_jsonl( + tmp_path: Path, password: str +) -> None: + database = { + "kind": "oceanbase", + "url": f"mysql+aoceanbase://eval:{password}@database.test:2881/eval-only?charset=utf8mb4", + } + environment_output = "POWERCONTEXT_SERVER_DATABASE=" + json.dumps(database) + event = json.dumps( + { + "type": "item.completed", + "item": {"type": "command_execution", "aggregated_output": environment_output}, + } + ) + store = ArtifactStore(tmp_path, forbidden_values=powercontext_sut.database_secret_variants(database)) + + with pytest.raises(SecretDetected): + store.write_text("codex/output.jsonl", event) + + +@pytest.mark.parametrize("off_backend", ["sqlite", "oceanbase"]) +def test_each_arm_receives_its_database_without_exposing_credentials(tmp_path: Path, off_backend: str) -> None: + databases = { + Arm.OFF: { + "kind": off_backend, + "url": ( + "sqlite+aiosqlite:////runtime/pc-home/powercontext.db" + if off_backend == "sqlite" + else "mysql+aoceanbase://eval-user:off-database-secret@database.test:2881/eval-off" + ), + }, + Arm.ON: { + "kind": "oceanbase", + "url": "mysql+aoceanbase://eval-user:on-database-secret@database.test:2881/eval-on", + }, + } + model_secret = "on-only-model-secret" + environment_files: list[Path] = [] + registered_scopes: list[str] = [] + + class EnvironmentFileDocker(TranscriptDocker): + def __init__(self, backend: str) -> None: + super().__init__(database_kind=backend) + self.environment: dict[str, str] = {} + self.codex_secrets: Sequence[str] = () + + def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: + if argv[:3] == ("docker", "run", "-d"): + environment_file = Path(argv[argv.index("--env-file") + 1]) + environment_files.append(environment_file) + assert stat.S_IMODE(environment_file.stat().st_mode) == 0o600 + self.environment = dict(line.split("=", 1) for line in environment_file.read_text().splitlines()) + if _is_codex_inference(argv): + self.codex_secrets = cast(Sequence[str], kwargs["secrets"]) + return super().run(argv, **kwargs) + + for arm in (Arm.OFF, Arm.ON): + root = tmp_path / arm.value + paths = make_paths(root) + config = replace( + sut_config(root), + proxy=None, + tokensflow_enabled=False, + tokensflow_binary=None, + tokensflow_egress_network=None, + database_configs=databases, + container_env={ + "OPENAI_API_KEY": model_secret, + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL": "openai:fixture-model", + "POWERCONTEXT_SERVER_DATABASE_KIND": "sqlite", + "POWERCONTEXT_SERVER_DATABASE_URL": "sqlite+aiosqlite:////wrong-database.db", + }, + ) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = EnvironmentFileDocker(databases[arm]["kind"]) + + outcome = DockerSut(docker).run_arm(config, arm, paths, b"prompt", ArtifactStore(paths.result_root)) + + assert json.loads(docker.environment.pop("POWERCONTEXT_SERVER_DATABASE")) == databases[arm] + assert docker.environment == ( + { + "OPENAI_API_KEY": model_secret, + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL": "openai:fixture-model", + } + if arm is Arm.ON + else {} + ) + registered_scopes.extend(created_scope_keys(docker.commands)) + assert outcome.evidence.database_kind == databases[arm]["kind"] + treatment = json.loads((paths.result_root / "powercontext/treatment.json").read_text()) + assert treatment["database_kind"] == databases[arm]["kind"] + assert treatment["database_fingerprint"] == "d" * 64 + serialized_commands = json.dumps(docker.commands) + for secret in ("off-database-secret", "on-database-secret", model_secret): + assert secret not in serialized_commands + assert all( + secret.encode() not in path.read_bytes() for path in paths.result_root.rglob("*") if path.is_file() + ) + assert "on-database-secret" in docker.codex_secrets + if off_backend == "oceanbase": + assert "off-database-secret" in docker.codex_secrets + runtime_install = next( + command + for command in docker.commands + if "sync" in command and "--project" in command and command[command.index("--project") + 1] == "/source" + ) + assert "--no-install-package" not in runtime_install + + assert environment_files[0] != environment_files[1] + assert registered_scopes == ["eval:run-1:off", "eval:run-1:on"] + + +@pytest.mark.parametrize( + "on_url", + [ + "mysql+aoceanbase://other-user:other-secret@database.test:3306/shared", + "mysql+aoceanbase://eval-user:rotated-secret@DATABASE.TEST./shared", + ], +) +def test_sut_rejects_shared_oceanbase_before_starting_evaluation(tmp_path: Path, on_url: str) -> None: + with pytest.raises(UnsafeSutConfiguration, match="separate OceanBase databases") as raised: + replace( + sut_config(tmp_path), + database_configs={ + Arm.OFF: {"kind": "oceanbase", "url": "mysql+aoceanbase://eval:original-secret@database.test/shared"}, + Arm.ON: {"kind": "oceanbase", "url": on_url}, + }, + ) + assert "secret" not in str(raised.value) + assert on_url not in str(raised.value) + + +@pytest.mark.parametrize( + ("observed_kind", "observed_fingerprint"), + [("sqlite", "d" * 64), ("oceanbase", "not-a-fingerprint")], +) +def test_sut_rejects_mismatched_or_incomplete_database_evidence( + tmp_path: Path, + observed_kind: str, + observed_fingerprint: str, +) -> None: + paths = make_paths(tmp_path) + config = replace( + sut_config(tmp_path), + proxy=None, + tokensflow_enabled=False, + tokensflow_binary=None, + tokensflow_egress_network=None, + database_configs={ + arm: { + "kind": "oceanbase", + "url": f"mysql+aoceanbase://eval-user:database-secret-456@database.test:2881/eval-{arm.value}", + } + for arm in (Arm.OFF, Arm.ON) + }, + ) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = TranscriptDocker(database_kind=observed_kind, database_fingerprint=observed_fingerprint) + + with pytest.raises(InvalidTreatment, match="database evidence"): + DockerSut(docker).run_arm(config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root)) + + assert not (paths.result_root / "powercontext/treatment.json").exists() + assert any(command[:3] == ("docker", "rm", "-f") for command in docker.commands) + + def _is_tokensflow(command: tuple[str, ...], action: str) -> bool: return any("/tools/tokensflow-dir/tokensflow" in part for part in command) and any( action in part for part in command @@ -3192,14 +3386,14 @@ def popen(argv: tuple[str, ...], **kwargs: object) -> FakeProcess: events.append(("popen", argv, kwargs)) return FakeProcess() - monkeypatch.setattr("powercontext_eval.powercontext_sut._reserve_port", lambda _address: 17890) - monkeypatch.setattr("powercontext_eval.powercontext_sut.subprocess.Popen", popen) + monkeypatch.setattr("powercontext_eval_swebench_pro.powercontext_sut._reserve_port", lambda _address: 17890) + monkeypatch.setattr("powercontext_eval_swebench_pro.powercontext_sut.subprocess.Popen", popen) monkeypatch.setattr( - "powercontext_eval.powercontext_sut.socket.create_connection", + "powercontext_eval_swebench_pro.powercontext_sut.socket.create_connection", lambda address, timeout: events.append(("ready", address, timeout)) or io.BytesIO(), ) monkeypatch.setattr( - "powercontext_eval.powercontext_sut.os.killpg", + "powercontext_eval_swebench_pro.powercontext_sut.os.killpg", lambda pid, sig: events.append(("killpg", pid, sig)), ) relay = SocatProxyRelay() @@ -3234,10 +3428,12 @@ def wait(self, timeout: float | None = None) -> int: del timeout return 0 - monkeypatch.setattr("powercontext_eval.powercontext_sut._reserve_port", lambda _address: 17891) - monkeypatch.setattr("powercontext_eval.powercontext_sut.subprocess.Popen", lambda *_args, **_kwargs: FakeProcess()) + monkeypatch.setattr("powercontext_eval_swebench_pro.powercontext_sut._reserve_port", lambda _address: 17891) + monkeypatch.setattr( + "powercontext_eval_swebench_pro.powercontext_sut.subprocess.Popen", lambda *_args, **_kwargs: FakeProcess() + ) monkeypatch.setattr( - "powercontext_eval.powercontext_sut.os.killpg", + "powercontext_eval_swebench_pro.powercontext_sut.os.killpg", lambda pid, sig: killed.append((pid, int(sig))), ) diff --git a/evaluation/tests/contract/test_git_contract.py b/evaluation/coding/swebench_pro/tests/contract/test_git_contract.py similarity index 99% rename from evaluation/tests/contract/test_git_contract.py rename to evaluation/coding/swebench_pro/tests/contract/test_git_contract.py index 994a4e95c0..55f2218173 100644 --- a/evaluation/tests/contract/test_git_contract.py +++ b/evaluation/coding/swebench_pro/tests/contract/test_git_contract.py @@ -29,11 +29,11 @@ import pytest -from powercontext_eval import git_source as git_source_module -from powercontext_eval.errors import CommandError, CommandFailed, GitSourceError -from powercontext_eval.git_source import GitSource, ResolvedGitSource -from powercontext_eval.models import PowerContextRef -from powercontext_eval.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro import git_source as git_source_module +from powercontext_eval_swebench_pro.errors import CommandError, CommandFailed, GitSourceError +from powercontext_eval_swebench_pro.git_source import GitSource, ResolvedGitSource +from powercontext_eval_swebench_pro.models import PowerContextRef +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner from .helpers import GitFixture, create_git_fixture, git diff --git a/evaluation/tests/contract/test_swebench_catalog.py b/evaluation/coding/swebench_pro/tests/contract/test_swebench_catalog.py similarity index 98% rename from evaluation/tests/contract/test_swebench_catalog.py rename to evaluation/coding/swebench_pro/tests/contract/test_swebench_catalog.py index 13d88371b3..148e86fdf5 100644 --- a/evaluation/tests/contract/test_swebench_catalog.py +++ b/evaluation/coding/swebench_pro/tests/contract/test_swebench_catalog.py @@ -21,8 +21,8 @@ import pytest -from powercontext_eval.benchmarks.swebench_pro.adapter import SweBenchProInstance -from powercontext_eval.benchmarks.swebench_pro.catalog import ( +from powercontext_eval_swebench_pro.adapter import SweBenchProInstance +from powercontext_eval_swebench_pro.catalog import ( PUBLIC_V2_COUNT, PUBLIC_V2_SHA256, PUBLIC_V2_TASK_SET, diff --git a/evaluation/tests/contract/test_swebench_contract.py b/evaluation/coding/swebench_pro/tests/contract/test_swebench_contract.py similarity index 97% rename from evaluation/tests/contract/test_swebench_contract.py rename to evaluation/coding/swebench_pro/tests/contract/test_swebench_contract.py index cacf5bd834..5a3317b401 100644 --- a/evaluation/tests/contract/test_swebench_contract.py +++ b/evaluation/coding/swebench_pro/tests/contract/test_swebench_contract.py @@ -27,22 +27,26 @@ import pytest -import powercontext_eval.powercontext_sut as sut_module -from powercontext_eval.benchmarks.base import GoldCheckFailed, GoldResult, run_after_gold -from powercontext_eval.benchmarks.swebench_pro.adapter import ( +import powercontext_eval_swebench_pro.powercontext_sut as sut_module +from powercontext_eval_swebench_pro.adapter import ( DATASET_REVISION, HARNESS_COMMIT, DatasetSchemaError, SweBenchProInstance, ) -from powercontext_eval.benchmarks.swebench_pro.evaluator import ( +from powercontext_eval_swebench_pro.evaluator import ( OfficialEvaluator, OfficialResultError, TestGroupResult, ) -from powercontext_eval.benchmarks.swebench_pro.prediction import encode_predictions -from powercontext_eval.powercontext_sut import LOOPBACK_NO_PROXY, ProxyRelayConfig, direct_egress_environment -from powercontext_eval.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.gold import GoldCheckFailed, GoldResult, run_after_gold +from powercontext_eval_swebench_pro.powercontext_sut import ( + LOOPBACK_NO_PROXY, + ProxyRelayConfig, + direct_egress_environment, +) +from powercontext_eval_swebench_pro.prediction import encode_predictions +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner INSTANCE_ID = "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9" DATASET_FIELDS = { diff --git a/evaluation/tests/unit/test_artifacts.py b/evaluation/coding/swebench_pro/tests/unit/test_artifacts.py similarity index 99% rename from evaluation/tests/unit/test_artifacts.py rename to evaluation/coding/swebench_pro/tests/unit/test_artifacts.py index 5819e5975f..5d19b7628e 100644 --- a/evaluation/tests/unit/test_artifacts.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_artifacts.py @@ -25,8 +25,8 @@ import pytest -import powercontext_eval.artifacts as artifacts_module -from powercontext_eval.artifacts import ( +import powercontext_eval_swebench_pro.artifacts as artifacts_module +from powercontext_eval_swebench_pro.artifacts import ( ArmState, ArmStateMachine, ArtifactAlreadyExists, diff --git a/evaluation/tests/unit/test_cli.py b/evaluation/coding/swebench_pro/tests/unit/test_cli.py similarity index 52% rename from evaluation/tests/unit/test_cli.py rename to evaluation/coding/swebench_pro/tests/unit/test_cli.py index 458b009bb1..3b99b6f34b 100644 --- a/evaluation/tests/unit/test_cli.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_cli.py @@ -22,184 +22,36 @@ from urllib.error import HTTPError from urllib.request import Request +import pytest +from click import unstyle from typer.testing import CliRunner -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( - LongMemEvalV2EnvironmentError, - LongMemEvalV2InputError, -) -from powercontext_eval.benchmarks.longmemeval_v2.smoke import PreparedSmokeRun -from powercontext_eval.cli import app -from powercontext_eval.runner import MinimalRunResult, RunConfig - - -def test_cli_help_describes_the_evaluation_runner() -> None: - result = CliRunner().invoke(app, ["--help"]) - - assert result.exit_code == 0 - assert "PowerContext evaluation runner" in result.output - assert not isinstance(result.exception, RuntimeError) - - -def test_longmemeval_v2_smoke_prepares_input_artifacts_without_a_model(monkeypatch, tmp_path: Path) -> None: - def prepare(**kwargs: object) -> PreparedSmokeRun: - assert kwargs == { - "data_root": Path("/data"), - "dataset_lock": Path("/dataset-lock.json"), - "harness_root": Path("/harness"), - "smoke_manifest": Path("/smoke.json"), - "output_dir": Path("/output"), - } - return PreparedSmokeRun( - output_dir=tmp_path / "output", - manifest_path=tmp_path / "output" / "manifest.json", - subset_path=tmp_path / "output" / "subset.json", - ) - - monkeypatch.setattr("powercontext_eval.cli.prepare_smoke_run", prepare) - result = CliRunner().invoke( - app, - [ - "longmemeval-v2", - "smoke", - "--data-root", - "/data", - "--dataset-lock", - "/dataset-lock.json", - "--harness-root", - "/harness", - "--smoke-manifest", - "/smoke.json", - "--output-dir", - "/output", - ], - ) - - assert result.exit_code == 0, result.output - assert '"classification": "smoke-subset"' in result.output - - -def test_longmemeval_v2_smoke_reports_invalid_configuration_as_bad_parameter(monkeypatch) -> None: - def prepare(**_kwargs: object) -> PreparedSmokeRun: - raise LongMemEvalV2InputError("Smoke manifest schema is unsupported") - - monkeypatch.setattr("powercontext_eval.cli.prepare_smoke_run", prepare) - result = CliRunner().invoke(app, ["longmemeval-v2", "smoke", *_longmemeval_smoke_arguments()]) - - assert result.exit_code == 2 - assert "Invalid value" in result.output - assert "Smoke manifest schema is unsupported" in result.output - - -def test_longmemeval_v2_smoke_reports_environment_failure_with_exit_one(monkeypatch) -> None: - def prepare(**_kwargs: object) -> PreparedSmokeRun: - raise LongMemEvalV2EnvironmentError("LongMemEval-V2 SHA-256 mismatch for questions.jsonl") - - monkeypatch.setattr("powercontext_eval.cli.prepare_smoke_run", prepare) - result = CliRunner().invoke(app, ["longmemeval-v2", "smoke", *_longmemeval_smoke_arguments()]) - - assert result.exit_code == 1 - assert "LongMemEval-V2 smoke failed" in result.output - assert "SHA-256 mismatch" in result.output - assert "Invalid value" not in result.output - - -def _longmemeval_smoke_arguments() -> list[str]: - return [ - "--data-root", - "/data", - "--dataset-lock", - "/dataset-lock.json", - "--harness-root", - "/harness", - "--smoke-manifest", - "/smoke.json", - "--output-dir", - "/output", - ] - - -def test_codex_contract_smoke_is_an_executable_injectable_cli(monkeypatch) -> None: - calls: list[dict[str, object]] = [] - - def fake_contract_smoke(**kwargs: object) -> dict[str, object]: - calls.append(kwargs) - return { - "off_prompt_sources": 0, - "on_prompt_sources": 1, - "status": "passed", - "tokensflow": { - "off": {"identity_match": True, "queue_caught_up": True}, - "on": {"identity_match": True, "queue_caught_up": True}, - }, - } - - monkeypatch.setattr("powercontext_eval.cli.run_codex_contract_smoke", fake_contract_smoke) - result = CliRunner().invoke( - app, - [ - "codex-contract-smoke", - "--run-root", - "/tmp/contract", - "--task-image", - "fixture:image", - "--codex-bin", - "/tools/codex", - "--tokensflow-bin", - "/tools/tokensflow", - "--tokensflow-user-home", - "/tokensflow-home", - "--tokensflow-egress-network", - "bridge", - "--uv-bin", - "/tools/uv", - "--powercontext-source", - "/source", - "--powercontext-sha", - "a" * 40, - "--auth-json", - "/auth.json", - "--proxy-url", - "http://127.0.0.1:18080", - ], - ) - - assert result.exit_code == 0, result.output - assert '"status": "passed"' in result.output - assert '"queue_caught_up": true' in result.output - assert "/tokensflow-home" not in result.output - assert calls == [ - { - "run_root": "/tmp/contract", - "task_image": "fixture:image", - "codex_bin": "/tools/codex", - "tokensflow_bin": "/tools/tokensflow", - "tokensflow_user_home": "/tokensflow-home", - "tokensflow_egress_network": "bridge", - "uv_bin": "/tools/uv", - "powercontext_source": "/source", - "powercontext_sha": "a" * 40, - "auth_json": "/auth.json", - "proxy_url": "http://127.0.0.1:18080", - "prompt": "Reply with exactly OK.", - } - ] - - -def test_cli_module_is_directly_executable() -> None: - result = subprocess.run( - [sys.executable, "-m", "powercontext_eval.cli", "--help"], - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0 - assert "codex-contract-smoke" in result.stdout +from powercontext_eval_swebench_pro.cli import app +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.runner import MinimalRunResult, RunConfig -def test_swebench_pro_run_derives_portable_paths_from_explicit_root(monkeypatch) -> None: +@pytest.mark.parametrize( + ("username", "password"), + [("eval", "private-test-password"), ("eval@tenant", "private-test-password%40%23%0A%0D%09")], +) +def test_swebench_pro_run_derives_portable_paths_from_explicit_root( + monkeypatch, tmp_path: Path, username: str, password: str +) -> None: calls: list[tuple[object, object]] = [] instance = object() + databases = { + arm: { + "kind": "oceanbase", + "url": ( + f"mysql+aoceanbase://{username}:{password}@localhost/test_{arm.value}" + "?charset=utf8mb4&connect_timeout=10" + ), + } + for arm in Arm + } + database_file = tmp_path / "database.json" + database_file.write_text(json.dumps(databases), encoding="utf-8") def fake_run(config: object, *, instance: object) -> MinimalRunResult: calls.append((config, instance)) @@ -211,14 +63,13 @@ def require(self, instance_id: str) -> object: return instance monkeypatch.setattr( - "powercontext_eval.cli.SweBenchProCatalog.load", + "powercontext_eval_swebench_pro.cli.SweBenchProCatalog.load", lambda path: FakeCatalog(), ) - monkeypatch.setattr("powercontext_eval.cli.run_swebench_pro_instance", fake_run) + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.run_swebench_pro_instance", fake_run) result = CliRunner().invoke( app, [ - "swebench-pro", "run", "--run-id", "run-fixed", @@ -233,6 +84,8 @@ def require(self, instance_id: str) -> object: "--tokensflow", "--model", "gpt-5.6-luna", + "--database-config", + str(database_file), ], ) @@ -249,6 +102,109 @@ def require(self, instance_id: str) -> object: assert config.tokensflow_egress_network == "bridge" assert config.model == "gpt-5.6-luna" assert config.reasoning_effort == "medium" + assert config.database_configs == databases + assert "private-test-password" not in repr(config) + assert "private-test-password" not in result.output + + +@pytest.mark.parametrize( + "database_json", + [ + None, + '{"off": "private-test-password"', + '"private-test-password"', + '{"kind":"oceanbase","url":"mysql+aoceanbase://eval:private-test-password@localhost/test_db"}', + '{"off":{},"on":{},"extra":"private-test-password"}', + '{"off":"private-test-password","on":{}}', + '{"off":{"kind":"unknown","url":"private-test-password"},"on":{}}', + json.dumps( + { + "off": { + "kind": "oceanbase", + "url": "mysql+aoceanbase://eval:private-test-password@localhost/test_db", + }, + "on": { + "kind": "oceanbase", + "url": "mysql+aoceanbase://other:another-private-password@localhost/test_db", + }, + } + ), + *( + json.dumps( + { + arm.value: { + "kind": "oceanbase", + "url": f"mysql+aoceanbase://eval@localhost/test_{arm.value}?charset=utf8mb4&{query}", + } + for arm in Arm + } + ) + for query in ( + "password=private-test-password", + "db=shared-private-test-password", + ) + ), + *( + json.dumps( + { + arm.value: { + "kind": "oceanbase", + "url": f"mysql+aoceanbase://eval:private-test-password{character}suffix@localhost/test_{arm.value}", + } + for arm in Arm + } + ) + for character in ("@", "\n", "\r", "\t") + ), + ], + ids=[ + "missing-file", + "invalid-json", + "scalar", + "legacy-shape", + "extra-arm", + "invalid-arm", + "invalid-kind", + "same-db", + "query-password", + "query-shared-db", + "raw-at-password", + "raw-newline-password", + "raw-carriage-return-password", + "raw-tab-password", + ], +) +def test_swebench_pro_run_rejects_invalid_database_config_without_exposing_secrets( + monkeypatch, tmp_path: Path, database_json: str | None +) -> None: + database_file = tmp_path / "database.json" + if database_json is not None: + database_file.write_text(database_json, encoding="utf-8") + + def unexpected_catalog_load(_path: Path) -> None: + pytest.fail("Invalid database configuration must be rejected before loading a task") + + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.SweBenchProCatalog.load", unexpected_catalog_load) + result = CliRunner().invoke( + app, + [ + "run", + "--root", + str(tmp_path / "evaluation"), + "--instance-id", + "instance_owner__repo-b", + "--database-config", + str(database_file), + ], + ) + + assert result.exit_code == 2, result.output + output = unstyle(result.output) + assert "--database-config" in output + assert "Database configuration" in output + assert "private-test-password" not in output + assert "another-private-password" not in output + assert not (tmp_path / "evaluation").exists() def test_swebench_pro_run_defaults_optional_integrations_off(monkeypatch) -> None: @@ -263,13 +219,12 @@ def fake_run(config: RunConfig, *, instance: object) -> MinimalRunResult: captured.append(config) return MinimalRunResult("run-direct", Path("/tmp/report.md"), False, False) - monkeypatch.setattr("powercontext_eval.cli.SweBenchProCatalog.load", lambda _path: FakeCatalog()) - monkeypatch.setattr("powercontext_eval.cli.run_swebench_pro_instance", fake_run) + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.SweBenchProCatalog.load", lambda _path: FakeCatalog()) + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.run_swebench_pro_instance", fake_run) result = CliRunner().invoke( app, [ - "swebench-pro", "run", "--root", "/srv/evaluation", @@ -285,6 +240,7 @@ def fake_run(config: RunConfig, *, instance: object) -> MinimalRunResult: assert captured[0].tokensflow_user_home is None assert captured[0].tokensflow_egress_network is None assert captured[0].proxy_url is None + assert captured[0].database_configs is None def test_cli_creates_a_single_arm_luna_batch_atomically_paused(monkeypatch) -> None: @@ -309,11 +265,10 @@ def fake_urlopen(request: Request, *, timeout: float) -> Response: return Response(b'{"models":["gpt-5.6-sol","gpt-5.6-luna"]}') return Response(b'{"batch_id":"batch-luna"}') - monkeypatch.setattr("powercontext_eval.cli.urlopen", fake_urlopen, raising=False) + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.urlopen", fake_urlopen, raising=False) result = CliRunner().invoke( app, [ - "swebench-pro", "create-batch", "--idempotency-key", "luna-paused-cli", @@ -355,11 +310,10 @@ def fake_urlopen(request: Request, *, timeout: float) -> None: fp=BytesIO(b'{"error":{"code":"invalid_request","message":"The evaluation request is invalid."}}'), ) - monkeypatch.setattr("powercontext_eval.cli.urlopen", fake_urlopen, raising=False) + monkeypatch.setattr("powercontext_eval_swebench_pro.cli.urlopen", fake_urlopen, raising=False) result = CliRunner().invoke( app, [ - "swebench-pro", "create-batch", "--idempotency-key", "unconfigured-model", @@ -371,3 +325,22 @@ def fake_urlopen(request: Request, *, timeout: float) -> None: assert result.exit_code == 2 assert "not enabled" in result.output assert [request.full_url for request in calls] == ["http://127.0.0.1:8787/api/batches"] + + +def test_cli_help_describes_the_evaluation_runner() -> None: + result = CliRunner().invoke(app, ["--help"]) + + assert result.exit_code == 0 + assert "Pinned SWE-bench Pro evaluation" in result.output + assert not isinstance(result.exception, RuntimeError) + + +def test_cli_module_is_directly_executable() -> None: + result = subprocess.run( + [sys.executable, "-m", "powercontext_eval_swebench_pro.cli", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert "codex-contract-smoke" in result.stdout diff --git a/evaluation/tests/unit/test_context_trace.py b/evaluation/coding/swebench_pro/tests/unit/test_context_trace.py similarity index 95% rename from evaluation/tests/unit/test_context_trace.py rename to evaluation/coding/swebench_pro/tests/unit/test_context_trace.py index fc958c9b3d..08d16b4fda 100644 --- a/evaluation/tests/unit/test_context_trace.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_context_trace.py @@ -23,16 +23,16 @@ import pytest -from powercontext_eval.artifacts import ArtifactStore, SecretDetected -from powercontext_eval.benchmarks.swebench_pro.evaluator import ( +from powercontext_eval_swebench_pro import record_codex_jsonl +from powercontext_eval_swebench_pro.artifacts import ArtifactStore, SecretDetected +from powercontext_eval_swebench_pro.context_trace import write_context_trace +from powercontext_eval_swebench_pro.evaluator import ( OfficialEvaluation, TestGroupResult, ) -from powercontext_eval.context_trace import write_context_trace -from powercontext_eval.models import Arm +from powercontext_eval_swebench_pro.models import Arm -REPOSITORY_ROOT = Path(__file__).resolve().parents[3] -RECORDER = REPOSITORY_ROOT / "evaluation" / "scripts" / "record_codex_jsonl.py" +RECORDER = Path(record_codex_jsonl.__file__) def _write_jsonl(path: Path, values: list[dict[str, object]]) -> Path: diff --git a/evaluation/tests/unit/test_gold_overrides.py b/evaluation/coding/swebench_pro/tests/unit/test_gold_overrides.py similarity index 97% rename from evaluation/tests/unit/test_gold_overrides.py rename to evaluation/coding/swebench_pro/tests/unit/test_gold_overrides.py index 1d1d24c0d7..0cc5f71e10 100644 --- a/evaluation/tests/unit/test_gold_overrides.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_gold_overrides.py @@ -20,8 +20,8 @@ import pytest from pydantic import ValidationError -from powercontext_eval.artifacts import ArmState -from powercontext_eval.benchmarks.swebench_pro.gold_overrides import ( +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.gold_overrides import ( SOURCE559_DATASET_PATCH_SHA256, SOURCE559_INSTANCE_ID, SOURCE559_REFERENCE_PATCH, @@ -29,7 +29,7 @@ GoldValidationOverrideError, select_gold_validation, ) -from powercontext_eval.report import ArmReport, GoldValidationAudit, ReportBundle, render_report +from powercontext_eval_swebench_pro.report import ArmReport, GoldValidationAudit, ReportBundle, render_report def _source559_dataset_patch() -> str: @@ -109,7 +109,7 @@ def test_source559_selects_override_for_real_patch_and_rejects_added_newline() - def test_exact_instance_selects_override_and_audits_pending_before_attempt(monkeypatch: pytest.MonkeyPatch) -> None: original = _source559_dataset_patch() - import powercontext_eval.benchmarks.swebench_pro.gold_overrides as overrides + import powercontext_eval_swebench_pro.gold_overrides as overrides monkeypatch.setattr(overrides, "SOURCE559_DATASET_PATCH_SHA256", hashlib.sha256(original.encode()).hexdigest()) selection = select_gold_validation(SOURCE559_INSTANCE_ID, original) @@ -138,7 +138,7 @@ def test_other_instances_keep_original_patch_and_have_no_reference_provenance() def test_source595_removes_only_the_invalid_non_test_integration_target(monkeypatch: pytest.MonkeyPatch) -> None: patch = "source595 Gold patch" - import powercontext_eval.benchmarks.swebench_pro.gold_overrides as overrides + import powercontext_eval_swebench_pro.gold_overrides as overrides monkeypatch.setattr(overrides, "SOURCE595_DATASET_PATCH_SHA256", hashlib.sha256(patch.encode()).hexdigest()) selection = select_gold_validation(SOURCE595_INSTANCE_ID, patch, SOURCE595_SELECTED_TEST_FILES) @@ -155,7 +155,7 @@ def test_source595_removes_only_the_invalid_non_test_integration_target(monkeypa def test_source595_selection_override_fails_closed_on_row_drift(monkeypatch: pytest.MonkeyPatch) -> None: patch = "source595 Gold patch" - import powercontext_eval.benchmarks.swebench_pro.gold_overrides as overrides + import powercontext_eval_swebench_pro.gold_overrides as overrides monkeypatch.setattr(overrides, "SOURCE595_DATASET_PATCH_SHA256", hashlib.sha256(patch.encode()).hexdigest()) with pytest.raises(GoldValidationOverrideError): diff --git a/evaluation/tests/unit/test_models.py b/evaluation/coding/swebench_pro/tests/unit/test_models.py similarity index 97% rename from evaluation/tests/unit/test_models.py rename to evaluation/coding/swebench_pro/tests/unit/test_models.py index 7bed259ab3..a8415c30be 100644 --- a/evaluation/tests/unit/test_models.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_models.py @@ -15,7 +15,7 @@ import pytest from pydantic import ValidationError -from powercontext_eval.models import Arm, PowerContextRef +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef @pytest.mark.parametrize( diff --git a/evaluation/tests/unit/test_paths.py b/evaluation/coding/swebench_pro/tests/unit/test_paths.py similarity index 93% rename from evaluation/tests/unit/test_paths.py rename to evaluation/coding/swebench_pro/tests/unit/test_paths.py index a7bc54d4e2..37e488d0a5 100644 --- a/evaluation/tests/unit/test_paths.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_paths.py @@ -17,8 +17,8 @@ import pytest -from powercontext_eval.models import Arm -from powercontext_eval.paths import EvaluationPaths +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.paths import EvaluationPaths def test_evaluation_paths_separate_ephemeral_work_from_retained_artifacts(tmp_path: Path) -> None: diff --git a/evaluation/tests/unit/test_process.py b/evaluation/coding/swebench_pro/tests/unit/test_process.py similarity index 99% rename from evaluation/tests/unit/test_process.py rename to evaluation/coding/swebench_pro/tests/unit/test_process.py index 62411048a5..0d45997f88 100644 --- a/evaluation/tests/unit/test_process.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_process.py @@ -28,8 +28,8 @@ import pytest -from powercontext_eval import process as process_module -from powercontext_eval.process import ( +from powercontext_eval_swebench_pro import process as process_module +from powercontext_eval_swebench_pro.process import ( CommandFailed, CommandNotFound, CommandResult, diff --git a/evaluation/tests/unit/test_report.py b/evaluation/coding/swebench_pro/tests/unit/test_report.py similarity index 98% rename from evaluation/tests/unit/test_report.py rename to evaluation/coding/swebench_pro/tests/unit/test_report.py index f55c5e5cda..20d0a34ff5 100644 --- a/evaluation/tests/unit/test_report.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_report.py @@ -19,8 +19,8 @@ import pytest from pydantic import ValidationError -from powercontext_eval.artifacts import ArmState -from powercontext_eval.report import ( +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.report import ( ArmReport, InvalidReportBundle, MetricSet, diff --git a/evaluation/tests/unit/test_runner_phases.py b/evaluation/coding/swebench_pro/tests/unit/test_runner_phases.py similarity index 92% rename from evaluation/tests/unit/test_runner_phases.py rename to evaluation/coding/swebench_pro/tests/unit/test_runner_phases.py index 27fd9f4143..6935b29a34 100644 --- a/evaluation/tests/unit/test_runner_phases.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_runner_phases.py @@ -25,10 +25,12 @@ import pytest -from powercontext_eval.artifacts import ArtifactStore, SecretDetected -from powercontext_eval.benchmarks.swebench_pro.adapter import SweBenchProInstance -from powercontext_eval.benchmarks.swebench_pro.evaluator import OfficialEvaluation -from powercontext_eval.benchmarks.swebench_pro.gold_overrides import ( +from powercontext_eval_swebench_pro.adapter import SweBenchProInstance +from powercontext_eval_swebench_pro.artifacts import ArtifactStore, SecretDetected +from powercontext_eval_swebench_pro.codex import CodexOutcome +from powercontext_eval_swebench_pro.errors import CommandFailed +from powercontext_eval_swebench_pro.evaluator import OfficialEvaluation +from powercontext_eval_swebench_pro.gold_overrides import ( SOURCE559_DATASET_PATCH_SHA256, SOURCE559_INSTANCE_ID, SOURCE559_REFERENCE_DATASET, @@ -38,13 +40,11 @@ SOURCE559_REFERENCE_REVISION, GoldValidationSelection, ) -from powercontext_eval.codex import CodexOutcome -from powercontext_eval.errors import CommandFailed -from powercontext_eval.models import Arm, TreatmentMode -from powercontext_eval.powercontext_sut import ProxyRelayConfig, SutConfig -from powercontext_eval.process import CommandResult, ProcessRunner -from powercontext_eval.report import ReportBundle -from powercontext_eval.runner import ( +from powercontext_eval_swebench_pro.models import Arm, TreatmentMode +from powercontext_eval_swebench_pro.powercontext_sut import ProxyRelayConfig, SutConfig, UnsafeSutConfiguration +from powercontext_eval_swebench_pro.process import CommandResult, ProcessRunner +from powercontext_eval_swebench_pro.report import ReportBundle +from powercontext_eval_swebench_pro.runner import ( MinimalRunResult, PhaseCallback, RunConfig, @@ -53,7 +53,7 @@ _resolve_task_image, run_swebench_pro_instance, ) -from powercontext_eval.tokensflow import TokensFlowInfrastructureError +from powercontext_eval_swebench_pro.tokensflow import TokensFlowInfrastructureError INSTANCE_ID = "instance_owner__repo-b" OPENLIBRARY_DYNAMIC_YEAR_INSTANCE_ID = ( @@ -279,6 +279,62 @@ def _config(tmp_path: Path) -> RunConfig: ) +def test_run_configuration_rejects_shared_oceanbase_before_gold(tmp_path: Path) -> None: + with pytest.raises(UnsafeSutConfiguration, match="separate OceanBase databases"): + replace( + _config(tmp_path), + database_configs={ + Arm.OFF: {"kind": "oceanbase", "url": "mysql+aoceanbase://eval:off-secret@DATABASE.test/shared"}, + Arm.ON: {"kind": "oceanbase", "url": "mysql+aoceanbase://eval:on-secret@database.test:3306/shared"}, + }, + ) + assert not (tmp_path / "eval").exists() + + +@pytest.mark.parametrize( + "option", + [ + "host", + "port", + "user", + "username", + "password", + "passwd", + "db", + "database", + "unix_socket", + "read_default_file", + "read_default_group", + "init_command", + "sql_mode", + "PASSWORD", + "%70assword", + "%64%62", + "Read_Default_File", + "%69nit_command", + "unused=value#fragment&password", + "password=first&password", + "PaSsWd", + "UNIX_SOCKET", + ], +) +def test_run_configuration_rejects_oceanbase_query_overrides_before_gold(tmp_path: Path, option: str) -> None: + password = "private-query-value" + on_url = f"mysql+aoceanbase://eval@database.test/on?charset=utf8mb4&{option}={password}" + with pytest.raises(UnsafeSutConfiguration) as raised: + replace( + _config(tmp_path), + database_configs={ + Arm.OFF: {"kind": "oceanbase", "url": "mysql+aoceanbase://eval:off-secret@database.test/off"}, + Arm.ON: {"kind": "oceanbase", "url": on_url}, + }, + ) + + assert password not in str(raised.value) + assert on_url not in str(raised.value) + assert not (tmp_path / "eval").exists() + + def _run_with_fakes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -438,10 +494,10 @@ def run_pair( outcomes[arm] = self._outcome(arm, stores[arm]) return outcomes - monkeypatch.setattr("powercontext_eval.runner.ProcessRunner", FakeProcess) - monkeypatch.setattr("powercontext_eval.runner.GitSource", lambda **kwargs: FakeSource()) - monkeypatch.setattr("powercontext_eval.runner.OfficialEvaluator", FakeEvaluator) - monkeypatch.setattr("powercontext_eval.runner.DockerSut", lambda *args, **kwargs: FakeSut()) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.ProcessRunner", FakeProcess) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.GitSource", lambda **kwargs: FakeSource()) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.OfficialEvaluator", FakeEvaluator) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.DockerSut", lambda *args, **kwargs: FakeSut()) callback = on_phase if on_phase is not None else lambda phase: events.append(phase) result = run_swebench_pro_instance(config, instance=instance, on_phase=callback) return config, result, observed @@ -649,7 +705,7 @@ def test_source559_gold_override_preserves_original_row_and_off_on_patches( source_file_oid=SOURCE559_REFERENCE_FILE_OID, source_kind="verified_reference_submission", ) - monkeypatch.setattr("powercontext_eval.runner.select_gold_validation", lambda *_args: selection) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.select_gold_validation", lambda *_args: selection) config, result, _observed = _run_with_fakes(tmp_path, monkeypatch, [], instance=instance) run_root = config.root / "runs" / result.run_id @@ -695,7 +751,7 @@ def test_source595_uses_only_required_unit_file_for_all_official_evaluations( official_evaluation_test_selection="required_unit_tests_only_for_invalid_integration_target", evaluator_selected_test_files_to_run=SOURCE595_EFFECTIVE_TEST_FILES, ) - monkeypatch.setattr("powercontext_eval.runner.select_gold_validation", lambda *_args: selection) + monkeypatch.setattr("powercontext_eval_swebench_pro.runner.select_gold_validation", lambda *_args: selection) config, result, _observed = _run_with_fakes(tmp_path, monkeypatch, [], instance=instance) diff --git a/evaluation/coding/swebench_pro/tests/unit/test_service_cli.py b/evaluation/coding/swebench_pro/tests/unit/test_service_cli.py new file mode 100644 index 0000000000..5ab0e56e61 --- /dev/null +++ b/evaluation/coding/swebench_pro/tests/unit/test_service_cli.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typer.testing import CliRunner + +from powercontext_eval_swebench_pro.cli import app + + +def test_codex_contract_smoke_is_an_executable_injectable_cli(monkeypatch) -> None: + calls: list[dict[str, object]] = [] + + def fake_contract_smoke(**kwargs: object) -> dict[str, object]: + calls.append(kwargs) + return { + "off_prompt_sources": 0, + "on_prompt_sources": 1, + "status": "passed", + "tokensflow": { + "off": {"identity_match": True, "queue_caught_up": True}, + "on": {"identity_match": True, "queue_caught_up": True}, + }, + } + + monkeypatch.setattr("powercontext_eval_swebench_pro.service_cli.run_codex_contract_smoke", fake_contract_smoke) + result = CliRunner().invoke( + app, + [ + "codex-contract-smoke", + "--run-root", + "/tmp/contract", + "--task-image", + "fixture:image", + "--codex-bin", + "/tools/codex", + "--tokensflow-bin", + "/tools/tokensflow", + "--tokensflow-user-home", + "/tokensflow-home", + "--tokensflow-egress-network", + "bridge", + "--uv-bin", + "/tools/uv", + "--powercontext-source", + "/source", + "--powercontext-sha", + "a" * 40, + "--auth-json", + "/auth.json", + "--proxy-url", + "http://127.0.0.1:18080", + ], + ) + + assert result.exit_code == 0, result.output + assert '"status": "passed"' in result.output + assert '"queue_caught_up": true' in result.output + assert "/tokensflow-home" not in result.output + assert calls == [ + { + "run_root": "/tmp/contract", + "task_image": "fixture:image", + "codex_bin": "/tools/codex", + "tokensflow_bin": "/tools/tokensflow", + "tokensflow_user_home": "/tokensflow-home", + "tokensflow_egress_network": "bridge", + "uv_bin": "/tools/uv", + "powercontext_source": "/source", + "powercontext_sha": "a" * 40, + "auth_json": "/auth.json", + "proxy_url": "http://127.0.0.1:18080", + "prompt": "Reply with exactly OK.", + } + ] diff --git a/evaluation/tests/unit/test_tokensflow.py b/evaluation/coding/swebench_pro/tests/unit/test_tokensflow.py similarity index 99% rename from evaluation/tests/unit/test_tokensflow.py rename to evaluation/coding/swebench_pro/tests/unit/test_tokensflow.py index a1deb0c4bb..44f793c3df 100644 --- a/evaluation/tests/unit/test_tokensflow.py +++ b/evaluation/coding/swebench_pro/tests/unit/test_tokensflow.py @@ -22,7 +22,7 @@ import pytest -from powercontext_eval.tokensflow import ( +from powercontext_eval_swebench_pro.tokensflow import ( DrainDeadline, TokensFlowInfrastructureError, UnsafeTokensFlowConfiguration, diff --git a/evaluation/tests/web/__init__.py b/evaluation/coding/swebench_pro/tests/web/__init__.py similarity index 100% rename from evaluation/tests/web/__init__.py rename to evaluation/coding/swebench_pro/tests/web/__init__.py diff --git a/evaluation/tests/web/fake_runner_app.py b/evaluation/coding/swebench_pro/tests/web/fake_runner_app.py similarity index 83% rename from evaluation/tests/web/fake_runner_app.py rename to evaluation/coding/swebench_pro/tests/web/fake_runner_app.py index ede378e35a..f087942219 100644 --- a/evaluation/tests/web/fake_runner_app.py +++ b/evaluation/coding/swebench_pro/tests/web/fake_runner_app.py @@ -16,39 +16,40 @@ from __future__ import annotations +import hashlib import json import shutil import signal +import subprocess import tempfile import threading import time from datetime import UTC, datetime, timedelta from pathlib import Path -from types import MappingProxyType, SimpleNamespace +from types import MappingProxyType from typing import Literal import uvicorn -from powercontext_eval.artifacts import ArmState -from powercontext_eval.benchmarks.swebench_pro.adapter import ( +from powercontext_eval_swebench_pro.adapter import ( DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance, ) -from powercontext_eval.codex import CodexInfrastructureError -from powercontext_eval.models import Arm -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.report import ArmReport, MetricSet, ReportBundle, TestGroupReport, render_report -from powercontext_eval.runner import MinimalRunResult, PhaseCallback, RunConfig, RunPhase -from powercontext_eval.web.api import create_app -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.store import TaskStore -from powercontext_eval.web.usage import UsageSnapshot -from powercontext_eval.web.worker import EvaluationWorker +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.codex import CodexInfrastructureError +from powercontext_eval_swebench_pro.models import Arm, PowerContextRef +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.report import ArmReport, MetricSet, ReportBundle, TestGroupReport, render_report +from powercontext_eval_swebench_pro.runner import MinimalRunResult, PhaseCallback, RunConfig, RunPhase +from powercontext_eval_swebench_pro.web.api import create_app +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.store import TaskStore +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.web.worker import EvaluationWorker PORT = 4177 PHASE_DELAY_SECONDS = 0.15 -PLUGIN_SHA = "a" * 40 class _SignalManagedServer(uvicorn.Server): @@ -84,12 +85,6 @@ def require(self, instance_id: str) -> SweBenchProInstance: return self._instances[instance_id] -class _Source: - def resolve(self, source: str | Path, requested: object) -> object: - del source, requested - return SimpleNamespace(sha=PLUGIN_SHA) - - class _ScriptedUsageProbe: """Derive repeatable account usage from the fixture's durable task progress.""" @@ -98,18 +93,19 @@ def __init__(self, store: TaskStore) -> None: def read(self, *, now: datetime) -> UsageSnapshot: terminal_tasks = 0 - threshold = 80 for batch in self._store.list_batches(): - threshold = batch.control.usage_pause_percent terminal_tasks += sum( - task.status.value in {"succeeded", "failed", "interrupted", "cancelled"} + any( + attempt.status.value in {"succeeded", "failed", "interrupted", "cancelled"} + for attempt in self._store.list_task_attempts(batch.batch_id, task.task_id) + ) for task in self._store.list_batch_tasks(batch.batch_id) ) if terminal_tasks == 0: used_percent = 20 elif terminal_tasks == 1: used_percent = 40 - elif terminal_tasks == 2 and threshold <= 80: + elif terminal_tasks == 2: used_percent = 81 else: used_percent = 50 @@ -157,17 +153,19 @@ def _arm( ) -def _evidence(run_id: str, arm: str) -> dict[str, object]: +def _evidence(run_id: str, arm: str, plugin_sha: str) -> dict[str, object]: enabled = arm == "on" return { "mcp_requests": 2 if enabled else 0, "prompt_sources": 2 if enabled else 0, - "plugin_checkout_sha": PLUGIN_SHA, + "plugin_checkout_sha": plugin_sha, "plugin_id": "powercontext@powercontext", "plugin_installed": True, "plugin_version": "0.1.0", "scope_id": f"eval:{run_id}:{arm}", "server_ready": True, + "database_kind": "sqlite", + "database_fingerprint": hashlib.sha256(f"browser-fixture:{run_id}:{arm}".encode()).hexdigest(), } @@ -313,8 +311,11 @@ def fake_runner( if config.run_id is None: raise ValueError("The browser worker must provide a run ID") + revision = PowerContextRef.parse(config.powercontext_ref) + if revision.kind != "commit" or revision.value is None: + raise ValueError("The browser worker must pin the source revision before execution") letter = instance.instance_id[-1] - fail_first_attempt = letter == "a" and ".attempt-" not in config.run_id + fail_first_attempt = letter == "a" and "-attempt-" not in config.run_id for phase in RunPhase: on_phase(phase) time.sleep(PHASE_DELAY_SECONDS) @@ -329,7 +330,7 @@ def fake_runner( revisions={ "dataset": DATASET_REVISION, "harness": HARNESS_COMMIT, - "powercontext": PLUGIN_SHA, + "powercontext": revision.value, }, configuration={ "codex": "0.145.0", @@ -357,25 +358,51 @@ def fake_runner( for arm, resolved in ((Arm.OFF, off_resolved), (Arm.ON, on_resolved)): _write_json( layout.arm_artifacts(arm) / "powercontext" / "treatment.json", - _evidence(config.run_id, arm.value), + _evidence(config.run_id, arm.value, revision.value), ) _timeline(layout, arm=arm.value, instance=instance, resolved=resolved) return MinimalRunResult(config.run_id, layout.run_artifacts / "report.md", off_resolved, on_resolved) def main() -> None: - repository = Path(__file__).resolve().parents[3] + suite = Path(__file__).resolve().parents[2] temporary_root = Path(tempfile.mkdtemp(prefix="powercontext-e2e-")) - frontend = temporary_root / "deploy" / "powercontext" / "evaluation" / "web" / "dist" - shutil.copytree(repository / "evaluation" / "web" / "dist", frontend) + source_root = temporary_root / "source" / "powercontext.git" + subprocess.run(["git", "init", "--initial-branch=master", str(source_root)], check=True, capture_output=True) + subprocess.run( + [ + "git", + "-C", + str(source_root), + "-c", + "user.name=Browser E2E", + "-c", + "user.email=browser-e2e@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--allow-empty", + "-m", + "Browser fixture source", + ], + check=True, + capture_output=True, + ) + frontend = temporary_root / "deploy" / "powercontext" / "evaluation" / "coding" / "swebench_pro" / "web" / "dist" + shutil.copytree(suite / "web" / "dist", frontend) config = WebConfig.for_root( temporary_root, tokensflow_egress_network="bridge", proxy_url="http://127.0.0.1:8081", database_path=temporary_root / "web" / "tasks.sqlite3", run_root=temporary_root, + powercontext_source=source_root, frontend_dist=frontend, poll_seconds=0.02, + task_parallelism=1, + usage_probe_seconds=10, lease_seconds=10, port=PORT, ) @@ -389,7 +416,6 @@ def main() -> None: store, usage_probe=usage_probe, runner=fake_runner, - source=_Source(), catalog=catalog, worker_id="browser-e2e-worker", ) diff --git a/evaluation/tests/web/test_api.py b/evaluation/coding/swebench_pro/tests/web/test_api.py similarity index 97% rename from evaluation/tests/web/test_api.py rename to evaluation/coding/swebench_pro/tests/web/test_api.py index af1179f107..137dff3c2c 100644 --- a/evaluation/tests/web/test_api.py +++ b/evaluation/coding/swebench_pro/tests/web/test_api.py @@ -17,6 +17,7 @@ import asyncio import json import time +from dataclasses import replace from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace @@ -27,10 +28,11 @@ from httpx import Response from starlette.requests import Request -from powercontext_eval.artifacts import ArmState -from powercontext_eval.benchmarks.swebench_pro.adapter import SweBenchProInstance -from powercontext_eval.benchmarks.swebench_pro.catalog import STABILITY_V1_CASES, STABILITY_V1_TASK_SET -from powercontext_eval.benchmarks.swebench_pro.gold_overrides import ( +from powercontext_eval_swebench_pro.adapter import SweBenchProInstance +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.catalog import STABILITY_V1_CASES, STABILITY_V1_TASK_SET +from powercontext_eval_swebench_pro.git_source import GitSource +from powercontext_eval_swebench_pro.gold_overrides import ( SOURCE559_DATASET_PATCH_SHA256, SOURCE559_INSTANCE_ID, SOURCE559_REFERENCE_DATASET, @@ -38,11 +40,17 @@ SOURCE559_REFERENCE_PATCH_SHA256, SOURCE559_REFERENCE_REVISION, ) -from powercontext_eval.git_source import GitSource -from powercontext_eval.report import ArmReport, GoldValidationAudit, MetricSet, ReportBundle, TestGroupReport -from powercontext_eval.web.api import TaskEventStream, create_app -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.powercontext_sut import TreatmentEvidence as SutTreatmentEvidence +from powercontext_eval_swebench_pro.report import ( + ArmReport, + GoldValidationAudit, + MetricSet, + ReportBundle, + TestGroupReport, +) +from powercontext_eval_swebench_pro.web.api import TaskEventStream, create_app +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.models import ( FailureCategory, FailureCode, RetryDisposition, @@ -52,9 +60,9 @@ TaskRecord, TaskResult, ) -from powercontext_eval.web.resources import FilesystemCapacity, ResourceUnavailable -from powercontext_eval.web.store import FinalizationState, TaskStore, TokensFlowFinalizationCreate -from powercontext_eval.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.web.resources import FilesystemCapacity, ResourceUnavailable +from powercontext_eval_swebench_pro.web.store import FinalizationState, TaskStore, TokensFlowFinalizationCreate +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot NOW = datetime(2026, 7, 29, 1, 2, 3, tzinfo=UTC) INSTANCE = "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9" @@ -587,8 +595,9 @@ def arm(name: Literal["off", "on"]) -> ArmReport: (run_dir / "report.md").write_text("# Résumé\n") +@pytest.mark.parametrize("database_identity", ("legacy", "unrecorded", "sqlite", "oceanbase")) def test_structured_and_raw_reports_use_validated_artifacts( - client: TestClient, config: WebConfig, store: TaskStore + client: TestClient, config: WebConfig, store: TaskStore, database_identity: str ) -> None: task = client.post("/api/tasks", json=payload("report-key")).json() created_at = datetime.fromisoformat(task["created_at"]) @@ -596,6 +605,16 @@ def test_structured_and_raw_reports_use_validated_artifacts( assert claimed is not None worker_run_dir = config.run_root / "runs" / task["task_id"] _write_report(config.run_root / "runs", task["task_id"]) + database_kind = database_identity if database_identity in {"sqlite", "oceanbase"} else None + if database_identity != "legacy": + for arm, fingerprint_digit in (("off", "1"), ("on", "2")): + evidence_path = worker_run_dir / "arms" / arm / "powercontext" / "treatment.json" + evidence = replace( + SutTreatmentEvidence.from_json(evidence_path.read_text()), + database_kind=database_kind, + database_fingerprint=fingerprint_digit * 64 if database_kind is not None else None, + ) + evidence_path.write_text(json.dumps(evidence.as_dict())) store.succeed( task["task_id"], "worker", @@ -614,6 +633,10 @@ def test_structured_and_raw_reports_use_validated_artifacts( assert structured.status_code == 200 assert structured.json()["task_id"] == task["task_id"] assert structured.json()["acceptance_valid"] is True + for arm, fingerprint_digit in (("off", "1"), ("on", "2")): + evidence = structured.json()["evidence"][arm] + assert evidence["database_kind"] == database_kind + assert evidence["database_fingerprint"] == (fingerprint_digit * 64 if database_kind is not None else None) assert structured.json()["off"] == { "arm": "off", "state": "treatment_validated", diff --git a/evaluation/tests/web/test_claiming.py b/evaluation/coding/swebench_pro/tests/web/test_claiming.py similarity index 96% rename from evaluation/tests/web/test_claiming.py rename to evaluation/coding/swebench_pro/tests/web/test_claiming.py index b87a7bfccd..fd1e113242 100644 --- a/evaluation/tests/web/test_claiming.py +++ b/evaluation/coding/swebench_pro/tests/web/test_claiming.py @@ -22,21 +22,21 @@ import pytest -from powercontext_eval.web.batches import BatchControlEventType, BatchCreate, BatchStatus -from powercontext_eval.web.claiming import ClaimCoordinator, PeriodicUsageRefresher -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.controls import BatchPauseReason -from powercontext_eval.web.models import TaskRecord -from powercontext_eval.web.resources import FilesystemCapacity, ResourceUnavailable -from powercontext_eval.web.store import TaskStore -from powercontext_eval.web.usage import UsageSnapshot, UsageUnavailable +from powercontext_eval_swebench_pro.web.batches import BatchControlEventType, BatchCreate, BatchStatus +from powercontext_eval_swebench_pro.web.claiming import ClaimCoordinator, PeriodicUsageRefresher +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.controls import BatchPauseReason +from powercontext_eval_swebench_pro.web.models import TaskRecord +from powercontext_eval_swebench_pro.web.resources import FilesystemCapacity, ResourceUnavailable +from powercontext_eval_swebench_pro.web.store import TaskStore +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot, UsageUnavailable NOW = datetime(2026, 8, 1, 1, 2, 3, tzinfo=UTC) @pytest.fixture(autouse=True) def _default_dependency_probe(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("powercontext_eval.web.resources.DockerDependencyProbe.check", lambda _self: None) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.resources.DockerDependencyProbe.check", lambda _self: None) def _usage(used_percent: int, *, observed_at: datetime = NOW) -> UsageSnapshot: diff --git a/evaluation/tests/web/test_cli.py b/evaluation/coding/swebench_pro/tests/web/test_cli.py similarity index 91% rename from evaluation/tests/web/test_cli.py rename to evaluation/coding/swebench_pro/tests/web/test_cli.py index 3cd39577b3..4b48d095e5 100644 --- a/evaluation/tests/web/test_cli.py +++ b/evaluation/coding/swebench_pro/tests/web/test_cli.py @@ -20,8 +20,9 @@ from typer.testing import CliRunner -from powercontext_eval.cli import _request_worker_stop, app -from powercontext_eval.web.config import WebConfig +from powercontext_eval_swebench_pro.cli import app +from powercontext_eval_swebench_pro.service_cli import _request_worker_stop +from powercontext_eval_swebench_pro.web.config import WebConfig def test_top_level_help_exposes_service_commands() -> None: @@ -45,7 +46,7 @@ def fake_create_app(config: object) -> object: calls["config"] = config return application - monkeypatch.setattr("powercontext_eval.web.api.create_app", fake_create_app) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.api.create_app", fake_create_app) def fake_run(app: object, *, host: str, port: int) -> None: calls.update(app=app, host=host, port=port) @@ -85,8 +86,8 @@ def stop(self) -> None: monkeypatch.setenv("POWERCONTEXT_EVAL_LEASE_SECONDS", "90") monkeypatch.setenv("POWERCONTEXT_EVAL_TOKENSFLOW_EGRESS_NETWORK", "tokensflow-egress") monkeypatch.setenv("POWERCONTEXT_EVAL_PROXY_URL", "http://127.0.0.1:8081") - monkeypatch.setattr("powercontext_eval.web.store.TaskStore", FakeStore) - monkeypatch.setattr("powercontext_eval.web.worker.EvaluationWorker", FakeWorker) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.store.TaskStore", FakeStore) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.EvaluationWorker", FakeWorker) monkeypatch.setattr("signal.getsignal", lambda _signal: signal.SIG_DFL) monkeypatch.setattr("signal.signal", lambda *args: calls.append(("signal", *args))) @@ -122,7 +123,7 @@ def test_worker_signal_handler_ignores_reentrant_sigterm_while_first_stop_is_run import signal import threading - from powercontext_eval.cli import _worker_signal_handlers + from powercontext_eval_swebench_pro.service_cli import _worker_signal_handlers class Worker: diff --git a/evaluation/tests/web/test_config.py b/evaluation/coding/swebench_pro/tests/web/test_config.py similarity index 98% rename from evaluation/tests/web/test_config.py rename to evaluation/coding/swebench_pro/tests/web/test_config.py index 7727c2cf44..0d4210afc6 100644 --- a/evaluation/tests/web/test_config.py +++ b/evaluation/coding/swebench_pro/tests/web/test_config.py @@ -19,12 +19,12 @@ import pytest from pydantic import ValidationError -from powercontext_eval.artifacts import ArmState -from powercontext_eval.runner import INSTANCE_ID -from powercontext_eval.web.batches import BatchCreate, PairCategory -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.controls import BatchPreviewRequest -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.runner import INSTANCE_ID +from powercontext_eval_swebench_pro.web.batches import BatchCreate, PairCategory +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.controls import BatchPreviewRequest +from powercontext_eval_swebench_pro.web.models import ( ArmResponse, ComparisonResponse, EvidenceResponse, @@ -62,7 +62,10 @@ def test_web_config_derives_confined_paths(tmp_path: Path) -> None: assert config.database_path == tmp_path / "web" / "tasks.sqlite3" assert config.run_root == tmp_path - assert config.frontend_dist == tmp_path / "deploy" / "powercontext" / "evaluation" / "web" / "dist" + assert ( + config.frontend_dist + == tmp_path / "deploy" / "powercontext" / "evaluation" / "coding" / "swebench_pro" / "web" / "dist" + ) assert config.tokensflow_enabled is False assert config.tokensflow_binary is None assert config.tokensflow_user_home is None @@ -104,7 +107,10 @@ def test_web_config_derives_portable_layout_from_any_root(tmp_path: Path) -> Non assert config.proxy_url == PROXY_URL assert config.docker_network_pool == "172.30.0.0/15" assert config.extra_no_proxy_hosts == () - assert config.frontend_dist == root / "deploy" / "powercontext" / "evaluation" / "web" / "dist" + assert ( + config.frontend_dist + == root / "deploy" / "powercontext" / "evaluation" / "coding" / "swebench_pro" / "web" / "dist" + ) assert config.usage_pause_percent == 80 assert config.usage_mode == "subscription" assert config.usage_probe_seconds == 60 diff --git a/evaluation/tests/web/test_controls.py b/evaluation/coding/swebench_pro/tests/web/test_controls.py similarity index 94% rename from evaluation/tests/web/test_controls.py rename to evaluation/coding/swebench_pro/tests/web/test_controls.py index 9e3d11c6a9..1eec1bb307 100644 --- a/evaluation/tests/web/test_controls.py +++ b/evaluation/coding/swebench_pro/tests/web/test_controls.py @@ -15,14 +15,14 @@ import pytest from pydantic import ValidationError -from powercontext_eval.web.batches import BatchStatus -from powercontext_eval.web.controls import ( +from powercontext_eval_swebench_pro.web.batches import BatchStatus +from powercontext_eval_swebench_pro.web.controls import ( BatchControlIntent, BatchPauseReason, BatchPreviewRequest, derive_controlled_batch_status, ) -from powercontext_eval.web.models import TaskStatus +from powercontext_eval_swebench_pro.web.models import TaskStatus def test_visible_batch_lifecycle_waits_for_running_task_before_pausing() -> None: diff --git a/evaluation/tests/web/test_deployment.py b/evaluation/coding/swebench_pro/tests/web/test_deployment.py similarity index 97% rename from evaluation/tests/web/test_deployment.py rename to evaluation/coding/swebench_pro/tests/web/test_deployment.py index b8610acaa6..2f42b721e0 100644 --- a/evaluation/tests/web/test_deployment.py +++ b/evaluation/coding/swebench_pro/tests/web/test_deployment.py @@ -20,7 +20,7 @@ import re from pathlib import Path -from powercontext_eval.web.config import WebConfig +from powercontext_eval_swebench_pro.web.config import WebConfig EVALUATION = Path(__file__).resolve().parents[2] DEPLOY = EVALUATION / "deploy" @@ -122,7 +122,7 @@ def test_frontend_build_uses_portable_rollup_runtime() -> None: def test_operator_guide_documents_portable_configuration_and_recovery() -> None: - guide = (EVALUATION / "README.md").read_text().casefold() + guide = (EVALUATION / "docs" / "console.md").read_text().casefold() required = { "powercontext_eval_proxy_url", "powercontext_eval_tokensflow_enabled=true", diff --git a/evaluation/tests/web/test_estimation.py b/evaluation/coding/swebench_pro/tests/web/test_estimation.py similarity index 97% rename from evaluation/tests/web/test_estimation.py rename to evaluation/coding/swebench_pro/tests/web/test_estimation.py index 3e2acfb53d..5fbe339dbc 100644 --- a/evaluation/tests/web/test_estimation.py +++ b/evaluation/coding/swebench_pro/tests/web/test_estimation.py @@ -16,7 +16,7 @@ import pytest -from powercontext_eval.web.estimation import ( +from powercontext_eval_swebench_pro.web.estimation import ( BatchEstimate, EstimateBasis, EstimateQuality, diff --git a/evaluation/tests/web/test_finalization.py b/evaluation/coding/swebench_pro/tests/web/test_finalization.py similarity index 98% rename from evaluation/tests/web/test_finalization.py rename to evaluation/coding/swebench_pro/tests/web/test_finalization.py index 3e5b9ab706..a278f1699b 100644 --- a/evaluation/tests/web/test_finalization.py +++ b/evaluation/coding/swebench_pro/tests/web/test_finalization.py @@ -24,12 +24,12 @@ import pytest -from powercontext_eval.powercontext_sut import DockerSut -from powercontext_eval.process import CommandResult -from powercontext_eval.web.finalization import DockerFinalizationRuntime, TokensFlowFinalizer -from powercontext_eval.web.models import TaskCreate, TaskResult, TaskStatus -from powercontext_eval.web.reporting import tokensflow_finalization_summary -from powercontext_eval.web.store import ( +from powercontext_eval_swebench_pro.powercontext_sut import DockerSut +from powercontext_eval_swebench_pro.process import CommandResult +from powercontext_eval_swebench_pro.web.finalization import DockerFinalizationRuntime, TokensFlowFinalizer +from powercontext_eval_swebench_pro.web.models import TaskCreate, TaskResult, TaskStatus +from powercontext_eval_swebench_pro.web.reporting import tokensflow_finalization_summary +from powercontext_eval_swebench_pro.web.store import ( FinalizationState, TaskStore, TokensFlowFinalizationCreate, diff --git a/evaluation/tests/web/test_reporting.py b/evaluation/coding/swebench_pro/tests/web/test_reporting.py similarity index 93% rename from evaluation/tests/web/test_reporting.py rename to evaluation/coding/swebench_pro/tests/web/test_reporting.py index b254787fba..92bc76abb5 100644 --- a/evaluation/tests/web/test_reporting.py +++ b/evaluation/coding/swebench_pro/tests/web/test_reporting.py @@ -20,10 +20,15 @@ import pytest -from powercontext_eval.artifacts import ArmState -from powercontext_eval.models import TreatmentMode -from powercontext_eval.report import ArmReport, MetricSet, ReportBundle -from powercontext_eval.web.reporting import InvalidReportArtifact, UnsafeReportPath, load_raw_report, load_report +from powercontext_eval_swebench_pro.artifacts import ArmState +from powercontext_eval_swebench_pro.models import TreatmentMode +from powercontext_eval_swebench_pro.report import ArmReport, MetricSet, ReportBundle +from powercontext_eval_swebench_pro.web.reporting import ( + InvalidReportArtifact, + UnsafeReportPath, + load_raw_report, + load_report, +) POWERCONTEXT_SHA = "a" * 40 HARNESS_SHA = "b" * 40 @@ -114,6 +119,10 @@ def test_loads_validated_report_and_derives_exact_comparisons(tmp_path: Path) -> assert response.comparison.input_tokens.percent == pytest.approx(-42.839, abs=0.001) assert response.evidence.off == response.evidence.off.model_copy(update={"scope_id": "eval:run-123:off"}) assert response.evidence.on.mcp_requests == 2 + assert response.evidence.off.database_kind is None + assert response.evidence.off.database_fingerprint is None + assert response.evidence.on.database_kind is None + assert response.evidence.on.database_fingerprint is None assert dict(response.revisions) == {"harness": HARNESS_SHA, "powercontext": POWERCONTEXT_SHA} assert dict(response.configuration) == dict(_bundle().configuration) @@ -224,6 +233,12 @@ def test_accepts_evidence_for_the_scopes_each_arm_registered(tmp_path: Path) -> ("arm", "update"), [ ("off", {"extra": "forbidden"}), + ("off", {"database_kind": "postgresql", "database_fingerprint": "a" * 64}), + ("off", {"database_kind": "sqlite"}), + ("off", {"database_fingerprint": "a" * 64}), + ("off", {"database_kind": "sqlite", "database_fingerprint": "a" * 63}), + ("off", {"database_kind": "oceanbase", "database_fingerprint": "A" * 64}), + ("off", {"database_kind": "oceanbase", "database_fingerprint": "a" * 64 + "\n"}), ("off", {"plugin_checkout_sha": "c" * 40}), ("off", {"plugin_id": "other@plugin"}), ("off", {"plugin_installed": False}), diff --git a/evaluation/tests/web/test_resources.py b/evaluation/coding/swebench_pro/tests/web/test_resources.py similarity index 96% rename from evaluation/tests/web/test_resources.py rename to evaluation/coding/swebench_pro/tests/web/test_resources.py index 463f7b0db9..9ca56a26c2 100644 --- a/evaluation/tests/web/test_resources.py +++ b/evaluation/coding/swebench_pro/tests/web/test_resources.py @@ -24,16 +24,23 @@ import pytest -from powercontext_eval.process import CommandResult -from powercontext_eval.web.batches import BatchCreate -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.models import FailureCategory, FailureCode, SafeFailure, TaskPhase, TaskResult, TaskStatus -from powercontext_eval.web.resources import ( +from powercontext_eval_swebench_pro.process import CommandResult +from powercontext_eval_swebench_pro.web.batches import BatchCreate +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.models import ( + FailureCategory, + FailureCode, + SafeFailure, + TaskPhase, + TaskResult, + TaskStatus, +) +from powercontext_eval_swebench_pro.web.resources import ( AttemptLifecycleCleaner, FilesystemResourceProbe, SucceededWorkspaceReclaimer, ) -from powercontext_eval.web.store import TaskStore, TokensFlowFinalizationCreate +from powercontext_eval_swebench_pro.web.store import TaskStore, TokensFlowFinalizationCreate NOW = datetime(2026, 8, 12, tzinfo=UTC) diff --git a/evaluation/tests/web/test_store.py b/evaluation/coding/swebench_pro/tests/web/test_store.py similarity index 99% rename from evaluation/tests/web/test_store.py rename to evaluation/coding/swebench_pro/tests/web/test_store.py index a6cfccd30e..22705bbd8d 100644 --- a/evaluation/tests/web/test_store.py +++ b/evaluation/coding/swebench_pro/tests/web/test_store.py @@ -21,17 +21,17 @@ import pytest -from powercontext_eval.models import Arm, TreatmentMode -from powercontext_eval.paths import EvaluationPaths -from powercontext_eval.web.baselines import ( +from powercontext_eval_swebench_pro.models import Arm, TreatmentMode +from powercontext_eval_swebench_pro.paths import EvaluationPaths +from powercontext_eval_swebench_pro.web.baselines import ( BaselineCreate, BaselineItemRecord, BaselineSelection, BaselineSnapshot, ) -from powercontext_eval.web.batches import BatchControlEventType, BatchCreate, BatchStatus -from powercontext_eval.web.controls import BatchControlIntent, BatchPauseReason -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.web.batches import BatchControlEventType, BatchCreate, BatchStatus +from powercontext_eval_swebench_pro.web.controls import BatchControlIntent, BatchPauseReason +from powercontext_eval_swebench_pro.web.models import ( FailureCategory, FailureCode, RetryDisposition, @@ -41,7 +41,7 @@ TaskResult, TaskStatus, ) -from powercontext_eval.web.store import ( +from powercontext_eval_swebench_pro.web.store import ( FinalizationState, TaskAdmissionRejected, TaskConflict, @@ -50,7 +50,7 @@ TaskStore, TokensFlowFinalizationCreate, ) -from powercontext_eval.web.usage import UsageSnapshot +from powercontext_eval_swebench_pro.web.usage import UsageSnapshot NOW = datetime(2026, 7, 29, 1, 2, 3, tzinfo=UTC) @@ -429,7 +429,7 @@ def connect_with_failing_drop( factory=FailingDropConnection, ) - monkeypatch.setattr("powercontext_eval.web.store.sqlite3.connect", connect_with_failing_drop) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.store.sqlite3.connect", connect_with_failing_drop) store = TaskStore(database, lease_duration=timedelta(seconds=60)) with pytest.raises(RuntimeError, match="injected migration failure"): diff --git a/evaluation/tests/web/test_usage.py b/evaluation/coding/swebench_pro/tests/web/test_usage.py similarity index 99% rename from evaluation/tests/web/test_usage.py rename to evaluation/coding/swebench_pro/tests/web/test_usage.py index 042bbfee79..5aed794f17 100644 --- a/evaluation/tests/web/test_usage.py +++ b/evaluation/coding/swebench_pro/tests/web/test_usage.py @@ -20,7 +20,7 @@ import pytest from pydantic import ValidationError -from powercontext_eval.web.usage import ( +from powercontext_eval_swebench_pro.web.usage import ( CLIENT_INFO, CodexUsageProbe, UsageProtocolError, diff --git a/evaluation/tests/web/test_worker.py b/evaluation/coding/swebench_pro/tests/web/test_worker.py similarity index 95% rename from evaluation/tests/web/test_worker.py rename to evaluation/coding/swebench_pro/tests/web/test_worker.py index e924454c32..8fc96e4a9a 100644 --- a/evaluation/tests/web/test_worker.py +++ b/evaluation/coding/swebench_pro/tests/web/test_worker.py @@ -25,13 +25,13 @@ import pytest -from powercontext_eval.benchmarks.base import GoldCheckFailed -from powercontext_eval.benchmarks.swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance -from powercontext_eval.benchmarks.swebench_pro.evaluator import OfficialResultError -from powercontext_eval.codex import CodexCapacityError, CodexInfrastructureError -from powercontext_eval.errors import CommandFailed, GitSourceError -from powercontext_eval.models import Arm -from powercontext_eval.powercontext_sut import ( +from powercontext_eval_swebench_pro.adapter import DatasetSchemaError, SweBenchProInstance +from powercontext_eval_swebench_pro.codex import CodexCapacityError, CodexInfrastructureError +from powercontext_eval_swebench_pro.errors import CommandFailed, GitSourceError +from powercontext_eval_swebench_pro.evaluator import OfficialResultError +from powercontext_eval_swebench_pro.gold import GoldCheckFailed +from powercontext_eval_swebench_pro.models import Arm +from powercontext_eval_swebench_pro.powercontext_sut import ( InvalidTreatment, PluginInspectionFailure, PluginInspectionFailureReason, @@ -39,13 +39,13 @@ ReadinessFailureReason, UnsafeSutConfiguration, ) -from powercontext_eval.process import CommandResult -from powercontext_eval.runner import MinimalRunConfig, MinimalRunResult, RunConfig, RunPhase -from powercontext_eval.tokensflow import TokensFlowFinalizationDescriptor, TokensFlowInfrastructureError -from powercontext_eval.web.batches import BatchControlEventType, BatchCreate, BatchStatus -from powercontext_eval.web.config import WebConfig -from powercontext_eval.web.controls import BatchControlIntent, BatchPauseReason -from powercontext_eval.web.models import ( +from powercontext_eval_swebench_pro.process import CommandResult +from powercontext_eval_swebench_pro.runner import MinimalRunConfig, MinimalRunResult, RunConfig, RunPhase +from powercontext_eval_swebench_pro.tokensflow import TokensFlowFinalizationDescriptor, TokensFlowInfrastructureError +from powercontext_eval_swebench_pro.web.batches import BatchControlEventType, BatchCreate, BatchStatus +from powercontext_eval_swebench_pro.web.config import WebConfig +from powercontext_eval_swebench_pro.web.controls import BatchControlIntent, BatchPauseReason +from powercontext_eval_swebench_pro.web.models import ( FailureCategory, FailureCode, RetryDisposition, @@ -54,11 +54,11 @@ TaskPhase, TaskStatus, ) -from powercontext_eval.web.resources import FilesystemCapacity, FilesystemResourceProbe -from powercontext_eval.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision -from powercontext_eval.web.store import TaskOwnershipError, TaskStore -from powercontext_eval.web.usage import CodexUsageProbe, UsageSnapshot, UsageUnavailable -from powercontext_eval.web.worker import EvaluationWorker, TaskPairWorker +from powercontext_eval_swebench_pro.web.resources import FilesystemCapacity, FilesystemResourceProbe +from powercontext_eval_swebench_pro.web.revision import RUNTIME_SCHEMA_VERSION, current_build_revision +from powercontext_eval_swebench_pro.web.store import TaskOwnershipError, TaskStore +from powercontext_eval_swebench_pro.web.usage import CodexUsageProbe, UsageSnapshot, UsageUnavailable +from powercontext_eval_swebench_pro.web.worker import EvaluationWorker, TaskPairWorker NOW = datetime(2026, 7, 29, 1, 2, 3, tzinfo=UTC) @@ -106,7 +106,7 @@ def _default_safe_usage_probe(monkeypatch: pytest.MonkeyPatch) -> None: total_inodes=200_000_000, ), ) - monkeypatch.setattr("powercontext_eval.web.resources.DockerDependencyProbe.check", lambda _self: None) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.resources.DockerDependencyProbe.check", lambda _self: None) def _config( @@ -331,7 +331,7 @@ def runner(run_config: Any, *, instance: object, on_phase: Any) -> MinimalRunRes report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -368,7 +368,7 @@ def runner(run_config: Any, *, instance: object, on_phase: Any) -> MinimalRunRes report.write_text("safe") return MinimalRunResult(run_config.run_id, report, False, False) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -505,7 +505,7 @@ def test_latest_is_pinned_once_and_every_child_uses_catalog_instance( source = FakeSource() catalog = FakeCatalog(instance_ids) calls: list[tuple[RunConfig, SweBenchProInstance]] = [] - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -538,7 +538,7 @@ def test_worker_uses_each_batch_immutable_model_for_runner_configuration( model="gpt-5.6-luna", ) calls: list[tuple[RunConfig, SweBenchProInstance]] = [] - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -622,7 +622,7 @@ def runner(run_config: Any, *, instance: object, on_phase: Any) -> MinimalRunRes report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -680,7 +680,7 @@ def runner(run_config: Any, *, instance: object, on_phase: Any) -> MinimalRunRes report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = _capacity_worker(config, store, instance_ids, runner) assert worker.run_once() is True @@ -827,7 +827,7 @@ def test_restart_reuses_persisted_batch_sha_and_completed_children( batch = _create_batch(store, instance_ids=instance_ids) catalog = FakeCatalog(instance_ids) calls: list[tuple[RunConfig, SweBenchProInstance]] = [] - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) first_source = FakeSource() first = EvaluationWorker( config, @@ -891,7 +891,7 @@ def runner(run_config: Any, *, on_phase: Any) -> MinimalRunResult: loaded = [] monkeypatch.setattr( - "powercontext_eval.web.worker.load_report", + "powercontext_eval_swebench_pro.web.worker.load_report", lambda run_dir, run_root: loaded.append((run_dir, run_root)) or object(), ) worker = EvaluationWorker(config, store, runner=runner, clock=lambda: NOW) @@ -1245,7 +1245,7 @@ def runner(run_config: Any, *, on_phase: Any) -> MinimalRunResult: path.write_text("report") return MinimalRunResult(task.task_id, path, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) def thread_factory(**kwargs: Any) -> RecordingThread: thread = RecordingThread(**kwargs) @@ -1404,7 +1404,7 @@ def runner(run_config: Any, *, on_phase: Any) -> MinimalRunResult: returned = expected return MinimalRunResult(task.task_id, returned, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) assert EvaluationWorker(config, store, runner=runner, clock=lambda: NOW).run_once() is True failed = store.get(task.task_id) assert failed.status is TaskStatus.FAILED @@ -1556,7 +1556,7 @@ def runner(run_config: Any, *, instance: object, on_phase: Any) -> MinimalRunRes report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker( config, store, @@ -1603,7 +1603,7 @@ def runner(run_config: Any, *, on_phase: Any) -> MinimalRunResult: report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) slot = TaskPairWorker(config, store, runner=runner, worker_id="direct-slot", clock=lambda: NOW) assert slot.run_once() is True @@ -1758,7 +1758,7 @@ def runner(run_config: Any, *, on_phase: Any) -> MinimalRunResult: report.write_text("safe") return MinimalRunResult(run_config.run_id, report, True, True) - monkeypatch.setattr("powercontext_eval.web.worker.load_report", lambda *args: object()) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.load_report", lambda *args: object()) worker = EvaluationWorker(config, store, runner=runner, worker_id="failing-supervisor", clock=lambda: NOW) failing_slot = worker._slots[0] @@ -1828,7 +1828,7 @@ def join(self) -> None: joined.append(self.index) self.inner.join() - monkeypatch.setattr("powercontext_eval.web.worker.threading.Thread", ControlledThread) + monkeypatch.setattr("powercontext_eval_swebench_pro.web.worker.threading.Thread", ControlledThread) with pytest.raises(RuntimeError, match="^Evaluation worker slot failed$") as raised: worker.run_forever() diff --git a/evaluation/web/e2e/console.e2e.spec.ts b/evaluation/coding/swebench_pro/web/e2e/console.e2e.spec.ts similarity index 73% rename from evaluation/web/e2e/console.e2e.spec.ts rename to evaluation/coding/swebench_pro/web/e2e/console.e2e.spec.ts index 26d57f6570..b22f286887 100644 --- a/evaluation/web/e2e/console.e2e.spec.ts +++ b/evaluation/coding/swebench_pro/web/e2e/console.e2e.spec.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { copyFile } from "node:fs/promises"; - import { expect, test, type Page } from "@playwright/test"; interface BatchTaskItem { @@ -56,8 +54,8 @@ async function batchTasks(page: Page, batchId: string): Promise { async function usedPercent(page: Page): Promise { return page.evaluate(async () => { const response = await fetch("/api/account-usage"); - const body = await response.json() as { used_percent: number }; - return body.used_percent; + const body = await response.json() as { usage: { used_percent: number } }; + return body.usage.used_percent; }); } @@ -81,20 +79,17 @@ test.afterEach(async ({ page }) => { expect((page as Page & { browserErrors?: string[] }).browserErrors ?? []).toEqual([]); }); -test("controls a serial batch at task boundaries and retains a retried attempt", async ({ page }, testInfo) => { +test("controls a serial batch at task boundaries and cancels a queued retry", async ({ page }, testInfo) => { await page.goto("/"); await expect(page.getByRole("navigation", { name: "报告导航" })).toBeVisible(); await expect(page.getByRole("heading", { name: "总体报告" })).toBeVisible(); await expect(page.getByLabel("PowerContext 版本")).toHaveValue("latest"); await expect(page.getByLabel("测试实例")).toHaveCount(0); - await page.getByRole("button", { name: "预览评测" }).click(); - await expect(page.getByRole("region", { name: "评测确认" })).toBeVisible(); - await expect(page.getByRole("heading", { name: "6 个基准任务" })).toBeVisible(); - await expect(page.getByText("当前用量 20%")).toBeVisible(); await expect(page.getByRole("link", { name: "任务详细报告" })).toHaveAttribute("aria-disabled", "true"); + expect(await usedPercent(page)).toBe(20); - await page.getByRole("button", { name: "确认并开始评测" }).click(); + await page.getByRole("button", { name: "开始评测" }).click(); await expect(page).toHaveURL(/\/report\/batch-/); const batchId = decodeURIComponent(new URL(page.url()).pathname.split("/")[2] ?? ""); expect(batchId).toMatch(/^batch-/); @@ -113,14 +108,20 @@ test("controls a serial batch at task boundaries and retains a retried attempt", await page.getByRole("button", { name: "暂停" }).click(); await expect(page.getByText("等待当前任务完成")).toBeVisible(); await waitForBatch(page, batchId, "paused"); + await expect.poll(async () => (await batchTasks(page, batchId)).items[0]?.attempt_count).toBe(2); const afterPause = await batchTasks(page, batchId); expect(afterPause.items[0]).toMatchObject({ instance_id: "instance_e2e__repo-a", - status: "failed", - attempt_count: 1, - retryable: true, + status: "queued", + attempt_count: 2, }); expect(afterPause.items.slice(1).every((task) => task.status === "queued")).toBe(true); + const attemptsResponse = await page.request.get( + `/api/batches/${encodeURIComponent(batchId)}/tasks/${encodeURIComponent(afterPause.items[0]!.task_id)}/attempts`, + ); + expect(attemptsResponse.ok()).toBe(true); + const pausedAttempts = await attemptsResponse.json() as { status: string; eligible_at: string }[]; + expect(pausedAttempts.map((attempt) => attempt.status)).toEqual(["failed", "queued"]); await page.reload(); await expect(page.getByText("Codex 账户用量").locator("..").getByText("40%")).toBeVisible(); @@ -131,28 +132,28 @@ test("controls a serial batch at task boundaries and retains a retried attempt", async () => (await batchTasks(page, batchId)).items[1]?.status, { intervals: [30] }, ).toBe("running"); - await waitForBatch(page, batchId, "paused"); - const thresholdPause = await batchState(page, batchId); - expect(thresholdPause.control).toMatchObject({ - intent: "pause", - pause_reason: "usage_threshold", + await expect.poll(async () => (await batchTasks(page, batchId)).items[1]?.status).toBe("succeeded"); + await expect.poll(async () => usedPercent(page)).toBe(81); + const thresholdGate = await batchState(page, batchId); + expect(thresholdGate.control).toMatchObject({ + intent: "run", + pause_reason: null, usage_pause_percent: 80, }); const afterThreshold = await batchTasks(page, batchId); + expect(afterThreshold.items[0]).toMatchObject({ status: "queued", attempt_count: 2 }); expect(afterThreshold.items[1]?.status).toBe("succeeded"); expect(afterThreshold.items[2]?.status).toBe("queued"); await page.reload(); await expect(page.getByText("Codex 账户用量").locator("..").getByText("81%")).toBeVisible(); - await expect(page.getByText("暂停原因：达到用量阈值")).toBeVisible(); + await expect(page.getByText("暂停原因：达到用量阈值")).toHaveCount(0); await page.getByLabel("批次暂停阈值").fill("90"); await page.getByRole("button", { name: "保存阈值" }).click(); await expect.poll(async () => (await batchState(page, batchId)).control.usage_pause_percent).toBe(90); - expect((await batchState(page, batchId)).status).toBe("paused"); - expect((await batchTasks(page, batchId)).items[2]?.status).toBe("queued"); + expect((await batchState(page, batchId)).control.intent).toBe("run"); expect(await usedPercent(page)).toBe(81); - await page.getByRole("button", { name: "继续运行" }).click(); await expect.poll( async () => (await batchTasks(page, batchId)).items[2]?.status, { intervals: [30] }, @@ -161,60 +162,34 @@ test("controls a serial batch at task boundaries and retains a retried attempt", await expect(page.getByText("等待当前任务完成后取消")).toBeVisible(); await waitForBatch(page, batchId, "cancelled"); const afterCancel = await batchTasks(page, batchId); + expect(afterCancel.items[0]).toMatchObject({ status: "cancelled", attempt_count: 2 }); expect(afterCancel.items[2]?.status).toBe("succeeded"); expect(afterCancel.items.slice(3).every((task) => task.status === "cancelled")).toBe(true); expect(await usedPercent(page)).toBe(50); await page.getByRole("navigation", { name: "报告导航" }).getByRole("link", { name: "任务详细报告" }).click(); await page.reload(); - await page.getByRole("button", { name: "评测执行失败" }).click(); await expect(page.getByText("e2e/repo-a")).toBeVisible(); - const failedRow = page.locator("tr", { hasText: "e2e/repo-a" }); - const taskId = (await failedRow.locator(".task-cell-id").textContent())?.trim() ?? ""; + const retriedRow = page.locator("tr", { hasText: "e2e/repo-a" }); + const taskId = (await retriedRow.locator(".task-cell-id").textContent())?.trim() ?? ""; expect(taskId).toMatch(/^run-/); - await failedRow.getByRole("link", { name: `查看 ${taskId}` }).click(); + await retriedRow.getByRole("link", { name: `查看 ${taskId}` }).click(); await expect(page.getByRole("heading", { name: "单任务详情" })).toBeVisible(); - await expect(page.getByText("评测执行失败", { exact: true }).first()).toBeVisible(); - await expect(page.getByRole("button", { name: "尝试 1" })).toBeVisible(); - await page.getByRole("button", { name: "重试此任务" }).click(); - await expect(page.getByText("已有尝试和证据不会被覆盖")).toBeVisible(); - await page.getByRole("button", { name: "确认重试" }).click(); - - await expect.poll( - async () => (await batchTasks(page, batchId)).items[0]?.status, - { intervals: [50], timeout: 15_000 }, - ).toBe("succeeded"); - await waitForBatch(page, batchId, "completed"); - await page.reload(); await expect(page.getByRole("button", { name: "尝试 1" })).toBeVisible(); await expect(page.getByRole("button", { name: "尝试 2" })).toBeVisible(); await page.getByRole("button", { name: "尝试 1" }).click(); - await expect(page.getByText("Codex execution failed.")).toBeVisible(); - await page.getByRole("button", { name: "尝试 2" }).click(); - await expect(page.getByText("OFF 未通过")).toBeVisible(); - await expect(page.getByText("ON 通过")).toBeVisible(); - await expect(page.getByRole("button", { name: /#3.*PowerContext 注入/ })).toBeVisible(); - await expect(page.getByRole("button", { name: /#4.*PowerContext 注入/ })).toBeVisible(); - await page.getByRole("button", { name: /#3.*PowerContext 注入/ }).click(); - const eventDetail = page.getByLabel("事件详情"); - await expect(eventDetail.getByText("e2e/repo-a architecture")).toBeVisible(); - await expect(eventDetail.getByText(/memory:\/\/architecture\/1/)).toBeVisible(); - await expect(eventDetail.getByText("PowerContext recalled the repository service boundary.")).toBeVisible(); - - await page.getByRole("button", { name: "OFF 时间线" }).click(); - await expect(page.getByRole("button", { name: /PowerContext 注入/ })).toHaveCount(0); - await page.getByRole("button", { name: "ON 时间线" }).click(); - await expect(page.getByRole("button", { name: /#3.*PowerContext 注入/ })).toBeVisible(); + await expect(page.getByRole("paragraph").filter({ hasText: "Codex execution infrastructure failed." })).toBeVisible(); + await expect(page.getByRole("button", { name: "尝试 2" })).toContainText("已取消"); await page.getByRole("navigation", { name: "报告导航" }).getByRole("link", { name: "总体报告" }).click(); await page.reload(); const correctness = page.getByLabel("正确性汇总"); await expect(correctness.getByText("6", { exact: true })).toBeVisible(); - await expect(page.getByText("可比较任务 3 / 6")).toBeVisible(); - await expect(page.getByRole("link", { name: /OFF 未通过.*ON 通过.*1/ })).toBeVisible(); + await expect(page.getByText("可比较任务 2 / 6")).toBeVisible(); + await expect(page.getByRole("link", { name: /OFF 未通过.*ON 通过.*0/ })).toBeVisible(); await expect(page.getByRole("link", { name: /OFF 通过.*ON 未通过.*1/ })).toBeVisible(); - await expect(page.getByText("已取消 3")).toBeVisible(); + await expect(page.getByText("已取消 4")).toBeVisible(); await expect(page.locator("body")).not.toContainText(/提升|退化|验收有效|验收无效|优先分析/); const controlEvents = await page.evaluate(async (id) => { const response = await fetch(`/api/batches/${encodeURIComponent(id)}/control-events`); @@ -227,12 +202,10 @@ test("controls a serial batch at task boundaries and retains a retried attempt", "paused", "resume_requested", "resumed", - "usage_threshold_reached", "threshold_changed", "cancel_requested", "cancelled", "task_retry_requested", - "batch_completed", ])); for (const width of [1440, 960]) { @@ -249,7 +222,48 @@ test("controls a serial batch at task boundaries and retains a retried attempt", } const reviewScreenshot = testInfo.outputPath("controlled-batch-overview-review.png"); await page.screenshot({ path: reviewScreenshot, fullPage: true }); - await copyFile(reviewScreenshot, "/tmp/powercontext-batch-evaluation-console.png"); +}); + +test("retains failed and successful evidence after an automatic retry", async ({ page }) => { + // Keep the production 30-second retry backoff instead of changing the worker's scheduler. + test.setTimeout(60_000); + await page.goto("/"); + await page.getByLabel("暂停阈值", { exact: true }).fill("90"); + await page.getByRole("button", { name: "开始评测" }).click(); + await expect(page).toHaveURL(/\/report\/batch-/); + const batchId = decodeURIComponent(new URL(page.url()).pathname.split("/")[2] ?? ""); + await expect.poll(async () => (await batchTasks(page, batchId)).items[0]?.attempt_count).toBe(2); + await expect.poll( + async () => (await batchTasks(page, batchId)).items[0]?.status, + { intervals: [100], timeout: 45_000 }, + ).toBe("succeeded"); + await waitForBatch(page, batchId, "completed"); + const tasks = await batchTasks(page, batchId); + expect(tasks.items.every((task) => task.status === "succeeded")).toBe(true); + const taskId = tasks.items[0]!.task_id; + + await page.getByRole("navigation", { name: "报告导航" }).getByRole("link", { name: "任务详细报告" }).click(); + const retriedRow = page.locator("tr", { hasText: "e2e/repo-a" }); + await retriedRow.getByRole("link", { name: `查看 ${taskId}` }).click(); + await expect(page.getByRole("heading", { name: "单任务详情" })).toBeVisible(); + await expect(page.getByRole("button", { name: "尝试 1" })).toBeVisible(); + await expect(page.getByRole("button", { name: "尝试 2" })).toBeVisible(); + await page.getByRole("button", { name: "尝试 1" }).click(); + await expect(page.getByRole("paragraph").filter({ hasText: "Codex execution infrastructure failed." })).toBeVisible(); + await page.getByRole("button", { name: "尝试 2" }).click(); + await expect(page.getByText("OFF 未通过")).toBeVisible(); + await expect(page.getByText("ON 通过")).toBeVisible(); + await expect(page.getByRole("button", { name: /#3.*PowerContext 注入/ })).toBeVisible(); + await expect(page.getByRole("button", { name: /#4.*PowerContext 注入/ })).toBeVisible(); + await page.getByRole("button", { name: /#3.*PowerContext 注入/ }).click(); + const eventDetail = page.getByLabel("事件详情"); + await expect(eventDetail.getByText("e2e/repo-a architecture")).toBeVisible(); + await expect(eventDetail.getByText(/memory:\/\/architecture\/1/)).toBeVisible(); + await expect(eventDetail.getByText("PowerContext recalled the repository service boundary.")).toBeVisible(); + await page.getByRole("button", { name: "OFF 时间线" }).click(); + await expect(page.getByRole("button", { name: /PowerContext 注入/ })).toHaveCount(0); + await page.getByRole("button", { name: "ON 时间线" }).click(); + await expect(page.getByRole("button", { name: /#3.*PowerContext 注入/ })).toBeVisible(); }); test("keeps task detail contextual at desktop width", async ({ page }) => { diff --git a/evaluation/web/index.html b/evaluation/coding/swebench_pro/web/index.html similarity index 100% rename from evaluation/web/index.html rename to evaluation/coding/swebench_pro/web/index.html diff --git a/evaluation/web/package-lock.json b/evaluation/coding/swebench_pro/web/package-lock.json similarity index 100% rename from evaluation/web/package-lock.json rename to evaluation/coding/swebench_pro/web/package-lock.json diff --git a/evaluation/web/package.json b/evaluation/coding/swebench_pro/web/package.json similarity index 100% rename from evaluation/web/package.json rename to evaluation/coding/swebench_pro/web/package.json diff --git a/evaluation/web/playwright.config.ts b/evaluation/coding/swebench_pro/web/playwright.config.ts similarity index 93% rename from evaluation/web/playwright.config.ts rename to evaluation/coding/swebench_pro/web/playwright.config.ts index a48211bf8f..90937c26d4 100644 --- a/evaluation/web/playwright.config.ts +++ b/evaluation/coding/swebench_pro/web/playwright.config.ts @@ -33,7 +33,7 @@ export default defineConfig({ trace: "retain-on-failure", }, webServer: { - command: "npm run build && exec ../.venv/bin/python ../tests/web/fake_runner_app.py", + command: "npm run build && exec ../../../.venv/bin/python ../tests/web/fake_runner_app.py", url: "http://localhost:4177/api/health", reuseExistingServer: false, timeout: 120_000, diff --git a/evaluation/web/src/App.test.tsx b/evaluation/coding/swebench_pro/web/src/App.test.tsx similarity index 100% rename from evaluation/web/src/App.test.tsx rename to evaluation/coding/swebench_pro/web/src/App.test.tsx diff --git a/evaluation/web/src/App.tsx b/evaluation/coding/swebench_pro/web/src/App.tsx similarity index 100% rename from evaluation/web/src/App.tsx rename to evaluation/coding/swebench_pro/web/src/App.tsx diff --git a/evaluation/web/src/api.test.ts b/evaluation/coding/swebench_pro/web/src/api.test.ts similarity index 88% rename from evaluation/web/src/api.test.ts rename to evaluation/coding/swebench_pro/web/src/api.test.ts index 153f66ecf4..10765f024c 100644 --- a/evaluation/web/src/api.test.ts +++ b/evaluation/coding/swebench_pro/web/src/api.test.ts @@ -17,7 +17,7 @@ import { describe, expect, it, vi } from "vitest"; import { ApiError, EvaluationApi } from "./api"; -import { batchReport } from "./test/fixtures"; +import { batchReport, report as sampleReport } from "./test/fixtures"; const validTask = { powercontext_ref: "latest", @@ -49,6 +49,26 @@ const queuedTask = { queue_position: 1, } as const; +const failedAttempt = { + attempt_id: "task-1.attempt-0001", + task_id: "task-1", + attempt_number: 1, + status: "failed", + phase: "running_off", + created_at: "2026-10-05T18:00:00Z", + eligible_at: "2026-10-05T18:00:00Z", + started_at: "2026-10-05T18:00:01Z", + finished_at: "2026-10-05T18:00:02Z", + version: 7, + failure_category: "codex_execution_failure", + failure_code: "codex_execution", + retry_disposition: "retry", + failure_phase: "running_off", + failure_summary: "Codex execution infrastructure failed.", + result: null, + retryable: true, +} as const; + function jsonResponse(value: unknown, status = 200): Response { return new Response(JSON.stringify(value), { status, @@ -65,6 +85,42 @@ function apiWithResponse(response: Response): { } describe("EvaluationApi HTTP", () => { + it("loads retained failed and successful attempts with retry metadata", async () => { + const response = [failedAttempt, { + ...failedAttempt, + attempt_id: "task-1.attempt-0002", + attempt_number: 2, + status: "succeeded", + phase: "generating_report", + created_at: "2026-10-05T18:00:03Z", + eligible_at: "2026-10-05T18:00:33Z", + started_at: "2026-10-05T18:00:33Z", + finished_at: "2026-10-05T18:00:34Z", + failure_category: null, + failure_code: null, + retry_disposition: null, + failure_phase: null, + failure_summary: null, + result: { + artifact_dir: "runs/task-1-attempt-0002", + report_path: "runs/task-1-attempt-0002/report.md", + off_resolved: false, + on_resolved: true, + }, + retryable: false, + }]; + const { api, fetch } = apiWithResponse(jsonResponse(response)); + + await expect(api.listTaskAttempts("batch-1", "task-1")).resolves.toEqual(response); + expect(fetch).toHaveBeenCalledWith("/api/batches/batch-1/tasks/task-1/attempts", expect.any(Object)); + }); + + it("rejects undeclared fields in attempt history", async () => { + const { api } = apiWithResponse(jsonResponse([{ ...failedAttempt, private_detail: "not public" }])); + + await expect(api.listTaskAttempts("batch-1", "task-1")).rejects.toMatchObject({ code: "invalid_response" }); + }); + it("accepts an API-key admission response without subscription usage", async () => { const response = { mode: "api_key", sufficient: true, usage: null } as const; const { api, fetch } = apiWithResponse(jsonResponse(response)); @@ -432,6 +488,44 @@ describe("EvaluationApi HTTP", () => { await expect(api.getReport("task-1")).resolves.toEqual(report); }); + it.each([true, false])("loads report database provenance when configured=%s", async (configured) => { + const report = { + ...sampleReport, + evidence: { + off: { + ...sampleReport.evidence.off, + database_kind: configured ? "sqlite" : null, + database_fingerprint: configured ? "a".repeat(64) : null, + }, + on: { + ...sampleReport.evidence.on, + database_kind: configured ? "oceanbase" : null, + database_fingerprint: configured ? "b".repeat(64) : null, + }, + }, + }; + const { api } = apiWithResponse(jsonResponse(report)); + + await expect(api.getReport("task-report")).resolves.toEqual(report); + }); + + it("rejects a database fingerprint with a trailing newline", async () => { + const report = { + ...sampleReport, + evidence: { + ...sampleReport.evidence, + on: { + ...sampleReport.evidence.on, + database_kind: "oceanbase", + database_fingerprint: "b".repeat(64) + "\n", + }, + }, + }; + const { api } = apiWithResponse(jsonResponse(report)); + + await expect(api.getReport("task-report")).rejects.toMatchObject({ code: "invalid_response" }); + }); + it("loads raw report markdown only from a text response", async () => { const fetch = vi.fn().mockResolvedValue( new Response("# Report", { headers: { "Content-Type": "text/plain; charset=utf-8" } }), diff --git a/evaluation/web/src/api.ts b/evaluation/coding/swebench_pro/web/src/api.ts similarity index 99% rename from evaluation/web/src/api.ts rename to evaluation/coding/swebench_pro/web/src/api.ts index 064abfeb52..32b292dbdf 100644 --- a/evaluation/web/src/api.ts +++ b/evaluation/coding/swebench_pro/web/src/api.ts @@ -325,6 +325,8 @@ const treatmentEvidenceSchema = z.strictObject({ scope_id: z.string(), scope_key: z.string().min(1).nullable(), server_ready: z.boolean(), + database_kind: z.enum(["sqlite", "oceanbase"]).nullable().optional(), + database_fingerprint: z.string().length(64).regex(/^[0-9a-f]{64}$/).nullable().optional(), }); const goldValidationAuditSchema = z.strictObject({ instance_id: z.string(), @@ -553,10 +555,13 @@ const taskAttemptSchema = z.strictObject({ status: taskStatusSchema, phase: taskPhaseSchema.nullable(), created_at: timestampSchema, + eligible_at: timestampSchema, started_at: timestampSchema.nullable(), finished_at: timestampSchema.nullable(), version: nonnegativeIntegerSchema, failure_category: failureCategorySchema.nullable(), + failure_code: z.string().nullable(), + retry_disposition: z.enum(["retry", "terminal"]).nullable(), failure_phase: taskPhaseSchema.nullable(), failure_summary: z.string().max(500).nullable(), result: taskResultSchema.nullable(), diff --git a/evaluation/web/src/batchStatus.ts b/evaluation/coding/swebench_pro/web/src/batchStatus.ts similarity index 100% rename from evaluation/web/src/batchStatus.ts rename to evaluation/coding/swebench_pro/web/src/batchStatus.ts diff --git a/evaluation/web/src/components/AppShell.tsx b/evaluation/coding/swebench_pro/web/src/components/AppShell.tsx similarity index 100% rename from evaluation/web/src/components/AppShell.tsx rename to evaluation/coding/swebench_pro/web/src/components/AppShell.tsx diff --git a/evaluation/web/src/components/AttemptHistory.test.tsx b/evaluation/coding/swebench_pro/web/src/components/AttemptHistory.test.tsx similarity index 96% rename from evaluation/web/src/components/AttemptHistory.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/AttemptHistory.test.tsx index d689912ecc..dd5a216f1c 100644 --- a/evaluation/web/src/components/AttemptHistory.test.tsx +++ b/evaluation/coding/swebench_pro/web/src/components/AttemptHistory.test.tsx @@ -29,10 +29,13 @@ const firstAttempt: TaskAttempt = { status: "failed", phase: "running_on", created_at: "2026-07-29T01:00:00Z", + eligible_at: "2026-07-29T01:00:00Z", started_at: "2026-07-29T01:01:00Z", finished_at: "2026-07-29T01:02:00Z", version: 3, failure_category: "codex_execution_failure", + failure_code: "codex_execution", + retry_disposition: "retry", failure_phase: "running_on", failure_summary: "Codex execution did not complete", result: null, @@ -89,6 +92,8 @@ describe("AttemptHistory", () => { attempt_number: 2, status: "succeeded" as const, failure_category: null, + failure_code: null, + retry_disposition: null, failure_phase: null, failure_summary: null, result: { diff --git a/evaluation/web/src/components/AttemptHistory.tsx b/evaluation/coding/swebench_pro/web/src/components/AttemptHistory.tsx similarity index 100% rename from evaluation/web/src/components/AttemptHistory.tsx rename to evaluation/coding/swebench_pro/web/src/components/AttemptHistory.tsx diff --git a/evaluation/web/src/components/AuthPanel.tsx b/evaluation/coding/swebench_pro/web/src/components/AuthPanel.tsx similarity index 100% rename from evaluation/web/src/components/AuthPanel.tsx rename to evaluation/coding/swebench_pro/web/src/components/AuthPanel.tsx diff --git a/evaluation/web/src/components/BaselineLibrary.tsx b/evaluation/coding/swebench_pro/web/src/components/BaselineLibrary.tsx similarity index 100% rename from evaluation/web/src/components/BaselineLibrary.tsx rename to evaluation/coding/swebench_pro/web/src/components/BaselineLibrary.tsx diff --git a/evaluation/web/src/components/BatchControls.test.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchControls.test.tsx similarity index 100% rename from evaluation/web/src/components/BatchControls.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchControls.test.tsx diff --git a/evaluation/web/src/components/BatchControls.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchControls.tsx similarity index 100% rename from evaluation/web/src/components/BatchControls.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchControls.tsx diff --git a/evaluation/web/src/components/BatchLauncher.test.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchLauncher.test.tsx similarity index 100% rename from evaluation/web/src/components/BatchLauncher.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchLauncher.test.tsx diff --git a/evaluation/web/src/components/BatchLauncher.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchLauncher.tsx similarity index 100% rename from evaluation/web/src/components/BatchLauncher.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchLauncher.tsx diff --git a/evaluation/web/src/components/BatchOverview.test.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchOverview.test.tsx similarity index 100% rename from evaluation/web/src/components/BatchOverview.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchOverview.test.tsx diff --git a/evaluation/web/src/components/BatchOverview.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchOverview.tsx similarity index 100% rename from evaluation/web/src/components/BatchOverview.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchOverview.tsx diff --git a/evaluation/web/src/components/BatchRuntime.test.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchRuntime.test.tsx similarity index 100% rename from evaluation/web/src/components/BatchRuntime.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchRuntime.test.tsx diff --git a/evaluation/web/src/components/BatchRuntime.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchRuntime.tsx similarity index 100% rename from evaluation/web/src/components/BatchRuntime.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchRuntime.tsx diff --git a/evaluation/web/src/components/BatchTaskReport.test.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchTaskReport.test.tsx similarity index 100% rename from evaluation/web/src/components/BatchTaskReport.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchTaskReport.test.tsx diff --git a/evaluation/web/src/components/BatchTaskReport.tsx b/evaluation/coding/swebench_pro/web/src/components/BatchTaskReport.tsx similarity index 100% rename from evaluation/web/src/components/BatchTaskReport.tsx rename to evaluation/coding/swebench_pro/web/src/components/BatchTaskReport.tsx diff --git a/evaluation/web/src/components/ContextTimeline.test.tsx b/evaluation/coding/swebench_pro/web/src/components/ContextTimeline.test.tsx similarity index 100% rename from evaluation/web/src/components/ContextTimeline.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/ContextTimeline.test.tsx diff --git a/evaluation/web/src/components/ContextTimeline.tsx b/evaluation/coding/swebench_pro/web/src/components/ContextTimeline.tsx similarity index 100% rename from evaluation/web/src/components/ContextTimeline.tsx rename to evaluation/coding/swebench_pro/web/src/components/ContextTimeline.tsx diff --git a/evaluation/web/src/components/ReportIndex.test.tsx b/evaluation/coding/swebench_pro/web/src/components/ReportIndex.test.tsx similarity index 100% rename from evaluation/web/src/components/ReportIndex.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/ReportIndex.test.tsx diff --git a/evaluation/web/src/components/ReportIndex.tsx b/evaluation/coding/swebench_pro/web/src/components/ReportIndex.tsx similarity index 100% rename from evaluation/web/src/components/ReportIndex.tsx rename to evaluation/coding/swebench_pro/web/src/components/ReportIndex.tsx diff --git a/evaluation/web/src/components/ReportView.test.tsx b/evaluation/coding/swebench_pro/web/src/components/ReportView.test.tsx similarity index 100% rename from evaluation/web/src/components/ReportView.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/ReportView.test.tsx diff --git a/evaluation/web/src/components/ReportView.tsx b/evaluation/coding/swebench_pro/web/src/components/ReportView.tsx similarity index 100% rename from evaluation/web/src/components/ReportView.tsx rename to evaluation/coding/swebench_pro/web/src/components/ReportView.tsx diff --git a/evaluation/web/src/components/TaskDetail.test.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskDetail.test.tsx similarity index 100% rename from evaluation/web/src/components/TaskDetail.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskDetail.test.tsx diff --git a/evaluation/web/src/components/TaskDetail.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskDetail.tsx similarity index 100% rename from evaluation/web/src/components/TaskDetail.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskDetail.tsx diff --git a/evaluation/web/src/components/TaskList.test.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskList.test.tsx similarity index 100% rename from evaluation/web/src/components/TaskList.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskList.test.tsx diff --git a/evaluation/web/src/components/TaskList.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskList.tsx similarity index 100% rename from evaluation/web/src/components/TaskList.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskList.tsx diff --git a/evaluation/web/src/components/TaskRunDetail.test.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskRunDetail.test.tsx similarity index 100% rename from evaluation/web/src/components/TaskRunDetail.test.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskRunDetail.test.tsx diff --git a/evaluation/web/src/components/TaskRunDetail.tsx b/evaluation/coding/swebench_pro/web/src/components/TaskRunDetail.tsx similarity index 100% rename from evaluation/web/src/components/TaskRunDetail.tsx rename to evaluation/coding/swebench_pro/web/src/components/TaskRunDetail.tsx diff --git a/evaluation/web/src/main.tsx b/evaluation/coding/swebench_pro/web/src/main.tsx similarity index 100% rename from evaluation/web/src/main.tsx rename to evaluation/coding/swebench_pro/web/src/main.tsx diff --git a/evaluation/web/src/styles.css b/evaluation/coding/swebench_pro/web/src/styles.css similarity index 100% rename from evaluation/web/src/styles.css rename to evaluation/coding/swebench_pro/web/src/styles.css diff --git a/evaluation/web/src/styles.test.ts b/evaluation/coding/swebench_pro/web/src/styles.test.ts similarity index 100% rename from evaluation/web/src/styles.test.ts rename to evaluation/coding/swebench_pro/web/src/styles.test.ts diff --git a/evaluation/web/src/test/fixtures.ts b/evaluation/coding/swebench_pro/web/src/test/fixtures.ts similarity index 100% rename from evaluation/web/src/test/fixtures.ts rename to evaluation/coding/swebench_pro/web/src/test/fixtures.ts diff --git a/evaluation/web/src/test/setup.ts b/evaluation/coding/swebench_pro/web/src/test/setup.ts similarity index 100% rename from evaluation/web/src/test/setup.ts rename to evaluation/coding/swebench_pro/web/src/test/setup.ts diff --git a/evaluation/web/src/types.ts b/evaluation/coding/swebench_pro/web/src/types.ts similarity index 98% rename from evaluation/web/src/types.ts rename to evaluation/coding/swebench_pro/web/src/types.ts index ac990ee421..c29feac5a4 100644 --- a/evaluation/web/src/types.ts +++ b/evaluation/coding/swebench_pro/web/src/types.ts @@ -235,6 +235,8 @@ export interface TreatmentEvidence { scope_id: string; scope_key: string | null; server_ready: boolean; + database_kind?: "sqlite" | "oceanbase" | null | undefined; + database_fingerprint?: string | null | undefined; } export interface EvidenceResponse { @@ -478,10 +480,13 @@ export interface TaskAttempt { status: TaskStatus; phase: TaskPhase | null; created_at: string; + eligible_at: string; started_at: string | null; finished_at: string | null; version: number; failure_category: FailureCategory | null; + failure_code: string | null; + retry_disposition: "retry" | "terminal" | null; failure_phase: TaskPhase | null; failure_summary: string | null; result: TaskResult | null; diff --git a/evaluation/web/src/usageFormat.ts b/evaluation/coding/swebench_pro/web/src/usageFormat.ts similarity index 100% rename from evaluation/web/src/usageFormat.ts rename to evaluation/coding/swebench_pro/web/src/usageFormat.ts diff --git a/evaluation/web/tsconfig.json b/evaluation/coding/swebench_pro/web/tsconfig.json similarity index 100% rename from evaluation/web/tsconfig.json rename to evaluation/coding/swebench_pro/web/tsconfig.json diff --git a/evaluation/web/vite.config.ts b/evaluation/coding/swebench_pro/web/vite.config.ts similarity index 100% rename from evaluation/web/vite.config.ts rename to evaluation/coding/swebench_pro/web/vite.config.ts diff --git a/evaluation/coding/work_continuity/README.md b/evaluation/coding/work_continuity/README.md new file mode 100644 index 0000000000..095f3f1436 --- /dev/null +++ b/evaluation/coding/work_continuity/README.md @@ -0,0 +1,27 @@ +# Work-continuity evaluation + +Compare full transcripts, compacted transcripts, informal summaries, and Rollover +Handoff as ways to resume coding and documentation tasks. The suite validates pinned +task inputs, assembles continuation contexts, scores recorded attempts, and compares +compatible runs. + +This deterministic harness does not call a model or database. Its checked-in attempts +are synthetic fixtures for harness validation, not real-host benchmark results. + +Run from the repository root: + +```bash +uv sync --project evaluation --frozen +uv run --project evaluation work-continuity validate \ + --task-lock evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json \ + --attempts evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json +uv run --project evaluation work-continuity --help +``` + +See the [benchmark guide](docs/work-continuity-benchmark.md) for the task set, +recording protocol, metrics, report commands, and comparison requirements. + +```bash +uv run --project evaluation pytest -c evaluation/pyproject.toml \ + evaluation/coding/work_continuity/tests -q +``` diff --git a/evaluation/docs/work-continuity-benchmark.md b/evaluation/coding/work_continuity/docs/work-continuity-benchmark.md similarity index 94% rename from evaluation/docs/work-continuity-benchmark.md rename to evaluation/coding/work_continuity/docs/work-continuity-benchmark.md index a9ca7aa7be..da14ae3bd7 100644 --- a/evaluation/docs/work-continuity-benchmark.md +++ b/evaluation/coding/work_continuity/docs/work-continuity-benchmark.md @@ -57,13 +57,13 @@ a task entry that is a bare count rather than a per-method one — is refused ra equal. The gate is reachable from the command line, not only from the library: -`powercontext-eval work-continuity compare --baseline --treatment ` applies exactly +`work-continuity compare --baseline --treatment ` applies exactly these checks to two published run directories and exits non-zero with the disagreements it found. ## Task set and ground truth -`evaluation/locks/work-continuity-v1.tasks.json` pins six tasks (four coding, two +`evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json` pins six tasks (four coding, two documentation; 73 turns total) under the schema `powercontext.work-continuity-task-lock.v1`. Each task declares: @@ -108,7 +108,7 @@ delivered: scoring the same attempts under `--max-bytes 1` is refused, because t the one-byte context does not match the digest the recording was taken under. A host that reports two model or host-revision configurations is rejected before scoring. -`powercontext-eval work-continuity validate` applies the same bindings, using the ceiling the +`work-continuity validate` applies the same bindings, using the ceiling the recording itself declares, so the documented preflight refuses what `run` refuses: an artifact that pins another task set, another task lock, or a context digest this project cannot assemble fails validation instead of passing a check that only looked at the file's shape. @@ -122,7 +122,7 @@ scored as an evidence conflict rather than as a recovery. Relying on a fact the unavailable is different in kind — the fact's absence is disclosed, not contradicted — so it is counted as an unverifiable claim and ranks below a clean recovery instead of vetoing success. -`evaluation/locks/work-continuity-v1.attempts-fixture.json` is a synthetic fixture +`evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json` is a synthetic fixture (6 tasks × 4 methods × 2 hosts = 48 attempts) that exercises every failure branch. It is labeled synthetic in the artifact itself, and it carries the protocol block and per-attempt digests described above. @@ -186,14 +186,14 @@ below a baseline on a comparison that never happened. ```bash # Validate the pinned inputs without writing anything. -uv run --project evaluation powercontext-eval work-continuity validate \ - --task-lock evaluation/locks/work-continuity-v1.tasks.json \ - --attempts evaluation/locks/work-continuity-v1.attempts-fixture.json +uv run --project evaluation work-continuity validate \ + --task-lock evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json \ + --attempts evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json # Assemble every method, score the recorded attempts, and write one run directory. -uv run --project evaluation powercontext-eval work-continuity run \ - --task-lock evaluation/locks/work-continuity-v1.tasks.json \ - --attempts evaluation/locks/work-continuity-v1.attempts-fixture.json \ +uv run --project evaluation work-continuity run \ + --task-lock evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json \ + --attempts evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json \ --output-dir /tmp/work-continuity-run \ --run-id fixture-smoke ``` diff --git a/evaluation/locks/work-continuity-v1.attempts-fixture.json b/evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json similarity index 100% rename from evaluation/locks/work-continuity-v1.attempts-fixture.json rename to evaluation/coding/work_continuity/locks/work-continuity-v1.attempts-fixture.json diff --git a/evaluation/locks/work-continuity-v1.tasks.json b/evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json similarity index 100% rename from evaluation/locks/work-continuity-v1.tasks.json rename to evaluation/coding/work_continuity/locks/work-continuity-v1.tasks.json diff --git a/evaluation/coding/work_continuity/ruff.toml b/evaluation/coding/work_continuity/ruff.toml new file mode 100644 index 0000000000..2c6e969e7e --- /dev/null +++ b/evaluation/coding/work_continuity/ruff.toml @@ -0,0 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +target-version = "py311" +line-length = 120 diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/__init__.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/__init__.py similarity index 100% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/__init__.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/__init__.py diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/analysis.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/analysis.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/analysis.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/analysis.py index ea460ece00..dbbb8c9694 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/analysis.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/analysis.py @@ -30,16 +30,16 @@ from dataclasses import dataclass from types import MappingProxyType -from powercontext_eval.benchmarks.work_continuity.arms import TREATMENT_ARM_ID -from powercontext_eval.benchmarks.work_continuity.assembly import ContinuationContext -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask -from powercontext_eval.benchmarks.work_continuity.quality import ( +from powercontext_eval_work_continuity.arms import TREATMENT_ARM_ID +from powercontext_eval_work_continuity.assembly import ContinuationContext +from powercontext_eval_work_continuity.catalog import ContinuationTask +from powercontext_eval_work_continuity.quality import ( EVIDENCE_REQUIREMENT, NEXT_ACTION_REQUIREMENT, OMISSIONS_REQUIREMENT, STATE_REQUIREMENT, ) -from powercontext_eval.benchmarks.work_continuity.rubric import AttemptScore +from powercontext_eval_work_continuity.rubric import AttemptScore BUDGET_TRUNCATION = "budget_truncation" CONTEXT_ABSENT = "context_absent" diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/arms.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/arms.py similarity index 99% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/arms.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/arms.py index d27d9644d0..d5d6c51b0e 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/arms.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/arms.py @@ -28,7 +28,7 @@ from dataclasses import dataclass from typing import Any -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_work_continuity.errors import PowerContextEvalError # The bounded continuation budget is part of the protocol, not of any arm: every # arm receives the same ceiling so injected bytes measure the method rather than diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/assembly.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/assembly.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/assembly.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/assembly.py index 4a1e2a1b8d..1fb7c3b0ed 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/assembly.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/assembly.py @@ -35,9 +35,9 @@ from collections.abc import Iterable, Sequence from dataclasses import dataclass -from powercontext_eval.benchmarks.work_continuity.arms import ROLLOVER_HANDOFF, ContinuationArm -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask, StateFact -from powercontext_eval.benchmarks.work_continuity.quality import ( +from powercontext_eval_work_continuity.arms import ROLLOVER_HANDOFF, ContinuationArm +from powercontext_eval_work_continuity.catalog import ContinuationTask, StateFact +from powercontext_eval_work_continuity.quality import ( HandoffContent, HandoffStatement, QualityReport, diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/attempts.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/attempts.py similarity index 98% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/attempts.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/attempts.py index 3abc73945a..c97efa2203 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/attempts.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/attempts.py @@ -41,9 +41,9 @@ from pathlib import Path from typing import cast -from powercontext_eval.benchmarks.work_continuity.arms import ContinuationArmError, get_continuation_arm -from powercontext_eval.benchmarks.work_continuity.catalog import TaskCatalog, WorkContinuityCatalogError -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_work_continuity.arms import ContinuationArmError, get_continuation_arm +from powercontext_eval_work_continuity.catalog import TaskCatalog, WorkContinuityCatalogError +from powercontext_eval_work_continuity.errors import PowerContextEvalError ATTEMPTS_SCHEMA = "powercontext.work-continuity-attempts.v1" diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/catalog.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/catalog.py similarity index 99% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/catalog.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/catalog.py index f55a00f291..c8c0439f8f 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/catalog.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/catalog.py @@ -39,7 +39,7 @@ from types import MappingProxyType from typing import Literal, TypeAlias, cast -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_work_continuity.errors import PowerContextEvalError TASK_LOCK_SCHEMA = "powercontext.work-continuity-task-lock.v1" diff --git a/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/cli.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/cli.py new file mode 100644 index 0000000000..9ee431ee53 --- /dev/null +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/cli.py @@ -0,0 +1,254 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Independent work-continuity command-line application.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated, cast + +import typer + +from powercontext_eval_work_continuity.arms import ( + DEFAULT_ASSEMBLY_MAX_BYTES, + ContinuationArmError, + declared_run_arm_ids, + ensure_comparable_work_continuity_runs, + supported_continuation_arm_ids, +) +from powercontext_eval_work_continuity.attempts import ( + AttemptInputError, + load_attempts, +) +from powercontext_eval_work_continuity.catalog import ( + TaskCatalog, + WorkContinuityCatalogError, + WorkContinuityInputError, +) +from powercontext_eval_work_continuity.report import ( + ReportError as WorkContinuityReportError, +) +from powercontext_eval_work_continuity.report import ( + build_report as build_work_continuity_report, +) +from powercontext_eval_work_continuity.runner import ( + WorkContinuityRunError, + execution_configuration, + require_recording_bindings, + run_summary, + run_work_continuity, + write_run_artifacts, +) + +app = typer.Typer( + no_args_is_help=True, + help="Continuation-method evaluation for Rollover Handoff and its alternatives.", +) + + +@app.command("validate") +def work_continuity_validate( + task_lock: Annotated[Path, typer.Option("--task-lock")], + attempts: Annotated[Path | None, typer.Option("--attempts")] = None, +) -> None: + """Validate the pinned task lock and any recorded attempts without writing anything.""" + + try: + catalog = TaskCatalog.load(task_lock) + recorded = load_attempts(attempts, catalog=catalog) if attempts is not None else None + if recorded is not None: + # The documented preflight has to reject what `run` would reject: + # a recording bound to another task lock or another context is not + # scoreable, and finding that out here is the point of the command. + require_recording_bindings(catalog=catalog, attempts=recorded) + except (WorkContinuityInputError, AttemptInputError) as error: + raise typer.BadParameter(str(error)) from None + except (WorkContinuityCatalogError, WorkContinuityRunError) as error: + typer.echo(f"Work-continuity validation failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "work-continuity-input-validation", + "task_set_id": catalog.task_set_id, + "task_count": len(catalog.tasks), + "task_ids": list(catalog.task_ids), + "task_lock_sha256": catalog.content_sha256, + "supported_arms": list(supported_continuation_arm_ids()), + "attempt_count": None if recorded is None else len(recorded.attempts), + "hosts": None if recorded is None else list(recorded.hosts), + "recording_protocol": ( + None + if recorded is None + else { + "task_set_id": recorded.protocol.task_set_id, + "task_lock_sha256": recorded.protocol.task_lock_sha256, + "assembly_max_bytes": recorded.protocol.assembly_max_bytes, + } + ), + "execution_configuration": None if recorded is None else execution_configuration(recorded), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +def _run_manifest(path: Path) -> dict[str, object]: + """Read one run manifest from a run directory or from a manifest file. + + A published run is a directory, so accepting the directory is what makes the + command usable on the artifacts a run actually writes. + """ + + candidate = path / "run-manifest.json" if path.is_dir() else path + if not candidate.is_file(): + raise WorkContinuityRunError(f"no run manifest at {candidate}") + try: + payload = json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise WorkContinuityRunError(f"cannot read run manifest {candidate}: {error}") from None + if not isinstance(payload, dict): + raise WorkContinuityRunError(f"run manifest {candidate} is not a JSON object") + return cast("dict[str, object]", payload) + + +@app.command("compare") +def work_continuity_compare( + baseline: Annotated[Path, typer.Option("--baseline")], + treatment: Annotated[Path, typer.Option("--treatment")], +) -> None: + """Decide whether two recorded runs may be compared, and refuse when they may not.""" + + try: + first = _run_manifest(baseline) + second = _run_manifest(treatment) + # The gate is the whole point of the command: it is the only supported way + # to have the documented comparison rules applied to two published runs. + ensure_comparable_work_continuity_runs(first, second) + except ContinuationArmError as error: + typer.echo(f"Work-continuity comparison refused: {error}", err=True) + raise typer.Exit(code=1) from None + except WorkContinuityRunError as error: + typer.echo(f"Work-continuity comparison failed: {error}", err=True) + raise typer.Exit(code=1) from None + first_arms = declared_run_arm_ids(first) or () + second_arms = declared_run_arm_ids(second) or () + typer.echo( + json.dumps( + { + "classification": "work-continuity-comparability", + "comparable": True, + "baseline_run_id": first.get("run_id"), + "treatment_run_id": second.get("run_id"), + "task_set_id": first.get("task_set_id"), + "baseline_arms": list(first_arms), + "treatment_arms": list(second_arms), + "shared_arms": sorted(set(first_arms) & set(second_arms)), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +def _method_rows(summary: dict[str, object]) -> list[dict[str, object]]: + """Return one per-method echo row, rejecting a summary that lost its shape.""" + + arms = summary.get("arms") + if not isinstance(arms, list): + raise WorkContinuityRunError("work-continuity run summary is missing its per-method rows") + rows: list[dict[str, object]] = [] + for arm in arms: + if not isinstance(arm, dict): + raise WorkContinuityRunError("work-continuity run summary holds a malformed method row") + method = cast("dict[str, object]", arm) + injected = method.get("injected_bytes") + outcome = method.get("outcome") + if not isinstance(injected, dict) or not isinstance(outcome, dict): + raise WorkContinuityRunError("work-continuity run summary holds a malformed method row") + rows.append( + { + "arm_id": method["arm_id"], + "injected_bytes_total": cast("dict[str, object]", injected)["total"], + "task_success": cast("dict[str, object]", outcome)["task_success"], + } + ) + return rows + + +@app.command("run") +def work_continuity_run( + task_lock: Annotated[Path, typer.Option("--task-lock")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + run_id: Annotated[str, typer.Option("--run-id")], + max_bytes: Annotated[int, typer.Option("--max-bytes", min=1)] = DEFAULT_ASSEMBLY_MAX_BYTES, + arm: Annotated[list[str] | None, typer.Option("--arm")] = None, + task_id: Annotated[list[str] | None, typer.Option("--task-id")] = None, + attempts: Annotated[Path | None, typer.Option("--attempts")] = None, + powercontext_revision: Annotated[str | None, typer.Option("--powercontext-revision")] = None, + integration_revision: Annotated[str | None, typer.Option("--integration-revision")] = None, +) -> None: + """Assemble every selected method, score any recorded attempts, and write one run directory.""" + + try: + result = run_work_continuity( + task_lock=task_lock, + run_id=run_id, + max_bytes=max_bytes, + arm_ids=tuple(arm) if arm else None, + task_ids=tuple(task_id) if task_id else None, + attempts_path=attempts, + powercontext_revision=powercontext_revision, + integration_revision=integration_revision, + ) + artifacts = write_run_artifacts(result, output_dir) + report = build_work_continuity_report(result, output_dir=artifacts.output_dir) + methods = _method_rows(run_summary(result)) + except ContinuationArmError as error: + raise typer.BadParameter(str(error)) from None + except (WorkContinuityInputError, AttemptInputError) as error: + raise typer.BadParameter(str(error)) from None + except (WorkContinuityRunError, WorkContinuityReportError, WorkContinuityCatalogError) as error: + typer.echo(f"Work-continuity run failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": result.classification, + "run_id": result.run_id, + "manifest": str(artifacts.manifest_path), + "assembly": str(artifacts.assembly_path), + "scores": str(artifacts.scores_path), + "summary": str(artifacts.summary_path), + "report": str(report.report_path), + "markdown": str(report.markdown_path), + "methods": methods, + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +def main() -> None: + """Run the work-continuity command-line application.""" + + app() + + +if __name__ == "__main__": + main() diff --git a/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/errors.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/errors.py new file mode 100644 index 0000000000..48a7bdaa3e --- /dev/null +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/errors.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Work-continuity evaluation errors.""" + + +class PowerContextEvalError(Exception): + """Base error for work-continuity evaluation failures.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/quality.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/quality.py similarity index 100% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/quality.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/quality.py diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/report.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/report.py similarity index 99% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/report.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/report.py index 2b25696046..ebb7d83667 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/report.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/report.py @@ -31,13 +31,13 @@ from pathlib import Path from typing import cast -from powercontext_eval.benchmarks.work_continuity.analysis import ( +from powercontext_eval_work_continuity.analysis import ( NO_RECORDING, ArmOutcome, WorkContinuityAnalysis, ) -from powercontext_eval.benchmarks.work_continuity.arms import TREATMENT_ARM_ID -from powercontext_eval.benchmarks.work_continuity.runner import WorkContinuityRun, run_summary +from powercontext_eval_work_continuity.arms import TREATMENT_ARM_ID +from powercontext_eval_work_continuity.runner import WorkContinuityRun, run_summary REPORT_SCHEMA = "powercontext.work-continuity-report.v1" diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/rubric.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/rubric.py similarity index 96% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/rubric.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/rubric.py index 6ba126abdd..d59f1eb19b 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/rubric.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/rubric.py @@ -30,9 +30,9 @@ from dataclasses import dataclass -from powercontext_eval.benchmarks.work_continuity.assembly import ContinuationContext -from powercontext_eval.benchmarks.work_continuity.attempts import RecordedAttempt -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask +from powercontext_eval_work_continuity.assembly import ContinuationContext +from powercontext_eval_work_continuity.attempts import RecordedAttempt +from powercontext_eval_work_continuity.catalog import ContinuationTask # A step that number was never reached counts as the worst possible recovery # time, so an unrecovered attempt never sorts above a slow recovery. diff --git a/evaluation/src/powercontext_eval/benchmarks/work_continuity/runner.py b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/runner.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/work_continuity/runner.py rename to evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/runner.py index 90575a45bc..31404ebda3 100644 --- a/evaluation/src/powercontext_eval/benchmarks/work_continuity/runner.py +++ b/evaluation/coding/work_continuity/src/powercontext_eval_work_continuity/runner.py @@ -32,14 +32,14 @@ from dataclasses import dataclass from pathlib import Path -from powercontext_eval.benchmarks.work_continuity.analysis import ( +from powercontext_eval_work_continuity.analysis import ( ArmOutcome, TaskAnalysis, WorkContinuityAnalysis, analyse_task_outcomes, analyse_work_continuity, ) -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( BASELINE_ARM_IDS, CONTINUATION_ARMS, DEFAULT_ASSEMBLY_MAX_BYTES, @@ -48,12 +48,12 @@ arm_manifest_record, resolve_continuation_arms, ) -from powercontext_eval.benchmarks.work_continuity.assembly import ContinuationContext, assemble_context -from powercontext_eval.benchmarks.work_continuity.attempts import AttemptSet, load_attempts -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask, TaskCatalog -from powercontext_eval.benchmarks.work_continuity.quality import QualityReport -from powercontext_eval.benchmarks.work_continuity.rubric import AttemptScore, score_attempt -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_work_continuity.assembly import ContinuationContext, assemble_context +from powercontext_eval_work_continuity.attempts import AttemptSet, load_attempts +from powercontext_eval_work_continuity.catalog import ContinuationTask, TaskCatalog +from powercontext_eval_work_continuity.errors import PowerContextEvalError +from powercontext_eval_work_continuity.quality import QualityReport +from powercontext_eval_work_continuity.rubric import AttemptScore, score_attempt ASSEMBLY_ONLY_CLASSIFICATION = "assembly-only-no-recorded-attempts" RECORDED_CLASSIFICATION = "recorded-attempts-not-a-complete-benchmark-result" diff --git a/evaluation/src/powercontext_eval/benchmarks/__init__.py b/evaluation/coding/work_continuity/tests/unit/__init__.py similarity index 93% rename from evaluation/src/powercontext_eval/benchmarks/__init__.py rename to evaluation/coding/work_continuity/tests/unit/__init__.py index 7c3f8292ba..30d266a9f0 100644 --- a/evaluation/src/powercontext_eval/benchmarks/__init__.py +++ b/evaluation/coding/work_continuity/tests/unit/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Benchmark adapters.""" +"""Work-continuity behavior tests.""" diff --git a/evaluation/tests/unit/test_work_continuity_analysis.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_analysis.py similarity index 96% rename from evaluation/tests/unit/test_work_continuity_analysis.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_analysis.py index 9448e5257f..60907cebcc 100644 --- a/evaluation/tests/unit/test_work_continuity_analysis.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_analysis.py @@ -26,9 +26,8 @@ from typing import Any import pytest -from work_continuity_fixtures import UNBOUND_CONTEXT_SHA256, analysis_lock, budget_through -from powercontext_eval.benchmarks.work_continuity.analysis import ( +from powercontext_eval_work_continuity.analysis import ( BUDGET_TRUNCATION, CONTEXT_ABSENT, FAILURE_REQUIREMENTS, @@ -43,7 +42,7 @@ classify_failure, facts_lost_to_the_budget, ) -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, @@ -51,16 +50,18 @@ TREATMENT_ARM_ID, ContinuationArm, ) -from powercontext_eval.benchmarks.work_continuity.assembly import ContinuationContext, assemble_context -from powercontext_eval.benchmarks.work_continuity.attempts import RecordedAttempt, RecordedStep -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask, TaskCatalog -from powercontext_eval.benchmarks.work_continuity.quality import ( +from powercontext_eval_work_continuity.assembly import ContinuationContext, assemble_context +from powercontext_eval_work_continuity.attempts import RecordedAttempt, RecordedStep +from powercontext_eval_work_continuity.catalog import ContinuationTask, TaskCatalog +from powercontext_eval_work_continuity.quality import ( EVIDENCE_REQUIREMENT, NEXT_ACTION_REQUIREMENT, OMISSIONS_REQUIREMENT, STATE_REQUIREMENT, ) -from powercontext_eval.benchmarks.work_continuity.rubric import AttemptScore, score_attempt +from powercontext_eval_work_continuity.rubric import AttemptScore, score_attempt + +from .work_continuity_fixtures import UNBOUND_CONTEXT_SHA256, analysis_lock, budget_through GENEROUS_BUDGET = 16_000 HOST_A = "host-a" diff --git a/evaluation/tests/unit/test_work_continuity_arms.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_arms.py similarity index 99% rename from evaluation/tests/unit/test_work_continuity_arms.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_arms.py index 3a71d971a4..c6b9c65e47 100644 --- a/evaluation/tests/unit/test_work_continuity_arms.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_arms.py @@ -26,7 +26,7 @@ import pytest -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( BASELINE_ARM_IDS, CONTINUATION_ARMS, FULL_TRANSCRIPT, diff --git a/evaluation/tests/unit/test_work_continuity_assembly.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_assembly.py similarity index 96% rename from evaluation/tests/unit/test_work_continuity_assembly.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_assembly.py index fb33ceef1c..d1df6d4938 100644 --- a/evaluation/tests/unit/test_work_continuity_assembly.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_assembly.py @@ -25,16 +25,15 @@ from pathlib import Path import pytest -from work_continuity_fixtures import analysis_lock, budget_through -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, ROLLOVER_HANDOFF, ContinuationArm, ) -from powercontext_eval.benchmarks.work_continuity.assembly import ( +from powercontext_eval_work_continuity.assembly import ( NEXT_ACTION_LABEL, OMISSIONS_LABEL, STATE_LABEL, @@ -43,8 +42,10 @@ _fit, assemble_context, ) -from powercontext_eval.benchmarks.work_continuity.catalog import ContinuationTask, TaskCatalog -from powercontext_eval.benchmarks.work_continuity.quality import NEXT_ACTION_REQUIREMENT +from powercontext_eval_work_continuity.catalog import ContinuationTask, TaskCatalog +from powercontext_eval_work_continuity.quality import NEXT_ACTION_REQUIREMENT + +from .work_continuity_fixtures import analysis_lock, budget_through GENEROUS_BUDGET = 16_000 diff --git a/evaluation/tests/unit/test_work_continuity_catalog.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_catalog.py similarity index 97% rename from evaluation/tests/unit/test_work_continuity_catalog.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_catalog.py index 048932bd91..17c31c9a79 100644 --- a/evaluation/tests/unit/test_work_continuity_catalog.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_catalog.py @@ -16,7 +16,17 @@ from pathlib import Path import pytest -from work_continuity_fixtures import ( + +from powercontext_eval_work_continuity.arms import get_continuation_arm +from powercontext_eval_work_continuity.assembly import assemble_context +from powercontext_eval_work_continuity.attempts import AttemptInputError, load_attempts +from powercontext_eval_work_continuity.catalog import ( + TaskCatalog, + WorkContinuityEnvironmentError, + WorkContinuityInputError, +) + +from .work_continuity_fixtures import ( ATTEMPTS_SCHEMA, attempts_protocol, coding_task, @@ -28,15 +38,6 @@ write_lock, ) -from powercontext_eval.benchmarks.work_continuity.arms import get_continuation_arm -from powercontext_eval.benchmarks.work_continuity.assembly import assemble_context -from powercontext_eval.benchmarks.work_continuity.attempts import AttemptInputError, load_attempts -from powercontext_eval.benchmarks.work_continuity.catalog import ( - TaskCatalog, - WorkContinuityEnvironmentError, - WorkContinuityInputError, -) - # The shipped lock is the benchmark's ground truth, so its digest is pinned here as # a golden anchor: an edit to the authored tasks has to be an intentional edit to # this constant too, and every number the documentation quotes moves with it. @@ -390,7 +391,7 @@ def test_shipped_fixture_lock_and_attempts_validate() -> None: def test_the_shipped_task_lock_is_pinned_by_its_digest() -> None: """The authored ground truth is anchored, not only read. - Every published number in `evaluation/docs/work-continuity-benchmark.md` is a + Every published number in `evaluation/coding/work_continuity/docs/work-continuity-benchmark.md` is a property of this file, so an edit to it must be a deliberate edit to the documented numbers as well. """ diff --git a/evaluation/tests/unit/test_work_continuity_cli.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_cli.py similarity index 90% rename from evaluation/tests/unit/test_work_continuity_cli.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_cli.py index 7f42cd38f5..0a1b7a45c1 100644 --- a/evaluation/tests/unit/test_work_continuity_cli.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_cli.py @@ -27,16 +27,17 @@ import pytest from typer.testing import CliRunner -from work_continuity_fixtures import analysis_lock, context_digest, write_attempts -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, ROLLOVER_HANDOFF, supported_continuation_arm_ids, ) -from powercontext_eval.cli import app +from powercontext_eval_work_continuity.cli import app + +from .work_continuity_fixtures import analysis_lock, context_digest, write_attempts HOST = "fixture-host" @@ -87,7 +88,7 @@ def payload(result: Any) -> dict[str, Any]: def test_validate_describes_the_pinned_inputs_without_writing_anything(tmp_path: Path) -> None: lock = analysis_lock(tmp_path) - body = payload(CliRunner().invoke(app, ["work-continuity", "validate", "--task-lock", str(lock)])) + body = payload(CliRunner().invoke(app, ["validate", "--task-lock", str(lock)])) assert body["classification"] == "work-continuity-input-validation" assert body["task_set_id"] == "test-set" @@ -112,7 +113,7 @@ def test_validate_also_checks_recorded_attempts(tmp_path: Path) -> None: body = payload( CliRunner().invoke( app, - ["work-continuity", "validate", "--task-lock", str(lock), "--attempts", str(attempts)], + ["validate", "--task-lock", str(lock), "--attempts", str(attempts)], ) ) @@ -146,7 +147,7 @@ def test_validate_also_checks_recorded_attempts(tmp_path: Path) -> None: def test_validate_reports_an_unusable_lock_without_a_traceback(tmp_path: Path) -> None: - result = CliRunner().invoke(app, ["work-continuity", "validate", "--task-lock", str(tmp_path / "absent.json")]) + result = CliRunner().invoke(app, ["validate", "--task-lock", str(tmp_path / "absent.json")]) assert result.exit_code == 1 assert "Work-continuity validation failed" in result.output @@ -159,9 +160,7 @@ def test_validate_rejects_an_attempts_artifact_that_cannot_be_scored(tmp_path: P entry["arm_id"] = "nope" attempts = write_attempts(tmp_path, [entry], lock=lock) - result = CliRunner().invoke( - app, ["work-continuity", "validate", "--task-lock", str(lock), "--attempts", str(attempts)] - ) + result = CliRunner().invoke(app, ["validate", "--task-lock", str(lock), "--attempts", str(attempts)]) assert result.exit_code == 2 assert "unknown continuation arm" in result.output @@ -187,9 +186,7 @@ def test_validate_rejects_a_recording_bound_to_another_protocol(tmp_path: Path, document["protocol"][field] = value attempts.write_text(json.dumps(document, ensure_ascii=False), encoding="utf-8") - result = CliRunner().invoke( - app, ["work-continuity", "validate", "--task-lock", str(lock), "--attempts", str(attempts)] - ) + result = CliRunner().invoke(app, ["validate", "--task-lock", str(lock), "--attempts", str(attempts)]) assert result.exit_code == 1 assert "Work-continuity validation failed" in result.output @@ -202,9 +199,7 @@ def test_validate_rejects_a_context_digest_this_project_cannot_assemble(tmp_path document["attempts"][0]["context_sha256"] = "c" * 64 attempts.write_text(json.dumps(document, ensure_ascii=False), encoding="utf-8") - result = CliRunner().invoke( - app, ["work-continuity", "validate", "--task-lock", str(lock), "--attempts", str(attempts)] - ) + result = CliRunner().invoke(app, ["validate", "--task-lock", str(lock), "--attempts", str(attempts)]) assert result.exit_code == 1 assert "Work-continuity validation failed" in result.output @@ -220,7 +215,6 @@ def test_run_assembles_scores_and_writes_one_run_directory(tmp_path: Path) -> No CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -275,7 +269,6 @@ def test_run_assembles_without_attempts_and_says_so(tmp_path: Path) -> None: CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -299,7 +292,6 @@ def test_run_narrows_to_one_arm_and_one_task(tmp_path: Path) -> None: CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -332,7 +324,7 @@ def test_run_refuses_to_reuse_an_output_directory(tmp_path: Path) -> None: result = CliRunner().invoke( app, - ["work-continuity", "run", "--task-lock", str(lock), "--output-dir", str(output), "--run-id", "again"], + ["run", "--task-lock", str(lock), "--output-dir", str(output), "--run-id", "again"], ) assert result.exit_code == 1 @@ -350,7 +342,6 @@ def test_run_refuses_to_rescore_recordings_under_a_smaller_ceiling(tmp_path: Pat result = CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -376,7 +367,6 @@ def test_run_rejects_an_unknown_arm_as_a_usage_error(tmp_path: Path) -> None: result = CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -406,7 +396,6 @@ def record_run(tmp_path: Path, lock: Path, attempts: Path, name: str, *extra: st CliRunner().invoke( app, [ - "work-continuity", "run", "--task-lock", str(lock), @@ -438,7 +427,7 @@ def test_compare_accepts_two_runs_that_differ_only_by_their_identity(tmp_path: P body = payload( CliRunner().invoke( app, - ["work-continuity", "compare", "--baseline", str(first), "--treatment", str(second)], + ["compare", "--baseline", str(first), "--treatment", str(second)], ) ) @@ -464,7 +453,7 @@ def test_compare_refuses_a_run_recorded_under_another_ceiling(tmp_path: Path) -> result = CliRunner().invoke( app, - ["work-continuity", "compare", "--baseline", str(first), "--treatment", str(second)], + ["compare", "--baseline", str(first), "--treatment", str(second)], ) assert result.exit_code == 1 @@ -479,7 +468,7 @@ def test_compare_reports_a_manifest_it_cannot_read(tmp_path: Path) -> None: result = CliRunner().invoke( app, - ["work-continuity", "compare", "--baseline", str(first), "--treatment", str(tmp_path / "absent")], + ["compare", "--baseline", str(first), "--treatment", str(tmp_path / "absent")], ) assert result.exit_code == 1 diff --git a/evaluation/tests/unit/test_work_continuity_quality.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_quality.py similarity index 99% rename from evaluation/tests/unit/test_work_continuity_quality.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_quality.py index feb3014581..00a10ac847 100644 --- a/evaluation/tests/unit/test_work_continuity_quality.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_quality.py @@ -23,7 +23,7 @@ import pytest -from powercontext_eval.benchmarks.work_continuity.quality import ( +from powercontext_eval_work_continuity.quality import ( CONTINUE_PREVIOUS_REQUIREMENT, CONVERSATION_ABOVE_REQUIREMENT, DISPOSITION_REQUIREMENT, diff --git a/evaluation/tests/unit/test_work_continuity_report.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_report.py similarity index 97% rename from evaluation/tests/unit/test_work_continuity_report.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_report.py index 9e418bd496..7b8e6e4d54 100644 --- a/evaluation/tests/unit/test_work_continuity_report.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_report.py @@ -27,24 +27,25 @@ from typing import Any import pytest -from work_continuity_fixtures import analysis_lock, context_digest, write_attempts -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, ROLLOVER_HANDOFF, TREATMENT_ARM_ID, ) -from powercontext_eval.benchmarks.work_continuity.quality import NEXT_ACTION_REQUIREMENT -from powercontext_eval.benchmarks.work_continuity.report import ( +from powercontext_eval_work_continuity.quality import NEXT_ACTION_REQUIREMENT +from powercontext_eval_work_continuity.report import ( NOT_A_BENCHMARK_BANNER, REPORT_SCHEMA, build_report, render_markdown, report_payload, ) -from powercontext_eval.benchmarks.work_continuity.runner import WorkContinuityRun, run_work_continuity +from powercontext_eval_work_continuity.runner import WorkContinuityRun, run_work_continuity + +from .work_continuity_fixtures import analysis_lock, context_digest, write_attempts HOST = "fixture-host" diff --git a/evaluation/tests/unit/test_work_continuity_rubric.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_rubric.py similarity index 96% rename from evaluation/tests/unit/test_work_continuity_rubric.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_rubric.py index 59f4c7a64a..d571479c3c 100644 --- a/evaluation/tests/unit/test_work_continuity_rubric.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_rubric.py @@ -21,25 +21,26 @@ from typing import Any import pytest -from work_continuity_fixtures import UNBOUND_CONTEXT_SHA256, analysis_lock -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, ROLLOVER_HANDOFF, ContinuationArm, ) -from powercontext_eval.benchmarks.work_continuity.assembly import ContinuationContext, assemble_context -from powercontext_eval.benchmarks.work_continuity.attempts import RecordedAttempt, RecordedStep -from powercontext_eval.benchmarks.work_continuity.catalog import TaskCatalog -from powercontext_eval.benchmarks.work_continuity.rubric import ( +from powercontext_eval_work_continuity.assembly import ContinuationContext, assemble_context +from powercontext_eval_work_continuity.attempts import RecordedAttempt, RecordedStep +from powercontext_eval_work_continuity.catalog import TaskCatalog +from powercontext_eval_work_continuity.rubric import ( UNRECOVERED_STEP_SENTINEL, AttemptScore, score_attempt, score_attempts, ) +from .work_continuity_fixtures import UNBOUND_CONTEXT_SHA256, analysis_lock + GENEROUS_BUDGET = 16_000 diff --git a/evaluation/tests/unit/test_work_continuity_runner.py b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_runner.py similarity index 98% rename from evaluation/tests/unit/test_work_continuity_runner.py rename to evaluation/coding/work_continuity/tests/unit/test_work_continuity_runner.py index 6d15d1d3d9..8ca22eb1ac 100644 --- a/evaluation/tests/unit/test_work_continuity_runner.py +++ b/evaluation/coding/work_continuity/tests/unit/test_work_continuity_runner.py @@ -27,17 +27,9 @@ from typing import Any import pytest -from work_continuity_fixtures import ( - analysis_lock, - audit_task, - context_digest, - documentation_task, - write_attempts, - write_lock, -) -from powercontext_eval.benchmarks.work_continuity.analysis import FAILURE_CLASSES, NO_RECORDING -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.analysis import FAILURE_CLASSES, NO_RECORDING +from powercontext_eval_work_continuity.arms import ( COMPACTED_TRANSCRIPT, FULL_TRANSCRIPT, INFORMAL_SUMMARY, @@ -46,8 +38,8 @@ ensure_comparable_work_continuity_runs, supported_continuation_arm_ids, ) -from powercontext_eval.benchmarks.work_continuity.catalog import TaskCatalog -from powercontext_eval.benchmarks.work_continuity.runner import ( +from powercontext_eval_work_continuity.catalog import TaskCatalog +from powercontext_eval_work_continuity.runner import ( ASSEMBLY_ONLY_CLASSIFICATION, RECORDED_CLASSIFICATION, RUN_MANIFEST_SCHEMA, @@ -58,6 +50,15 @@ write_run_artifacts, ) +from .work_continuity_fixtures import ( + analysis_lock, + audit_task, + context_digest, + documentation_task, + write_attempts, + write_lock, +) + HOST = "fixture-host" ARMS = (FULL_TRANSCRIPT, COMPACTED_TRANSCRIPT, INFORMAL_SUMMARY, ROLLOVER_HANDOFF) diff --git a/evaluation/tests/unit/work_continuity_fixtures.py b/evaluation/coding/work_continuity/tests/unit/work_continuity_fixtures.py similarity index 97% rename from evaluation/tests/unit/work_continuity_fixtures.py rename to evaluation/coding/work_continuity/tests/unit/work_continuity_fixtures.py index c70be150cb..d23b888ec5 100644 --- a/evaluation/tests/unit/work_continuity_fixtures.py +++ b/evaluation/coding/work_continuity/tests/unit/work_continuity_fixtures.py @@ -25,13 +25,13 @@ import json from pathlib import Path -from powercontext_eval.benchmarks.work_continuity.arms import ( +from powercontext_eval_work_continuity.arms import ( ContinuationArm, ContinuationArmError, get_continuation_arm, ) -from powercontext_eval.benchmarks.work_continuity.assembly import assemble_context, build_items -from powercontext_eval.benchmarks.work_continuity.catalog import ( +from powercontext_eval_work_continuity.assembly import assemble_context, build_items +from powercontext_eval_work_continuity.catalog import ( TASK_LOCK_SCHEMA, ContinuationTask, TaskCatalog, diff --git a/evaluation/memory/README.md b/evaluation/memory/README.md new file mode 100644 index 0000000000..c938ba80b4 --- /dev/null +++ b/evaluation/memory/README.md @@ -0,0 +1,15 @@ +# Memory quality evaluation + +These suites measure conversation memory, retrieval, and answer quality against labeled datasets. +Run their commands from the repository root. LoCoMo suites use the root Python environment; LongMemEval-V2 uses +the shared `evaluation/` environment. + +| Suite | Purpose | +| --- | --- | +| [LoCoMo](locomo/README.md) | End-to-end Source ingestion, Memory retrieval, answer generation, and judging on LoCoMo. | +| [LongMemEval-V2](longmemeval_v2/README.md) | Long-horizon memory retrieval, prepared context, reader answers, and official scoring. | +| [LoCoMo-Plus](locomo_plus/README.md) | Factual and cognitive memory evaluation with bounded smoke and full profiles. | + +Add dataset-specific Memory quality suites here. Put latency, throughput, storage, and capacity measurements in +[performance](../performance/README.md). Keep each suite's execution and grading tools with the suite; see +[evaluation](../README.md) for the shared Python environment. LongMemEval-V2 provides its own `longmemeval-v2` command. diff --git a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/__init__.py b/evaluation/memory/__init__.py similarity index 93% rename from evaluation/src/powercontext_eval/benchmarks/swebench_pro/__init__.py rename to evaluation/memory/__init__.py index 18cc38f1b5..dc0a075cc6 100644 --- a/evaluation/src/powercontext_eval/benchmarks/swebench_pro/__init__.py +++ b/evaluation/memory/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Pinned SWE-bench Pro adapter.""" +"""Memory quality evaluation suites.""" diff --git a/benchmark/locomo/.env.example b/evaluation/memory/locomo/.env.example similarity index 100% rename from benchmark/locomo/.env.example rename to evaluation/memory/locomo/.env.example diff --git a/benchmark/locomo/.gitignore b/evaluation/memory/locomo/.gitignore similarity index 100% rename from benchmark/locomo/.gitignore rename to evaluation/memory/locomo/.gitignore diff --git a/benchmark/locomo/LICENSE b/evaluation/memory/locomo/LICENSE similarity index 100% rename from benchmark/locomo/LICENSE rename to evaluation/memory/locomo/LICENSE diff --git a/benchmark/locomo/README.md b/evaluation/memory/locomo/README.md similarity index 87% rename from benchmark/locomo/README.md rename to evaluation/memory/locomo/README.md index 6e17a84095..c7e1c6e5bd 100644 --- a/benchmark/locomo/README.md +++ b/evaluation/memory/locomo/README.md @@ -15,7 +15,8 @@ timestamped dialogue Source - Dataset: 10 conversations, 272 timestamped sessions, 5,882 dialogue turns, and 1,986 questions. - Scored set: categories 1-4, 1,540 questions. Category 5 is excluded by this benchmark's scored-set contract. -- Scope: one isolated PowerContext scope per two-speaker conversation. The run ID creates a separate namespace. +- Scope: one registered PowerContext scope per two-speaker conversation. The run ID creates a separate namespace; + a stable idempotency key reuses the same scope on resume. `ingestion.json` records its namespace and scope ID. - Ingestion: each session is captured as one `ContentSource`, then flushed through the configured Memory extraction model. Gold answers and QA evidence annotations are never included in captured Source content. - Extraction profile: `coding` keeps the default work-context policy; `conversation` preserves independently @@ -53,7 +54,7 @@ The runner reads the same `.env` contract as the PowerContext Server. At minimum model, and a complete embedding profile. Secret values and the database URL are never written to result artifacts. ```bash -uv run python -m benchmark.locomo inspect --env-file .env +uv run python -m evaluation.memory.locomo inspect --env-file .env ``` The checked `.env.example` lists variable names only. Keep real credentials in the repository-root `.env`, which is @@ -64,7 +65,7 @@ already ignored. A small real-service smoke run uses the first conversation and five scored questions: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-smoke \ --conversation-limit 1 \ @@ -74,7 +75,7 @@ uv run python -m benchmark.locomo run \ The complete reference-shaped run is: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-full \ --top-k 30 @@ -83,7 +84,7 @@ uv run python -m benchmark.locomo run \ Retrieve a broad candidate pool, then send a more precise context to the answer model: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-rerank \ --top-k 30 \ @@ -103,7 +104,7 @@ answer model. For an isolated inference-answering treatment, keep retrieval fixed and enable the versioned prompt explicitly: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-inference-aware \ --top-k 30 \ @@ -119,7 +120,7 @@ and an independent `rejudge` run when comparing it with a baseline. To preserve direct-answer behavior while giving abstentions one inference attempt, enable the exact-Unknown fallback: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-unknown-fallback-inference \ --top-k 30 \ @@ -138,13 +139,13 @@ Run an isolated extraction-profile A/B with distinct database scopes and output other setting fixed: ```bash -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-ab-coding \ --memory-extraction-profile coding \ --top-k 30 -uv run python -m benchmark.locomo run \ +uv run python -m evaluation.memory.locomo run \ --env-file .env \ --run-id locomo-ab-conversation \ --memory-extraction-profile conversation \ @@ -156,6 +157,10 @@ the persisted Source cursor. Per-question results are appended to `observations. successful questions and retries errors. Pass `--keep-errors` only when you deliberately want existing errors to remain zero-scored without retrying. +Evaluation requires the registered Scope mapping in `ingestion.json` and the original database state. Missing +ingestion or deleted Scopes fail explicitly. Historical runs that used logical names directly as Scope IDs require +a new run ID and output directory; they are not reassigned to new Scopes during resume. + Each inference operation retries timeout and temporary-provider failures up to three attempts by default. The runner does not retry database, schema, invalid-output, or consistency failures. Change the bound with `--operation-retries`, using a new run identity because it is part of `run.json`. @@ -164,10 +169,10 @@ Useful stage controls: ```bash # Reuse an already-ingested database namespace. -uv run python -m benchmark.locomo run --env-file .env --run-id locomo-full --skip-ingestion +uv run python -m evaluation.memory.locomo run --env-file .env --run-id locomo-full --skip-ingestion # Ingest without spending answer/judge requests yet. -uv run python -m benchmark.locomo run --env-file .env --run-id locomo-full --skip-evaluation +uv run python -m evaluation.memory.locomo run --env-file .env --run-id locomo-full --skip-evaluation ``` Do not change dataset selection, candidate K, answer K, rerank policy, model profile, or limits while reusing an @@ -175,7 +180,7 @@ output directory. `run.json` rejects such drift before stateful work begins. ## Outputs -Each run writes under `benchmark/locomo/results//`: +Each run writes under `evaluation/memory/locomo/results//`: - `run.json`: immutable dataset, selection, metric, and non-secret deployment identity. - `ingestion.json`: session progress, extracted Memory counts, no-change flushes, and extraction latency. diff --git a/benchmark/locomo/__init__.py b/evaluation/memory/locomo/__init__.py similarity index 100% rename from benchmark/locomo/__init__.py rename to evaluation/memory/locomo/__init__.py diff --git a/benchmark/locomo_plus/__main__.py b/evaluation/memory/locomo/__main__.py similarity index 92% rename from benchmark/locomo_plus/__main__.py rename to evaluation/memory/locomo/__main__.py index 8cf663d849..8afdd5ad16 100644 --- a/benchmark/locomo_plus/__main__.py +++ b/evaluation/memory/locomo/__main__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run ``python -m benchmark.locomo_plus``.""" +"""Run ``python -m evaluation.memory.locomo``.""" from .cli import main diff --git a/benchmark/locomo/cli.py b/evaluation/memory/locomo/cli.py similarity index 99% rename from benchmark/locomo/cli.py rename to evaluation/memory/locomo/cli.py index b55eabbc0d..3525cf23d3 100644 --- a/benchmark/locomo/cli.py +++ b/evaluation/memory/locomo/cli.py @@ -52,7 +52,7 @@ def main(argv: list[str] | None = None) -> int: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - prog="python -m benchmark.locomo", + prog="python -m evaluation.memory.locomo", description="Run LoCoMo through PowerContext Source, Memory, Hybrid retrieval, answer, and judge stages.", ) commands = parser.add_subparsers(dest="command", required=True) diff --git a/benchmark/locomo/dataset.py b/evaluation/memory/locomo/dataset.py similarity index 100% rename from benchmark/locomo/dataset.py rename to evaluation/memory/locomo/dataset.py diff --git a/benchmark/locomo/dataset/locomo10.json b/evaluation/memory/locomo/dataset/locomo10.json similarity index 100% rename from benchmark/locomo/dataset/locomo10.json rename to evaluation/memory/locomo/dataset/locomo10.json diff --git a/benchmark/locomo/metrics.py b/evaluation/memory/locomo/metrics.py similarity index 100% rename from benchmark/locomo/metrics.py rename to evaluation/memory/locomo/metrics.py diff --git a/benchmark/locomo/prompts.py b/evaluation/memory/locomo/prompts.py similarity index 100% rename from benchmark/locomo/prompts.py rename to evaluation/memory/locomo/prompts.py diff --git a/benchmark/locomo/results/.gitkeep b/evaluation/memory/locomo/results/.gitkeep similarity index 100% rename from benchmark/locomo/results/.gitkeep rename to evaluation/memory/locomo/results/.gitkeep diff --git a/benchmark/locomo/runner.py b/evaluation/memory/locomo/runner.py similarity index 94% rename from benchmark/locomo/runner.py rename to evaluation/memory/locomo/runner.py index 596556e361..b777700409 100644 --- a/benchmark/locomo/runner.py +++ b/evaluation/memory/locomo/runner.py @@ -42,6 +42,7 @@ from powercontext.builtin.inference.errors import InferenceTimeoutError, InferenceUnavailableError from powercontext.builtin.inference.pydantic_ai import InferenceLimits, PydanticAIStructuredGenerator from powercontext.builtin.runtime import BuiltinConfig, CaptureSource, SearchMemoryRequest, open_builtin_runtime +from powercontext.builtin.scope import ScopeDraft from powercontext.server.settings import ServerSettings from .dataset import LoCoMoConversation, LoCoMoDataset, LoCoMoQuestion, load_locomo, render_session @@ -229,6 +230,7 @@ def prepare_run( "operation_retries": operation_retries, "generation_temperature": BENCHMARK_TEMPERATURE, "ingestion": "source-capture-and-memory-extraction", + "scope_registration": "idempotent-registry-v1", "retrieval_mode": "hybrid", "answer_instructions": answer_instructions_version, "judge_profile": judge_profile.value, @@ -264,7 +266,12 @@ async def ingest_dataset( if operation_retries < 1: raise ValueError("operation_retries must be positive") # noqa: TRY003 conversations = dataset.conversations if conversation_limit is None else dataset.conversations[:conversation_limit] - config = _runtime_config(settings, generation=True) + persisted_scopes = ( + _ingested_scope_ids(output_directory, run_id, conversations) + if (output_directory / "ingestion.json").is_file() + else {} + ) + config = _runtime_config(settings) started = perf_counter() total_sessions = sum(len(conversation.sessions) for conversation in conversations) completed_sessions = 0 @@ -280,7 +287,9 @@ async def ingest_dataset( async def ingest_conversation(conversation: LoCoMoConversation) -> None: nonlocal completed_sessions, resumed_sessions, transient_retries, unchanged_flushes async with semaphore: - scope = scope_id(run_id, conversation.sample_id) + scope = await _ingestion_scope_id( + runtime, run_id, conversation.sample_id, persisted_scopes.get(conversation.sample_id) + ) source_app = runtime.sources.for_scope(scope) memory_app = runtime.memory.for_scope(scope) for session in conversation.sessions: @@ -332,6 +341,7 @@ async def ingest_conversation(conversation: LoCoMoConversation) -> None: entries = await memory_app.list() conversation_results[conversation.sample_id] = { "scope_id": scope, + "namespace": scope_id(run_id, conversation.sample_id), "session_count": len(conversation.sessions), "memory_entry_count": len(entries.entries), "memory_revision": None if entries.memory_ref is None else entries.memory_ref.revision, @@ -421,9 +431,12 @@ async def evaluate_dataset( # noqa: C901 ) progress(f"[evaluate] selected={len(selected)} resumed={len(selected) - len(pending)} pending={len(pending)}") if pending: + conversations = ( + dataset.conversations if conversation_limit is None else dataset.conversations[:conversation_limit] + ) + scope_ids = _ingested_scope_ids(output_directory, run_id, conversations) config = _runtime_config( settings, - generation=False, rerank_mode=rerank_mode, rerank_candidate_limit=top_k, ) @@ -464,7 +477,7 @@ async def evaluate_dataset( # noqa: C901 limits=limits, model_settings=_benchmark_model_settings(), ) - entry_sources = await _entry_source_maps(runtime, dataset, run_id, conversation_limit) + entry_sources = await _entry_source_maps(runtime, scope_ids) async def evaluate_one(question: LoCoMoQuestion) -> dict[str, Any]: async with semaphore: @@ -476,7 +489,7 @@ async def evaluate_one(question: LoCoMoQuestion) -> dict[str, Any]: question=question, conversation=conversation_by_id[question.sample_id], entry_sources=entry_sources[question.sample_id], - run_id=run_id, + scope=scope_ids[question.sample_id], top_k=top_k, answer_k=selected_answer_k, rerank_mode=rerank_mode, @@ -899,7 +912,7 @@ async def _evaluate_question( question: LoCoMoQuestion, conversation: LoCoMoConversation, entry_sources: Mapping[tuple[str, str], tuple[str, ...]], - run_id: str, + scope: str, top_k: int, answer_k: int, rerank_mode: MemoryRerankMode, @@ -911,7 +924,7 @@ async def _evaluate_question( try: search_started = perf_counter() result, search_retries = await _retry_transient( - lambda: runtime.memory.for_scope(scope_id(run_id, question.sample_id)).search( + lambda: runtime.memory.for_scope(scope).search( SearchMemoryRequest( query=question.question, limit=answer_k if rerank_mode is MemoryRerankMode.LLM else top_k, @@ -1161,12 +1174,12 @@ def _answer_source_sessions( ) -async def _entry_source_maps(runtime, dataset: LoCoMoDataset, run_id: str, conversation_limit: int | None): - conversations = dataset.conversations if conversation_limit is None else dataset.conversations[:conversation_limit] +async def _entry_source_maps(runtime, scope_ids: Mapping[str, str]): mappings: dict[str, dict[tuple[str, str], tuple[str, ...]]] = {} - for conversation in conversations: - page = await runtime.memory.for_scope(scope_id(run_id, conversation.sample_id)).list() - mappings[conversation.sample_id] = { + for sample_id, scope in scope_ids.items(): + await runtime.scopes.get(scope) + page = await runtime.memory.for_scope(scope).list() + mappings[sample_id] = { (record.entry.entry_id, record.entry.entry_version_id): tuple( source.source_id for source in record.entry.sources ) @@ -1178,16 +1191,10 @@ async def _entry_source_maps(runtime, dataset: LoCoMoDataset, run_id: str, conve def _runtime_config( settings: ServerSettings, *, - generation: bool, rerank_mode: MemoryRerankMode = MemoryRerankMode.NONE, rerank_candidate_limit: int = 30, ) -> BuiltinConfig: rerank_enabled = rerank_mode is MemoryRerankMode.LLM - inference = ( - settings.inference - if generation or rerank_enabled - else settings.inference.model_copy(update={"generation_model": None}) - ) return BuiltinConfig( runtime=settings.runtime.model_copy( update={ @@ -1198,7 +1205,7 @@ def _runtime_config( } ), database=settings.database, - inference=inference, + inference=settings.inference, external_skills=settings.external_skills, ) @@ -1217,11 +1224,53 @@ async def _retry_transient(action: Callable[[], Awaitable[ResultT]], *, attempts def scope_id(run_id: str, sample_id: str) -> str: - """Return the isolated, stable PowerContext namespace for one conversation.""" + """Return the stable logical namespace used to register one conversation's Scope.""" return f"benchmark:locomo:{normalize_run_id(run_id)}:{sample_id}" +def _ingested_scope_ids( + output_directory: Path, run_id: str, conversations: Sequence[LoCoMoConversation] +) -> dict[str, str]: + path = output_directory / "ingestion.json" + if not path.is_file(): + raise ValueError("LoCoMo evaluation requires ingestion.json; run ingestion first") # noqa: TRY003 + ingestion = json.loads(path.read_text(encoding="utf-8")) + if ingestion.get("run_id") != normalize_run_id(run_id): + raise ValueError("LoCoMo ingestion belongs to a different run; use its original run ID") # noqa: TRY003 + records = ingestion.get("conversations", {}) + scopes: dict[str, str] = {} + for conversation in conversations: + record = records.get(conversation.sample_id, {}) + namespace = scope_id(run_id, conversation.sample_id) + registered_scope = record.get("scope_id") + if ( + record.get("namespace") != namespace + or not isinstance(registered_scope, str) + or not registered_scope.strip() + ): + raise ValueError( # noqa: TRY003 + "LoCoMo ingestion lacks a registered Scope mapping; legacy logical scopes require a new run" + ) + scopes[conversation.sample_id] = registered_scope + return scopes + + +async def _ingestion_scope_id(runtime, run_id: str, sample_id: str, persisted_scope: str | None) -> str: + if persisted_scope is not None: + await runtime.scopes.get(persisted_scope) + return persisted_scope + namespace = scope_id(run_id, sample_id) + registered = await runtime.scopes.create( + ScopeDraft( + title="LoCoMo conversation", + summary=f"Isolated evaluation namespace: {namespace}", + idempotency_key=hashlib.sha256(namespace.encode()).hexdigest(), + ) + ) + return registered.scope_id + + def normalize_run_id(value: str) -> str: normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value.strip()).strip("-") if not normalized: diff --git a/benchmark/locomo_plus/.env.example b/evaluation/memory/locomo_plus/.env.example similarity index 65% rename from benchmark/locomo_plus/.env.example rename to evaluation/memory/locomo_plus/.env.example index 31671bca0b..37d6312544 100644 --- a/benchmark/locomo_plus/.env.example +++ b/evaluation/memory/locomo_plus/.env.example @@ -1,6 +1,12 @@ -# Copy to benchmark/locomo_plus/.env and pass that path with --env-file. -# Each run creates its own SQLite database in results//. -# No external database configuration is required. +# Copy to evaluation/memory/locomo_plus/.env and pass that path with --env-file. +# Without database settings, each run creates its own SQLite database in results//. +# To use OceanBase, create a dedicated evaluation database and set both values below. +# The runner uses this database and isolates runs by Scope; it does not create or drop databases. +# POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase +# POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://USER:PASSWORD@HOST:2881/EVALUATION_DB?charset=utf8mb4 +# An explicitly configured SQLite database is also supported; use a persistent file for resume. +# POWERCONTEXT_SERVER_DATABASE_KIND=sqlite +# POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:////absolute/path/to/evaluation.sqlite3 # Generation model for conversation Memory extraction and benchmark answers. POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL=openai:your-chat-model diff --git a/benchmark/locomo_plus/.gitignore b/evaluation/memory/locomo_plus/.gitignore similarity index 100% rename from benchmark/locomo_plus/.gitignore rename to evaluation/memory/locomo_plus/.gitignore diff --git a/benchmark/locomo_plus/README.md b/evaluation/memory/locomo_plus/README.md similarity index 72% rename from benchmark/locomo_plus/README.md rename to evaluation/memory/locomo_plus/README.md index f07a2be639..a31d3e1744 100644 --- a/benchmark/locomo_plus/README.md +++ b/evaluation/memory/locomo_plus/README.md @@ -1,8 +1,9 @@ # LoCoMo-Plus benchmark The default `memory` arm captures timestamped dialogue Sources, extracts conversational Memory, retrieves Memory -with hybrid search, generates an answer, and grades it with an explicitly selected judge model. Each run uses its own -SQLite database inside its results directory; the database configuration in the supplied environment file is not used. +with hybrid search, generates an answer, and grades it with an explicitly selected judge model. The runner uses the +Server database configuration from the supplied environment file. When no database settings are supplied, each run +uses its own SQLite database inside its results directory. ## Dataset and evaluation contract @@ -51,9 +52,9 @@ prompt policy, model choice, and history limits all affect the result. Upstream Use the project environment installed with `uv sync`: ```bash -uv run python -m benchmark.locomo_plus inspect -uv run python -m benchmark.locomo_plus run --dry-run -uv run python -m benchmark.locomo_plus run --profile full --limit 2 --dry-run +uv run python -m evaluation.memory.locomo_plus inspect +uv run python -m evaluation.memory.locomo_plus run --dry-run +uv run python -m evaluation.memory.locomo_plus run --profile full --limit 2 --dry-run ``` These commands may download the pinned dataset but require no model credentials and make no inference requests. @@ -63,10 +64,10 @@ policy. `--dry-run` reports the selected cases and ingestion plan before incurri Inspect and plan the bundled ten cases without downloading any data: ```bash -uv run python -m benchmark.locomo_plus inspect \ - --dataset-file benchmark/locomo_plus/dataset/locomo_plus_smoke10.json -uv run python -m benchmark.locomo_plus run \ - --dataset-file benchmark/locomo_plus/dataset/locomo_plus_smoke10.json \ +uv run python -m evaluation.memory.locomo_plus inspect \ + --dataset-file evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json +uv run python -m evaluation.memory.locomo_plus run \ + --dataset-file evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json \ --limit 10 --dry-run ``` @@ -76,25 +77,47 @@ selection, or full profile. Omitting `--dataset-file` preserves the upstream-loa ## Smoke and full runs -The runner reads generation and embedding settings from the same `.env` contract as the PowerContext Server. +The runner reads database, generation and embedding settings from the same `.env` contract as the PowerContext Server. Copy [.env.example](.env.example) to a local file and fill in the model names, endpoints, API key, and embedding dimension. From the repository root: ```bash -cp benchmark/locomo_plus/.env.example benchmark/locomo_plus/.env +cp evaluation/memory/locomo_plus/.env.example evaluation/memory/locomo_plus/.env ``` -The commands below use `--env-file benchmark/locomo_plus/.env`; an existing compatible environment file can also be -passed directly. No external database configuration is needed because each run uses its own SQLite database. +The commands below use `--env-file evaluation/memory/locomo_plus/.env`; an existing compatible environment file can also be +passed directly. Without database settings, the runner creates `state.sqlite3` in the results directory. Explicit +database settings use `settings.database`, including a configured SQLite URL, OceanBase URL or seekdb path. +Memory arms require persistent storage; in-memory SQLite is rejected because it cannot retain state for resume. Keep credentials outside tracked files. The judge model must be passed explicitly on every paid run. To use an independent judge, choose a different model from the configured generator; explicit selection alone does not establish judge independence or human agreement. Using the same model is supported and flagged in the manifest. +For OceanBase, set `POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase` and +`POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://USER:PASSWORD@HOST:2881/EVALUATION_DB?charset=utf8mb4` in the selected environment +file. Create a dedicated evaluation database first: the runner initializes PowerContext tables in the configured +database and does not create or drop databases. Each results directory receives a persistent random Scope namespace, +so separate runs remain isolated even when they share a database and the same `--run-id`. Benchmark state remains in +that database for resume; dispose of a dedicated evaluation database only after its runs are no longer needed. + +`run.json` and the summaries record the actual backend and a database target fingerprint. Connection credentials and +the raw database URL are excluded. OceanBase fingerprints use the installed dialect's effective host, port, +tenant/user and database, including query-string overrides such as `db`, `user`, `host` and `port`. When a Unix socket +is configured, its resolved path replaces the TCP host and port. Passwords in either the URL authority or query string, +TLS/authentication material and connection tuning options do not affect the fingerprint. The runtime overrides +`init_command` with its fixed transaction initialization command. Duplicate query parameters, `read_default_file`, +`read_default_group` and `sql_mode` are rejected before opening services because their routing is ambiguous or can +depend on external configuration or SQL. SQLite and seekdb fingerprints continue to identify the local database path. +Reusing a results directory +with a different database target is rejected before opening the database or calling models. Pending Memory runs +also verify that their saved Scopes still exist, including judge-only retries; missing Scopes are not recreated. +Completed runs can regenerate their summary without opening the database; use `replay` for explicitly offline checks. + A single command runs the fixed four-case smoke profile, covering causal, state, goal, and value: ```bash -uv run python -m benchmark.locomo_plus run \ - --env-file benchmark/locomo_plus/.env \ +uv run python -m evaluation.memory.locomo_plus run \ + --env-file evaluation/memory/locomo_plus/.env \ --judge-model "openai:YOUR_JUDGE_MODEL" \ --run-id locomo-plus-smoke ``` @@ -108,9 +131,9 @@ count, SHA-256, and a comparison with the corresponding full-dataset history. For a ten-case smoke using the bundled complete histories: ```bash -uv run python -m benchmark.locomo_plus run \ - --dataset-file benchmark/locomo_plus/dataset/locomo_plus_smoke10.json \ - --env-file benchmark/locomo_plus/.env \ +uv run python -m evaluation.memory.locomo_plus run \ + --dataset-file evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json \ + --env-file evaluation/memory/locomo_plus/.env \ --judge-model "openai:YOUR_JUDGE_MODEL" \ --run-id locomo-plus-smoke-10 \ --limit 10 @@ -124,9 +147,9 @@ random population sample. Reducing the limit never shortens any selected case's The full profile is explicit and retains complete histories by default: ```bash -uv run python -m benchmark.locomo_plus run \ +uv run python -m evaluation.memory.locomo_plus run \ --profile full \ - --env-file benchmark/locomo_plus/.env \ + --env-file evaluation/memory/locomo_plus/.env \ --judge-model "openai:YOUR_JUDGE_MODEL" \ --run-id locomo-plus-full ``` @@ -165,7 +188,7 @@ Factual and cognitive scores are reported separately, with cognitive results spl The report distinguishes quality among completed evaluations from end-to-end success over all planned valid cases. Dataset exclusions are audited separately from inference and infrastructure failures. -Outputs default to `benchmark/locomo_plus/results//`: +Outputs default to `evaluation/memory/locomo_plus/results//`: - `run.json`: immutable run identity, selection, model configuration, harness fingerprint, and execution settings. - `dataset-audit.json`: pinned provenance, source counts, construction policy, and excluded rows. @@ -178,7 +201,7 @@ Outputs default to `benchmark/locomo_plus/results//`: - `observations.jsonl`: frozen retrieval/context, rendered generator and judge inputs, answers, judgments, usage, latency, and classified errors. - `summary.json` and `summary.md`: grouped quality, completion, retrieval, usage, and cost reports. -- `state.sqlite3`: the run's isolated local PowerContext database for Memory arms. +- `state.sqlite3`: the run's isolated local PowerContext database when no database settings were supplied. The benchmark-local `.gitignore` ignores generated results while retaining `results/.gitkeep`. @@ -186,14 +209,19 @@ Artifacts contain dataset text and model output, so keep them local unless delib Credentials and the environment database URL are not included in run manifests. Reuse the same run ID and output location with the same settings to resume. Successful cases are retained; failed cases are retried. A judge-only failure reuses the frozen generated answer. Configuration or dataset changes require a separate run. +Runs without a saved database fingerprint and Scope namespace cannot be resumed; their saved results remain +available through `replay`. Preserve `ingestion.json` alongside `run.json` to retain the original Scope identities. +OceanBase manifests must also record `database_fingerprint_version: oceanbase-target-v2`. Unversioned OceanBase +fingerprints are not reinterpreted or migrated: their saved results remain replayable, but a new run needs a new +output directory. This version requirement does not apply to SQLite or seekdb manifests. The run command exits with status `1` if cases fail to execute or remain unobserved. A completed evaluation with an incorrect model answer still exits with status `0`; answer quality is recorded in the report. Rebuild summary artifacts from saved observations without downloading data, opening the database, or calling a model: ```bash -uv run python -m benchmark.locomo_plus replay \ - --run-directory benchmark/locomo_plus/results/locomo-plus-smoke +uv run python -m evaluation.memory.locomo_plus replay \ + --run-directory evaluation/memory/locomo_plus/results/locomo-plus-smoke ``` Replay uses saved answers and judgments; it does not ask a new judge to grade them. diff --git a/benchmark/locomo_plus/__init__.py b/evaluation/memory/locomo_plus/__init__.py similarity index 100% rename from benchmark/locomo_plus/__init__.py rename to evaluation/memory/locomo_plus/__init__.py diff --git a/benchmark/locomo/__main__.py b/evaluation/memory/locomo_plus/__main__.py similarity index 92% rename from benchmark/locomo/__main__.py rename to evaluation/memory/locomo_plus/__main__.py index 6f9d686ee2..1538c2a01b 100644 --- a/benchmark/locomo/__main__.py +++ b/evaluation/memory/locomo_plus/__main__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run ``python -m benchmark.locomo``.""" +"""Run ``python -m evaluation.memory.locomo_plus``.""" from .cli import main diff --git a/benchmark/locomo_plus/cli.py b/evaluation/memory/locomo_plus/cli.py similarity index 98% rename from benchmark/locomo_plus/cli.py rename to evaluation/memory/locomo_plus/cli.py index 25dec8c360..825279ab7c 100644 --- a/benchmark/locomo_plus/cli.py +++ b/evaluation/memory/locomo_plus/cli.py @@ -47,7 +47,7 @@ def main(argv: list[str] | None = None) -> int: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - prog="python -m benchmark.locomo_plus", + prog="python -m evaluation.memory.locomo_plus", description="Evaluate pinned LoCoMo-Plus factual and cognitive cases through PowerContext.", ) commands = parser.add_subparsers(dest="command", required=True) @@ -67,7 +67,7 @@ def _parser() -> argparse.ArgumentParser: run.add_argument("--env-file", type=Path, default=Path(".env")) run.add_argument("--judge-model", type=_nonempty_text, help="explicit judge model, for example openai:gpt-4o-mini") run.add_argument("--run-id", type=_nonempty_text, help="stable isolated namespace; reuse it to resume") - run.add_argument("--output-directory", type=Path, help="defaults to benchmark/locomo_plus/results/") + run.add_argument("--output-directory", type=Path, help="defaults to evaluation/memory/locomo_plus/results/") run.add_argument("--top-k", type=_positive_int, default=5) run.add_argument( "--max-tokens", type=_positive_int, default=512, help="output cap for each answer and judge request" diff --git a/benchmark/locomo_plus/dataset.py b/evaluation/memory/locomo_plus/dataset.py similarity index 99% rename from benchmark/locomo_plus/dataset.py rename to evaluation/memory/locomo_plus/dataset.py index daf9dbb756..d9d6363ab7 100644 --- a/benchmark/locomo_plus/dataset.py +++ b/evaluation/memory/locomo_plus/dataset.py @@ -29,7 +29,7 @@ from pydantic import TypeAdapter -from benchmark.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn, load_locomo +from evaluation.memory.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn, load_locomo DEFAULT_DATA_DIR = Path(".cache/locomo_plus") DEFAULT_SMOKE_PATH = Path(__file__).with_name("dataset") / "locomo_plus_smoke10.json" diff --git a/benchmark/locomo_plus/dataset/README.md b/evaluation/memory/locomo_plus/dataset/README.md similarity index 94% rename from benchmark/locomo_plus/dataset/README.md rename to evaluation/memory/locomo_plus/dataset/README.md index 4b8b2b283a..fe5096c510 100644 --- a/benchmark/locomo_plus/dataset/README.md +++ b/evaluation/memory/locomo_plus/dataset/README.md @@ -30,8 +30,8 @@ The upstream data retains its original provenance; the pinned upstream revision From the repository root, inspect the ten-case workload without network or model calls: ```bash -uv run python -m benchmark.locomo_plus run \ - --dataset-file benchmark/locomo_plus/dataset/locomo_plus_smoke10.json \ +uv run python -m evaluation.memory.locomo_plus run \ + --dataset-file evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json \ --limit 10 --dry-run ``` diff --git a/benchmark/locomo_plus/dataset/locomo_plus_smoke10.json b/evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json similarity index 100% rename from benchmark/locomo_plus/dataset/locomo_plus_smoke10.json rename to evaluation/memory/locomo_plus/dataset/locomo_plus_smoke10.json diff --git a/benchmark/locomo_plus/metrics.py b/evaluation/memory/locomo_plus/metrics.py similarity index 99% rename from benchmark/locomo_plus/metrics.py rename to evaluation/memory/locomo_plus/metrics.py index 50acf40c82..540122fcce 100644 --- a/benchmark/locomo_plus/metrics.py +++ b/evaluation/memory/locomo_plus/metrics.py @@ -22,8 +22,8 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Any -from benchmark.locomo.metrics import percentile -from benchmark.locomo_plus.prompts import PARTIAL_CATEGORIES, category_name +from evaluation.memory.locomo.metrics import percentile +from evaluation.memory.locomo_plus.prompts import PARTIAL_CATEGORIES, category_name FAILURE_STAGES = ("dataset", "infrastructure", "retrieval", "generation", "judge") _CONSTRAINT_TYPES = frozenset({"causal", "state", "goal", "value"}) diff --git a/benchmark/locomo_plus/models.py b/evaluation/memory/locomo_plus/models.py similarity index 100% rename from benchmark/locomo_plus/models.py rename to evaluation/memory/locomo_plus/models.py diff --git a/benchmark/locomo_plus/prompts.py b/evaluation/memory/locomo_plus/prompts.py similarity index 100% rename from benchmark/locomo_plus/prompts.py rename to evaluation/memory/locomo_plus/prompts.py diff --git a/benchmark/locomo_plus/results/.gitkeep b/evaluation/memory/locomo_plus/results/.gitkeep similarity index 100% rename from benchmark/locomo_plus/results/.gitkeep rename to evaluation/memory/locomo_plus/results/.gitkeep diff --git a/benchmark/locomo_plus/runner.py b/evaluation/memory/locomo_plus/runner.py similarity index 82% rename from benchmark/locomo_plus/runner.py rename to evaluation/memory/locomo_plus/runner.py index 19babf0b4f..7754ea3ee8 100644 --- a/benchmark/locomo_plus/runner.py +++ b/evaluation/memory/locomo_plus/runner.py @@ -27,14 +27,18 @@ from pathlib import Path from time import perf_counter from typing import Any +from uuid import uuid4 from pydantic import ValidationError from pydantic_ai import capture_run_messages from pydantic_ai.messages import ModelMessagesTypeAdapter +from pyobvector import AsyncOceanBaseDialect +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError -from benchmark.locomo.dataset import LoCoMoSession -from benchmark.locomo.metrics import retrieval_metrics -from benchmark.locomo.runner import load_settings, normalize_run_id, public_configuration +from evaluation.memory.locomo.dataset import LoCoMoSession +from evaluation.memory.locomo.metrics import retrieval_metrics +from evaluation.memory.locomo.runner import load_settings, normalize_run_id, public_configuration from powercontext.builtin.artifacts.memory.prompts import memory_extraction_instructions_version from powercontext.builtin.inference import InvalidInferenceOutputError, character_token_estimator from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -46,6 +50,7 @@ SearchMemoryRequest, open_builtin_runtime, ) +from powercontext.builtin.runtime.config import DatabaseConfig from powercontext.builtin.scope import ScopeDraft from powercontext.server.settings import ServerSettings @@ -215,10 +220,15 @@ def dry_run_plan( def _harness_identity() -> dict[str, Any]: - root = Path(__file__).resolve().parents[2] + root = Path(__file__).resolve().parents[3] digest = hashlib.sha256() paths = sorted((root / "src" / "powercontext").rglob("*.py")) - paths += sorted((root / "benchmark").rglob("*.py")) + # Unrelated memory suites must not invalidate a frozen LoCoMo-Plus run. + memory_root = root / "evaluation" / "memory" + paths += sorted(memory_root.glob("*.py")) + for suite in ("locomo", "locomo_plus"): + paths += sorted((memory_root / suite).rglob("*.py")) + paths += sorted((root / "evaluation" / "performance").rglob("*.py")) paths += [root / "uv.lock"] for path in paths: digest.update(str(path.relative_to(root)).encode()) @@ -266,12 +276,91 @@ def _cost(usage: dict[str, Any], model: str | None, prices: dict[str, Any] | Non return {**usage, "model": model, "cost_usd": cost, "pricing_version": None if prices is None else prices["version"]} -def _configuration(settings, judge_model, max_tokens) -> dict[str, Any]: +def _database_config(settings: ServerSettings, output_directory: Path) -> DatabaseConfig: + if "database" not in settings.model_fields_set: + return SQLiteConfig(url=f"sqlite+aiosqlite:///{output_directory / 'state.sqlite3'}") + return settings.database + + +def _oceanbase_target(url: URL) -> dict[str, Any]: + if any(not isinstance(value, str) for value in url.query.values()): + raise ValueError("OceanBase URL query parameters must not be repeated") # noqa: TRY003 + if {key.casefold() for key in url.query} & {"read_default_file", "read_default_group", "sql_mode"}: + raise ValueError("OceanBase external defaults and SQL mode overrides are not supported for resumable runs") # noqa: TRY003 + try: + # URL query values override the translated authority/path in the installed dialect. + _, options = AsyncOceanBaseDialect().create_connect_args(url) + username = options.get("user") + database = options.get("db") + host = options.get("host") + port = int(options.get("port", 3306)) + if ( + not isinstance(host, str) + or not host + or not all(isinstance(value, str) and value for value in (username, database)) + or not 1 <= port <= 65535 + ): + raise ValueError # noqa: TRY301 + except (ArgumentError, TypeError, ValueError): + raise ValueError( # noqa: TRY003 + "OceanBase connection target must explicitly identify its host, port, user and database" + ) from None + socket = options.get("unix_socket") + # Passwords, TLS/authentication material and tuning options do not identify the database. + # init_command is replaced by the runtime's fixed SET autocommit = 0 command. + return { + "driver": url.drivername, + "username": username, + "database": database, + "host": None if socket else host.lower(), + "port": None if socket else port, + "unix_socket": str(Path(socket).resolve()) if socket else None, + } + + +def _database_identity(database: DatabaseConfig) -> dict[str, str]: + if database.kind == "oceanbase": + target = _oceanbase_target(make_url(database.url.get_secret_value())) + return { + "database_kind": database.kind, + "database_fingerprint_version": "oceanbase-target-v2", + "database_fingerprint": _digest(json.dumps(target, sort_keys=True)), + } + target: dict[str, Any] + if database.kind == "seekdb": + target = {"path": str(database.path.resolve()), "database": database.database} + else: + url = make_url(database.url) + name = url.database + if name and name != ":memory:" and not name.startswith("file:"): + name = str(Path(name).resolve()) + url = url.set(database=name) + target = { + "driver": url.drivername, + "host": (url.host or "").lower(), + "port": url.port or None, + "username": url.username, + "database": url.database, + "query": dict(url.query), + } + if (url.database or "").startswith("file:"): + target["working_directory"] = str(Path.cwd().resolve()) + return { + "database_kind": database.kind, + "database_fingerprint": _digest(json.dumps(target, sort_keys=True)), + } + + +def _configuration(settings, judge_model, max_tokens, database) -> dict[str, Any]: inference = settings.inference return { **public_configuration(settings), - "database_kind": "sqlite", - "persistence": "isolated output-directory/state.sqlite3", + **_database_identity(database), + "persistence": ( + "configured database; isolated run scopes" + if "database" in settings.model_fields_set + else "isolated output-directory/state.sqlite3" + ), "judge_model": judge_model, "judge_shares_generator_model": judge_model == inference.generation_model, "temperature": 0.0, @@ -468,6 +557,8 @@ async def _evaluate( max_tokens, prices, ingestion, + scope_ids, + scope_namespace, previous, ): started = perf_counter() @@ -496,14 +587,20 @@ async def _evaluate( prepared = perf_counter() hits, selected_ids, retrieval = [], [], {} if arm.startswith("memory"): - registered = await runtime.scopes.create( - ScopeDraft( - title="Benchmark conversation", - summary="Isolated conversation evidence for a reproducible local evaluation.", - idempotency_key=_digest(_scope(run_id, case.sample_id)), + namespace = _scope(run_id, case.sample_id) + scope = scope_ids.get(namespace) + if scope is None: + registered = await runtime.scopes.create( + ScopeDraft( + title="Benchmark conversation", + summary="Isolated conversation evidence for a reproducible evaluation.", + idempotency_key=_digest(f"{scope_namespace}:{namespace}"), + ) ) - ) - scope = registered.scope_id + scope = registered.scope_id + scope_ids[namespace] = scope + ingestion[scope] = {"scope_id": scope, "namespace": namespace, "latency_ms": 0.0} + _write_json(output_directory / "ingestion.json", ingestion) observation["scope_id"] = scope page = await _ingest(runtime, case, sessions, scope, output_directory, ingestion, prices, settings) phase = "retrieval" @@ -665,17 +762,27 @@ async def run_benchmark( # noqa: C901 run_id = normalize_run_id(run_id) output_directory = output_directory.resolve() # noqa: ASYNC240 - small local artifact operation output_directory.mkdir(parents=True, exist_ok=True) + database = _database_config(settings, output_directory) + if arm.startswith("memory") and database.kind == "sqlite": + url = make_url(database.url) + if database.is_in_memory or url.query.get("mode") == "memory" or url.database == "file::memory:": + raise ValueError("Memory runs require a persistent database for resume; configure a SQLite file") # noqa: TRY003 + manifest_path = output_directory / "run.json" + saved_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else None + scope_namespace = saved_manifest.get("scope_namespace") if saved_manifest is not None else uuid4().hex + if not isinstance(scope_namespace, str) or not scope_namespace: + raise ValueError("run identity changed; use a new output directory (existing runs remain replayable)") # noqa: TRY003 manifest = { "run_id": run_id, + "scope_namespace": scope_namespace, "harness": _harness_identity(), "dataset": dataset.manifest, "selection": plan, - "configuration": _configuration(settings, judge_model, max_tokens), + "configuration": _configuration(settings, judge_model, max_tokens, database), "top_k": top_k, "prices": rates, } - manifest_path = output_directory / "run.json" - if manifest_path.exists() and json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: + if saved_manifest is not None and saved_manifest != manifest: raise ValueError("run identity changed; use a new output directory") # noqa: TRY003 _write_json(manifest_path, manifest) _write_json( @@ -699,6 +806,10 @@ async def run_benchmark( # noqa: C901 pending = [case for case in cases if observed.get(case.case_id, {}).get("status") != "ok"] ingestion_path = output_directory / "ingestion.json" ingestion = json.loads(ingestion_path.read_text(encoding="utf-8")) if ingestion_path.exists() else {} + scope_ids = {record["namespace"]: record["scope_id"] for record in ingestion.values()} + recorded_scopes = set(scope_ids.values()) + if any(row.get("scope_id") not in recorded_scopes for row in observed.values() if row.get("scope_id")): + raise ValueError("saved Scope mapping is missing; restore the run's ingestion.json before resuming") # noqa: TRY003 if not pending: return _summarize(output_directory) try: @@ -706,7 +817,7 @@ async def run_benchmark( # noqa: C901 runtime = None if arm.startswith("memory"): runtime_config = BuiltinConfig( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{output_directory / 'state.sqlite3'}"), + database=database, runtime=RuntimeConfig( memory_extraction_profile=MemoryExtractionProfile.CONVERSATION, dream_enabled=False, @@ -715,6 +826,10 @@ async def run_benchmark( # noqa: C901 inference=settings.inference, ) runtime = await resources.enter_async_context(open_builtin_runtime(runtime_config)) + if runtime.scopes is None: + raise RuntimeError("Memory runs require a Scope registry") # noqa: TRY003, TRY301 + for scope_id in scope_ids.values(): + await runtime.scopes.get(scope_id) answer = await open_model(settings.inference.generation_model, settings.inference, resources) judge = await open_model(judge_model, settings.inference, resources) for index, case in enumerate(pending, 1): @@ -733,6 +848,8 @@ async def run_benchmark( # noqa: C901 max_tokens=max_tokens, prices=rates, ingestion=ingestion, + scope_ids=scope_ids, + scope_namespace=scope_namespace, previous=observed.get(case.case_id), ) _append(output_directory / "observations.jsonl", row) diff --git a/evaluation/memory/longmemeval_v2/README.md b/evaluation/memory/longmemeval_v2/README.md new file mode 100644 index 0000000000..1f27cb77a4 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/README.md @@ -0,0 +1,379 @@ +# LongMemEval-V2 memory evaluation + +Source, data locks, configuration, documentation, and tests for the LongMemEval-V2 suite live here. +The suite is installed through the shared `evaluation/` environment and provides the independent `longmemeval-v2` command. + +## LongMemEval-V2 smoke workload + +The LongMemEval-V2 commands run a fixed ten-question smoke subset end to end: preflight +validation of the pinned inputs, real PowerContext HTTP retrieval, prompt preparation, an +optional Reader, upstream scoring, offline score replay, and a one-command orchestration +(`run-smoke`) that writes one run directory with a unified report. A full-tier run has not been +executed; its recorded configuration lives in +[docs/longmemeval-v2-full-run.md](docs/longmemeval-v2-full-run.md). + +Prepare a detached upstream checkout at the pinned harness commit and download +the matching LongMemEval-V2 data root outside this repository. The checked-in +small-tier lock and smoke manifest fix the data revision, three core data-file +hashes, ten source-ordered questions, all five published abilities, and both +published domains. The dataset lock has this shape: + +```json +{ + "schema": "powercontext.longmemeval-v2-dataset-lock.v1", + "upstream": { + "repository": "https://github.com/xiaowu0162/LongMemEval-V2", + "harness_commit": "2cc8c540bdb87fe6761629b585e727e1c4704520" + }, + "dataset_revision": "DATASET_REVISION", + "tier": "small", + "files": { + "questions.jsonl": "SHA256", + "trajectories.jsonl": "SHA256", + "haystacks/lme_v2_small.json": "SHA256" + } +} +``` + +The smoke manifest fixes question IDs, their upstream source order, and coverage +of the published ability categories. Run the preflight against a new output +directory: + +```bash +uv run --project evaluation longmemeval-v2 smoke \ + --harness-root /path/to/LongMemEval-V2 \ + --data-root /path/to/longmemeval-v2-data \ + --dataset-lock evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json \ + --smoke-manifest evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json \ + --output-dir /path/to/new-smoke-artifacts +``` + +The command refuses a harness checkout at a different commit, mismatched input +hashes, invalid or incomplete smoke coverage, and an existing output directory. +It writes `manifest.json` and `subset.json`, both labelled as a smoke subset. + +### PowerContext Memory adapter bootstrap + +Run the bootstrap with a Python environment containing the pinned upstream +harness dependencies. It validates the clean harness commit, registers +`memory_type: powercontext` in memory, and then forwards all remaining arguments +to the unchanged upstream harness. From a source checkout, the script makes +the suite package available to that interpreter: + +```bash +python evaluation/memory/longmemeval_v2/scripts/run_longmemeval_v2_harness.py \ + --harness-root /path/to/LongMemEval-V2 \ + -- +``` + +If the evaluation wheel is installed in the harness environment, use its module entry point: + +```bash +python -m powercontext_eval_longmemeval_v2.harness_bootstrap \ + --harness-root /path/to/LongMemEval-V2 \ + -- +``` + +The adapter memory configuration requires a dedicated evaluation Scope and +audit artifact path. Credentials are resolved only from `token_env` at runtime: + +```json +{ + "memory_type": "powercontext", + "memory_params": { + "scope_id": "longmemeval-v2-smoke-run-id", + "audit_path": "/path/to/run/context/powercontext-memory.jsonl", + "base_url": "http://127.0.0.1:8765", + "token_env": "POWERCONTEXT_TOKEN", + "search_mode": "auto", + "search_limit": 10 + } +} +``` + +Each trajectory chunk is captured through the public Content Source endpoint +and one deterministic bounded projection per trajectory is explicitly +remembered through the public Memory endpoint. The projection keeps goal, +outcome, URL, action, thought, and bounded observation evidence without calling +a model. The adapter audit correlates the returned Source references and Memory +citation; it does not claim that this correlation is native Memory lineage. +Queries use the public Memory search endpoint and return upstream-compatible +text context items. Query images are neither read nor sent because the current +Memory search contract is text only. + +With a ready PowerContext Server, run the fixed ten-question retrieval-only +smoke workload without a Reader, Judge, model credential, or scoring step: + +```bash +uv run --project evaluation longmemeval-v2 retrieval-smoke \ + --harness-root /path/to/LongMemEval-V2 \ + --data-root /path/to/longmemeval-v2-data \ + --dataset-lock evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json \ + --smoke-manifest evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json \ + --base-url http://127.0.0.1:8000 \ + --run-id retrieval-smoke-001 \ + --powercontext-revision POWERCONTEXT_GIT_SHA \ + --integration-revision INTEGRATION_GIT_SHA \ + --output-dir /path/to/new-retrieval-smoke-artifacts +``` + +The runner verifies the locked input digests while streaming +`trajectories.jsonl`, retains only one trajectory object at a time, and creates +one isolated child Scope for each distinct ordered haystack. Identical +haystacks reuse their ingestion, while different haystacks cannot retrieve each +other's Memory. It writes `retrieval-manifest.json`, +`retrieval-results.jsonl`, `adapter-audit.jsonl`, `failures.jsonl`, and +`summary.json` beside the preflight `manifest.json` and `subset.json`. Accuracy, +Reader, and Judge fields remain null because retrieval-only output is not a +benchmark score. + +### Experiment arms + +Retrieval behaviour is selected through a registered experiment arm, not a free-form search mode: + +```bash +uv run --project evaluation longmemeval-v2 retrieval-smoke \ + ... \ + --experiment-arm current-memory-hybrid-v1 +``` + +An arm is a frozen configuration identity: same questions and the same downstream token +budget, with only the declared retrieval/projection knobs allowed to differ. Five arms are +currently registered: + +| Arm ID | Strategy | Notes | +| --- | --- | --- | +| `current-memory-fts-v1` | Memory search (`fts`) | Default; identical to the previous `--search-mode fts` behaviour. | +| `current-memory-hybrid-v1` | Memory search (`hybrid`) | Requires a Server whose `/v1/capabilities` advertises `hybrid`. | +| `query-time-compact-v1` | PreparedContext (8,000 bytes) | Uses public `/v1/context/prepare`; does not alter ingestion or persist a new index schema. | +| `write-time-l0-l1-v1` | Memory search (`fts`) | Writes deterministic bounded L0 index and L1 summary entries through public `remember`; no schema fields are added. | +| `task-lensed-selection-v1` | Memory search (`fts`) | Projects the question into a deterministic keyword lens; never reads question type, gold answer, or Judge data. | + +Unregistered arm IDs (currently the unsupported temporal-filter arm) are rejected +before any work runs instead of silently falling back to FTS. Before ingesting anything, the +runner checks `/v1/capabilities`: a Server without the arm's search mode or PreparedContext +schema fails as a capability error. Every explicitly requested search mode must match the +executed mode the Server reports (only `auto` accepts the Server's own choice) — a mismatch is +recorded as an integrity failure rather than a benchmark result. Every manifest, summary, and +unified report records the full arm block, `adapter-audit.jsonl` records the requested and +actual strategy/mode per query, and `ensure_comparable_experiment_runs` refuses to compare two +runs whose dataset lock, question manifest, harness commit, processor, context budget, search +limit, Reader/Judge configuration, or revisions differ beyond the arm. + +Prepare bounded, replayable Reader inputs without calling a Reader. This command +uses the pinned upstream harness to count and truncate Memory context, and +requires an immutable Hugging Face processor revision rather than resolving the +processor from a moving branch: + +```bash +uv run --project evaluation longmemeval-v2 prepare-smoke \ + --retrieval-dir /path/to/retrieval-smoke-artifacts \ + --harness-root /path/to/LongMemEval-V2 \ + --harness-python /path/to/longmemeval-harness-python \ + --processor-revision PROCESSOR_GIT_SHA \ + --memory-context-max-tokens 200000 \ + --output-dir /path/to/new-prepared-prompt-artifacts +``` + +It writes `prepare-manifest.json`, `prepared-prompts.jsonl`, +`prepare-failures.jsonl`, and `prepare-summary.json`. Each prepared prompt +records the original and bounded Context token counts, the bounded Context +bytes, final system/user messages, a prompt SHA-256, and prepare latency. This +is still a no-model stage: Reader, Judge, and accuracy remain absent. + +Run an Anthropic-compatible Reader or DeepSeek's direct OpenAI-compatible API +over prepared prompts. Credentials are read only from the named process +environment variable and are never written to the run artifacts: + +```bash +export DEEPSEEK_API_KEY=YOUR_KEY +uv run --project evaluation longmemeval-v2 reader-smoke \ + --prepared-dir /path/to/prepared-prompt-artifacts \ + --provider deepseek-openai \ + --model deepseek-flash \ + --token-env DEEPSEEK_API_KEY \ + --max-tokens 512 \ + --temperature 0 \ + --output-dir /path/to/new-reader-artifacts +``` + +Reader and Judge endpoints require HTTPS by default. For a trusted HTTP model +gateway, explicitly pass `--allow-insecure-http` to `reader-smoke`, +`--judge-allow-insecure-http` to `score-smoke`, or the separate +`--reader-allow-insecure-http` and `--judge-allow-insecure-http` flags to +`run-smoke`. HTTP sends prompts and the bearer token without transport encryption. +The opt-in is recorded separately for each model role in the run manifests. +URLs containing credentials, query strings, or fragments are rejected in both +modes, and model requests never follow redirects. + +The OpenAI-compatible `--base-url` includes any gateway prefix such as `/v1`; +the Reader appends `/chat/completions`. For `anthropic-compatible`, supply the +gateway root because the Reader appends `/v1/messages`. Credentials still come +from the environment variable selected by `--token-env` or `--judge-token-env`. + +The Reader writes `reader-manifest.json`, `reader-outputs.jsonl`, +`reader-failures.jsonl`, and `reader-summary.json`. It records provider usage +returned by the API but does not score answers. Keep OpenTelemetry variables +out of this command when prompt telemetry is not approved. + +Score Reader outputs with the pinned deterministic metric functions. The +abstention and gotchas metric types require an LLM Judge and send the question, +reference answer, full Reader response, and parsed answer to the configured +Judge provider: + +```bash +export DEEPSEEK_API_KEY=YOUR_KEY +uv run --project evaluation longmemeval-v2 score-smoke \ + --reader-dir /path/to/reader-artifacts \ + --data-root /path/to/longmemeval-v2-data \ + --dataset-lock evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json \ + --smoke-manifest evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json \ + --harness-root /path/to/LongMemEval-V2 \ + --judge-model deepseek-flash \ + --judge-token-env DEEPSEEK_API_KEY \ + --judge-max-tokens 256 \ + --judge-temperature 0 \ + --output-dir /path/to/new-score-artifacts +``` + +The score run writes `per-question.jsonl`, `judge-outputs.jsonl`, +`score-failures.jsonl`, and `score-summary.json`. It also writes +`scoring-inputs.local.jsonl`, which contains reference answers for local replay +and must remain outside Git, shared reports, and telemetry. + +Replay the saved deterministic and Judge decisions without a Reader, Judge +provider, token, or network request: + +```bash +uv run --project evaluation longmemeval-v2 replay-score \ + --score-dir /path/to/score-artifacts \ + --harness-root /path/to/LongMemEval-V2 \ + --output-dir /path/to/new-score-replay-artifacts +``` + +The replay records source digests and writes `replay-manifest.json`, +`replay-per-question.jsonl`, `replay-failures.jsonl`, and +`replay-summary.json`. It reuses only saved Judge labels for LLM-scored cases. + +### One-command smoke run + +`run-smoke` chains every stage above into one fail-closed run directory. With a +ready PowerContext Server and no model credentials, run the model-free mode +first. The retrieval arm defaults to `current-memory-fts-v1` and can be selected +explicitly with `--experiment-arm`: + +```bash +uv run --project evaluation longmemeval-v2 run-smoke \ + --data-root /path/to/longmemeval-v2-data \ + --dataset-lock evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json \ + --smoke-manifest evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json \ + --harness-root /path/to/LongMemEval-V2 \ + --harness-python /path/to/longmemeval-harness-python \ + --processor-revision PROCESSOR_GIT_SHA \ + --powercontext-revision POWERCONTEXT_GIT_SHA \ + --integration-revision INTEGRATION_GIT_SHA \ + --powercontext-base-url http://127.0.0.1:18765 \ + --experiment-arm current-memory-fts-v1 \ + --skip-reader \ + --output-dir /path/to/new-run-artifacts +``` + +The full mode drops `--skip-reader`, requires `DEEPSEEK_API_KEY` in the environment, and also +runs the Reader, Judge scoring, and score replay. `--skip-score` keeps the Reader but skips +Judge scoring and replay. Every mode writes the same layout: + +```text +/ + run-manifest.json # exclusive-create; inputs, revisions, providers (no secret values) + run-summary.json # completed/skipped/failed phases; accuracy only when scored + failures.jsonl # one row per failed phase, with an error class + report.json # unified machine-readable summary + report.md # human summary with the fixed boundary banner + 01-inputs/ 02-retrieval/ 03-prepare/ 04-reader/ 05-score/ 06-replay/ +``` + +The command exits non-zero unless the run completed: `--skip-reader` and `--skip-score` runs end +as `partial`, and a failed stage ends as `failed` while keeping the earlier stage artifacts. +Recorded failure summaries are redacted against the configured token values, including +`POWERCONTEXT_TOKEN` in model-free mode. Reference answers are read only by the score stage, +which writes the local replay artifact, and by the replay stage reading that artifact; the +adapter, retrieval, prepare, and reader stages never read them. + +### Cost reporting with an explicit price policy + +Token usage is always recorded from the provider's own response. Model cost is reported only when +an explicit price policy is passed; prices are never hardcoded, and an unconfigured cost stays +`null` with a reason instead of `0`: + +```bash +uv run --project evaluation longmemeval-v2 run-smoke \ + ... \ + --price-policy '{"provider":"deepseek-openai","model":"deepseek-flash","currency":"USD","input_cache_hit_price_per_million":0.006,"input_cache_miss_price_per_million":0.3,"output_price_per_million":1.2,"price_policy_revision":"deepseek-public-list-2026-09"}' \ + --judge-price-policy '{"provider":"deepseek-openai","model":"deepseek-flash","currency":"USD","input_cache_hit_price_per_million":0.006,"input_cache_miss_price_per_million":0.3,"output_price_per_million":1.2,"price_policy_revision":"deepseek-public-list-2026-09"}' \ + --output-dir /path/to/new-run-artifacts +``` + +`run-smoke` takes a separate policy per model role because the Reader and the Judge may run +different models; `reader-smoke` takes `--price-policy` and `score-smoke` takes +`--judge-price-policy`. A policy must contain exactly `provider`, `model`, `currency`, +`input_cache_hit_price_per_million`, `input_cache_miss_price_per_million`, +`output_price_per_million`, and `price_policy_revision`. Every field is required, prices must be +finite and non-negative, and `currency` must be `USD` because the recorded amount fields are named +`*_usd`. A policy is applied only when both its `provider` and its `model` match the configured +stage; otherwise the stage records `null` plus the mismatch reason instead of borrowing an +unrelated price. + +Cached and uncached input are priced separately. DeepSeek reports +`prompt_cache_hit_tokens` and `prompt_cache_miss_tokens` alongside `prompt_tokens`, and the +cache-hit rate is roughly fifty times cheaper than the cache-miss rate, so the reader and judge +records keep that split instead of collapsing it into one blended input total. A response that +reports no split cannot be priced at either rate, so its stage records `null` with that reason +rather than an estimate. + +Reader cost comes from the Reader's reported usage, Judge cost from the Judge's reported usage, +and each stage summary and manifest records the policy identity it was priced under. The unified +report adds `usage.reader_cost`, `usage.judge_cost`, `usage.estimated_cost_usd` (the sum, only +when every model stage that actually ran was priced in USD under one policy revision) and +`usage.estimated_cost_note` explaining how the total was computed or why it stays `null`. The +report decides which stages ran from the run manifest `modes` plus the stage summaries, so a +missing, unpriced, mismatched, or differently-revisioned stage cost keeps the total `null` +instead of silently summing the stages that happen to be present. Ingestion is model-free, so the +report records `usage.ingestion_cost` as zero tokens, `cost_usd: 0.0`, and that reason. Without a +policy the report keeps the `null` behavior and explains it. + +Regenerate or inspect a report for any saved run without a model, provider, or server: + +```bash +uv run --project evaluation longmemeval-v2 report \ + --run-dir /path/to/saved-run \ + --output-dir /path/to/new-report-artifacts +``` + +### LongMemEval-V2 full run (not executed) + +A full-tier run has not been executed and is not approved, and its results must never be +confused with the smoke subset. The recorded prerequisites, configuration template, +cost-estimation and approval gates, output isolation, result labels, and failure-recovery +procedure live in [docs/longmemeval-v2-full-run.md](docs/longmemeval-v2-full-run.md); the +secret-free environment template lives in +[config/longmemeval-v2-run-config.example.env](config/longmemeval-v2-run-config.example.env). A +full-tier dataset lock and question manifest do not exist yet and must be created and reviewed +before any full run. + +### What the LongMemEval-V2 workload evaluates + +The smoke workload and any future full run measure one pipeline over fixed LongMemEval-V2 +trajectories: whether the PowerContext Memory adapter retrieves citable evidence through public +interfaces under fixed upstream data, fixed questions, and a fixed context budget; the answer +accuracy, latency, context size, failures, and abstention of the configured Reader and Judge; and +whether saved outputs replay the same deterministic scoring inputs without a model. + +They do not evaluate Handoff, cross-host recovery, normal Runtime persistence, or Work +Continuity. A ten-question smoke subset cannot be extrapolated to full benchmark performance, +general model capability, or product leadership, and it does not replace LoCoMo, SWE-bench Pro, +or real user-task acceptance. Scores must never be improved by rewriting gold prompts, reference +answers, or the Memory schema. Every smoke artifact is labelled `smoke-subset`; none of them is +a complete benchmark result. + +The implemented scope and local validation evidence are summarized in +[docs/longmemeval-v2-smoke-delivery.md](docs/longmemeval-v2-smoke-delivery.md). diff --git a/evaluation/deploy/longmemeval-v2-run-config.example.env b/evaluation/memory/longmemeval_v2/config/longmemeval-v2-run-config.example.env similarity index 92% rename from evaluation/deploy/longmemeval-v2-run-config.example.env rename to evaluation/memory/longmemeval_v2/config/longmemeval-v2-run-config.example.env index 4494edb2a2..1e283ef9b4 100644 --- a/evaluation/deploy/longmemeval-v2-run-config.example.env +++ b/evaluation/memory/longmemeval_v2/config/longmemeval-v2-run-config.example.env @@ -23,4 +23,4 @@ DEEPSEEK_API_KEY=set-outside-repo # --- Optional execution controls ------------------------------------------------ # Provider timeouts, models, and limits are passed as command-line options # (--reader-model, --judge-model, --reader-timeout-seconds, ...); see -# evaluation/docs/longmemeval-v2-full-run.md for the recorded configuration template. +# evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md for the recorded configuration template. diff --git a/evaluation/docs/longmemeval-v2-full-run.md b/evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md similarity index 94% rename from evaluation/docs/longmemeval-v2-full-run.md rename to evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md index 1dcc325f92..62e2e88ebf 100644 --- a/evaluation/docs/longmemeval-v2-full-run.md +++ b/evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md @@ -8,7 +8,7 @@ smoke subset described in [../README.md](../README.md) must never be presented a | Item | State | | --- | --- | -| Full-tier dataset lock | **To be created and reviewed.** `evaluation/locks/` currently contains only the small-tier lock `longmemeval-v2-small-v1.dataset-lock.json`. | +| Full-tier dataset lock | **To be created and reviewed.** `evaluation/memory/longmemeval_v2/locks/` currently contains only the small-tier lock `longmemeval-v2-small-v1.dataset-lock.json`. | | Full-tier question manifest | **To be created and reviewed.** Only the ten-question smoke manifest `longmemeval-v2-small-v1.smoke.json` exists. | | Cost guards (`--max-cases`, `--max-estimated-cost-usd`) | **Not implemented.** Must be implemented and reviewed before any paid Reader/Judge run (see [Estimates and spend limits](#estimates-and-spend-limits)). | | Full run execution | **Not executed and not approved.** Model spend and data egress require separate, explicit user approval. | @@ -50,7 +50,7 @@ rewriting gold prompts, reference answers, or the Memory schema. See ## Configuration template -Copy [longmemeval-v2-run-config.example.env](../deploy/longmemeval-v2-run-config.example.env) to a protected +Copy [longmemeval-v2-run-config.example.env](../config/longmemeval-v2-run-config.example.env) to a protected location outside this repository, fill in the values, and load it into the shell before running. The file contains variable names and placeholders only; never commit a filled copy, and never pass a token as a command-line argument. @@ -58,8 +58,8 @@ a token as a command-line argument. | Setting | Example | Notes | | --- | --- | --- | | Data root | `D:\powercontext-eval\cache\longmemeval-v2-data` | Passed as `--data-root`. | -| Dataset lock | `evaluation\locks\.json` | **Must be created first**; pins dataset revision and file digests. | -| Question manifest | `evaluation\locks\.json` | **Must be created first**; pins the full question set. | +| Dataset lock | `evaluation\memory\longmemeval_v2\locks\.json` | **Must be created first**; pins dataset revision and file digests. | +| Question manifest | `evaluation\memory\longmemeval_v2\locks\.json` | **Must be created first**; pins the full question set. | | Harness root | `D:\powercontext-eval\cache\LongMemEval-V2` | Detached checkout at the pinned commit. | | Harness Python | `D:\powercontext-eval\venvs\longmemeval-v2-2cc8c540\Scripts\python.exe` | Environment with harness dependencies. | | PowerContext base URL | `http://127.0.0.1:18765` | Loopback only; no credentials in the URL. | @@ -75,10 +75,10 @@ Once the full-tier lock and manifest exist and are reviewed, a full run uses the lock exists yet**, so this command must not be run as-is: ```bash -uv run --project evaluation powercontext-eval longmemeval-v2 run-smoke \ +uv run --project evaluation longmemeval-v2 run-smoke \ --data-root D:\powercontext-eval\cache\longmemeval-v2-data \ - --dataset-lock evaluation\locks\.json \ - --smoke-manifest evaluation\locks\.json \ + --dataset-lock evaluation\memory\longmemeval_v2\locks\.json \ + --smoke-manifest evaluation\memory\longmemeval_v2\locks\.json \ --harness-root D:\powercontext-eval\cache\LongMemEval-V2 \ --harness-python D:\powercontext-eval\venvs\longmemeval-v2-2cc8c540\Scripts\python.exe \ --processor-revision \ diff --git a/evaluation/docs/longmemeval-v2-smoke-delivery.md b/evaluation/memory/longmemeval_v2/docs/longmemeval-v2-smoke-delivery.md similarity index 98% rename from evaluation/docs/longmemeval-v2-smoke-delivery.md rename to evaluation/memory/longmemeval_v2/docs/longmemeval-v2-smoke-delivery.md index 37ffe51701..af16289ccb 100644 --- a/evaluation/docs/longmemeval-v2-smoke-delivery.md +++ b/evaluation/memory/longmemeval_v2/docs/longmemeval-v2-smoke-delivery.md @@ -6,7 +6,7 @@ scoring behavior, and labels every result as a smoke subset rather than a comple ## Delivered workflow -The `powercontext-eval longmemeval-v2` command group now supports the complete local workflow: +The `longmemeval-v2` command group now supports the complete local workflow: 1. validate the pinned dataset, fixed ten-question manifest, and upstream harness checkout; 2. ingest trajectory evidence through public PowerContext Source and Memory APIs; diff --git a/evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json b/evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json similarity index 100% rename from evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json rename to evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.dataset-lock.json diff --git a/evaluation/locks/longmemeval-v2-small-v1.smoke.json b/evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json similarity index 100% rename from evaluation/locks/longmemeval-v2-small-v1.smoke.json rename to evaluation/memory/longmemeval_v2/locks/longmemeval-v2-small-v1.smoke.json diff --git a/evaluation/memory/longmemeval_v2/ruff.toml b/evaluation/memory/longmemeval_v2/ruff.toml new file mode 100644 index 0000000000..2c6e969e7e --- /dev/null +++ b/evaluation/memory/longmemeval_v2/ruff.toml @@ -0,0 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +target-version = "py311" +line-length = 120 diff --git a/evaluation/scripts/run_longmemeval_v2_harness.py b/evaluation/memory/longmemeval_v2/scripts/run_longmemeval_v2_harness.py similarity index 80% rename from evaluation/scripts/run_longmemeval_v2_harness.py rename to evaluation/memory/longmemeval_v2/scripts/run_longmemeval_v2_harness.py index 4680b12331..3b1ca4984e 100644 --- a/evaluation/scripts/run_longmemeval_v2_harness.py +++ b/evaluation/memory/longmemeval_v2/scripts/run_longmemeval_v2_harness.py @@ -20,9 +20,9 @@ from importlib import import_module from pathlib import Path -EVALUATION_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(EVALUATION_ROOT / "src")) -main = import_module("powercontext_eval.benchmarks.longmemeval_v2.harness_bootstrap").main +SUITE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SUITE_ROOT / "src")) +main = import_module("powercontext_eval_longmemeval_v2.harness_bootstrap").main if __name__ == "__main__": diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/__init__.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/__init__.py similarity index 100% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/__init__.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/__init__.py diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/adapter.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/adapter.py similarity index 98% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/adapter.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/adapter.py index 4d789842d7..1d549969a8 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/adapter.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/adapter.py @@ -227,7 +227,9 @@ def __init__(self, memory_params: dict[str, object]) -> None: maximum=32_768, ) self.memory_projection = _optional_nonblank( - self.memory_params.get("memory_projection"), "memory_projection", "deterministic-compact-v1" + self.memory_params.get("memory_projection"), + "memory_projection", + "deterministic-compact-v1", ) if self.memory_projection not in MEMORY_PROJECTIONS: raise PowerContextMemoryAdapterError( @@ -237,7 +239,12 @@ def __init__(self, memory_params: dict[str, object]) -> None: if task_lens is not None and task_lens not in TASK_LENSES: raise PowerContextMemoryAdapterError(f"unsupported task_lens: {task_lens!r}") self.task_lens = cast("str | None", task_lens) - self.search_limit = _integer(self.memory_params.get("search_limit", 10), "search_limit", minimum=1, maximum=50) + self.search_limit = _integer( + self.memory_params.get("search_limit", 10), + "search_limit", + minimum=1, + maximum=50, + ) self.source_chunk_bytes = _integer( self.memory_params.get("source_chunk_bytes", DEFAULT_SOURCE_CHUNK_BYTES), "source_chunk_bytes", @@ -255,7 +262,10 @@ def __init__(self, memory_params: dict[str, object]) -> None: def memory_config(self) -> dict[str, object]: """Return only reconstructable configuration, never runtime credentials.""" - return {"memory_type": self.memory_type, "memory_params": dict(self.memory_params)} + return { + "memory_type": self.memory_type, + "memory_params": dict(self.memory_params), + } def configure_runtime(self, **kwargs: object) -> None: """Inject a fake/runtime clock after construction without persisting it.""" @@ -482,7 +492,11 @@ def _http_runtime(self) -> PowerContextHTTPRuntime: return PowerContextHTTPRuntime(base_url, token=os.getenv(token_env), timeout_seconds=timeout) def _write_audit(self, event: dict[str, object]) -> None: - record = {"schema": AUDIT_SCHEMA, "observed_at": datetime.now(UTC).isoformat(), **event} + record = { + "schema": AUDIT_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + **event, + } line = json.dumps(record, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n" with self._audit_lock: self.audit_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/arms.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/arms.py similarity index 98% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/arms.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/arms.py index 791a7867c0..169dc61264 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/arms.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/arms.py @@ -20,10 +20,10 @@ from dataclasses import dataclass from typing import Any -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError -class ExperimentArmError(PowerContextEvalError): +class ExperimentArmError(LongMemEvalError): """An experiment arm lookup or run comparison cannot preserve its contract.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/catalog.py similarity index 96% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/catalog.py index 2e59b14275..ca526396fe 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/catalog.py @@ -25,7 +25,7 @@ from types import MappingProxyType from typing import Literal, TypeAlias, cast -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError UPSTREAM_REPOSITORY = "https://github.com/xiaowu0162/LongMemEval-V2" UPSTREAM_HARNESS_COMMIT = "2cc8c540bdb87fe6761629b585e727e1c4704520" @@ -65,7 +65,7 @@ ) -class LongMemEvalV2CatalogError(PowerContextEvalError): +class LongMemEvalV2CatalogError(LongMemEvalError): """The pinned LongMemEval-V2 inputs cannot be trusted.""" @@ -244,7 +244,10 @@ def load_smoke_manifest(path: Path) -> SmokeSelection: raise LongMemEvalV2InputError("Smoke manifest cases must be an array") cases: list[SmokeCase] = [] for index, raw_case in enumerate(raw_cases): - if not isinstance(raw_case, dict) or set(raw_case) != {"question_id", "ability"}: + if not isinstance(raw_case, dict) or set(raw_case) != { + "question_id", + "ability", + }: raise LongMemEvalV2InputError(f"Smoke manifest case {index} has an invalid shape") question_id = raw_case.get("question_id") ability = raw_case.get("ability") @@ -269,14 +272,23 @@ def load_dataset_lock(path: Path) -> DatasetLock: value = json.loads(content) except json.JSONDecodeError as error: raise LongMemEvalV2InputError("LongMemEval-V2 dataset lock is not valid UTF-8 JSON") from error - if not isinstance(value, dict) or set(value) != {"schema", "upstream", "dataset_revision", "tier", "files"}: + if not isinstance(value, dict) or set(value) != { + "schema", + "upstream", + "dataset_revision", + "tier", + "files", + }: raise LongMemEvalV2InputError( "Dataset lock must contain only schema, upstream, dataset_revision, tier, and files" ) if value["schema"] != DATASET_LOCK_SCHEMA: raise LongMemEvalV2InputError("Dataset lock schema is unsupported") upstream = value["upstream"] - if not isinstance(upstream, dict) or set(upstream) != {"repository", "harness_commit"}: + if not isinstance(upstream, dict) or set(upstream) != { + "repository", + "harness_commit", + }: raise LongMemEvalV2InputError("Dataset lock upstream identity is invalid") if upstream["repository"] != UPSTREAM_REPOSITORY or upstream["harness_commit"] != UPSTREAM_HARNESS_COMMIT: raise LongMemEvalV2InputError("Dataset lock does not match the pinned LongMemEval-V2 harness") @@ -296,7 +308,11 @@ def load_dataset_lock(path: Path) -> DatasetLock: raise LongMemEvalV2InputError("Dataset lock must provide digests for exactly the required files") if any(not isinstance(digest, str) or len(digest) != 64 or not _is_lower_hex(digest) for digest in files.values()): raise LongMemEvalV2InputError("Dataset lock file digests must be lowercase SHA-256 values") - return DatasetLock(tier=tier, dataset_revision=dataset_revision, file_digests=MappingProxyType(dict(files))) + return DatasetLock( + tier=tier, + dataset_revision=dataset_revision, + file_digests=MappingProxyType(dict(files)), + ) def validate_harness_checkout(harness_root: Path) -> None: diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py new file mode 100644 index 0000000000..bab17f8a35 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py @@ -0,0 +1,506 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line interface for the LongMemEval-V2 memory evaluation suite.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer + +from powercontext_eval_longmemeval_v2.adapter import PowerContextMemoryAdapterError +from powercontext_eval_longmemeval_v2.arms import ( + DEFAULT_EXPERIMENT_ARM_ID, + ExperimentArmError, +) +from powercontext_eval_longmemeval_v2.catalog import ( + LongMemEvalV2CatalogError, + LongMemEvalV2EnvironmentError, + LongMemEvalV2InputError, +) +from powercontext_eval_longmemeval_v2.costs import ( + CostPolicyError, + ModelPricePolicy, + parse_cost_policy, +) +from powercontext_eval_longmemeval_v2.prepare_smoke import ( + DEFAULT_PROCESSOR_MODEL, + PrepareSmokeError, + prepare_reader_inputs_smoke, +) +from powercontext_eval_longmemeval_v2.reader_smoke import ( + DEFAULT_ANTHROPIC_BASE_URL_ENV, + ReaderSmokeError, + run_reader_smoke, +) +from powercontext_eval_longmemeval_v2.replay_score import ( + ReplayScoreError, + replay_score_smoke, +) +from powercontext_eval_longmemeval_v2.report import ReportError, build_report +from powercontext_eval_longmemeval_v2.retrieval_smoke import ( + RetrievalSmokeError, + run_retrieval_smoke, +) +from powercontext_eval_longmemeval_v2.run_smoke import RunSmokeError, run_smoke +from powercontext_eval_longmemeval_v2.score_smoke import ( + DEFAULT_DEEPSEEK_BASE_URL as SCORE_DEFAULT_DEEPSEEK_BASE_URL, +) +from powercontext_eval_longmemeval_v2.score_smoke import ( + DEFAULT_DEEPSEEK_MODEL as SCORE_DEFAULT_DEEPSEEK_MODEL, +) +from powercontext_eval_longmemeval_v2.score_smoke import ( + DEFAULT_DEEPSEEK_TOKEN_ENV as SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, +) +from powercontext_eval_longmemeval_v2.score_smoke import ( + ScoreSmokeError, + run_score_smoke, +) +from powercontext_eval_longmemeval_v2.smoke import prepare_smoke_run + +app = typer.Typer(no_args_is_help=True, help="Pinned LongMemEval-V2 evaluation.") + + +def _parse_price_policy_option(value: str | None, *, label: str) -> ModelPricePolicy | None: + """Parse an explicitly passed JSON price policy, or keep costs unconfigured as null.""" + + if value is None: + return None + try: + parsed = json.loads(value) + except json.JSONDecodeError as error: + raise typer.BadParameter(f"{label} must be a JSON object: {error}") from None + try: + return parse_cost_policy(parsed, label=label) + except CostPolicyError as error: + raise typer.BadParameter(str(error)) from None + + +@app.command("smoke") +def longmemeval_v2_smoke( + data_root: Annotated[Path, typer.Option("--data-root")], + dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], + output_dir: Annotated[Path, typer.Option("--output-dir")], +) -> None: + """Validate fixed LongMemEval-V2 inputs and write smoke artifacts without calling a model.""" + + try: + prepared = prepare_smoke_run( + data_root=data_root, + dataset_lock=dataset_lock, + harness_root=harness_root, + smoke_manifest=smoke_manifest, + output_dir=output_dir, + ) + except LongMemEvalV2InputError as error: + raise typer.BadParameter(str(error)) from None + except LongMemEvalV2EnvironmentError as error: + typer.echo(f"LongMemEval-V2 smoke failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset", + "manifest": str(prepared.manifest_path), + "subset": str(prepared.subset_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("retrieval-smoke") +def longmemeval_v2_retrieval_smoke( + data_root: Annotated[Path, typer.Option("--data-root")], + dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + run_id: Annotated[str, typer.Option("--run-id")], + powercontext_revision: Annotated[str, typer.Option("--powercontext-revision")], + integration_revision: Annotated[str, typer.Option("--integration-revision")], + base_url: Annotated[str, typer.Option("--base-url")] = "http://127.0.0.1:8000", + token_env: Annotated[str, typer.Option("--token-env")] = "POWERCONTEXT_TOKEN", + experiment_arm: Annotated[str, typer.Option("--experiment-arm")] = DEFAULT_EXPERIMENT_ARM_ID, + search_limit: Annotated[int, typer.Option("--search-limit", min=1, max=50)] = 10, + timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=0.1)] = 30.0, +) -> None: + """Run the fixed LongMemEval-V2 subset through Memory retrieval without a model.""" + + try: + result = run_retrieval_smoke( + data_root=data_root, + dataset_lock=dataset_lock, + harness_root=harness_root, + smoke_manifest=smoke_manifest, + output_dir=output_dir, + run_id=run_id, + powercontext_revision=powercontext_revision, + integration_revision=integration_revision, + base_url=base_url, + token_env=token_env, + experiment_arm=experiment_arm, + search_limit=search_limit, + timeout_seconds=timeout_seconds, + ) + except ExperimentArmError as error: + raise typer.BadParameter(str(error)) from None + except ( + LongMemEvalV2CatalogError, + RetrievalSmokeError, + PowerContextMemoryAdapterError, + ) as error: + typer.echo(f"LongMemEval-V2 retrieval smoke failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset-retrieval-only", + "manifest": str(result.manifest_path), + "results": str(result.results_path), + "summary": str(result.summary_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("prepare-smoke") +def longmemeval_v2_prepare_smoke( + retrieval_dir: Annotated[Path, typer.Option("--retrieval-dir")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + harness_python: Annotated[Path, typer.Option("--harness-python")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + processor_revision: Annotated[str, typer.Option("--processor-revision")], + processor_model: Annotated[str, typer.Option("--processor-model")] = DEFAULT_PROCESSOR_MODEL, + memory_context_max_tokens: Annotated[int, typer.Option("--memory-context-max-tokens", min=1)] = 200_000, +) -> None: + """Build pinned, bounded Reader inputs from retrieval-only smoke artifacts.""" + + try: + result = prepare_reader_inputs_smoke( + retrieval_dir=retrieval_dir, + harness_root=harness_root, + harness_python=harness_python, + output_dir=output_dir, + processor_model=processor_model, + processor_revision=processor_revision, + memory_context_max_tokens=memory_context_max_tokens, + ) + except PrepareSmokeError as error: + typer.echo(f"LongMemEval-V2 prompt preparation failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset-prepare-only", + "manifest": str(result.manifest_path), + "prompts": str(result.prompts_path), + "summary": str(result.summary_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("reader-smoke") +def longmemeval_v2_reader_smoke( + prepared_dir: Annotated[Path, typer.Option("--prepared-dir")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + provider: Annotated[str, typer.Option("--provider")] = "anthropic-compatible", + model: Annotated[str | None, typer.Option("--model")] = None, + base_url: Annotated[str | None, typer.Option("--base-url")] = None, + base_url_env: Annotated[str, typer.Option("--base-url-env")] = DEFAULT_ANTHROPIC_BASE_URL_ENV, + token_env: Annotated[str | None, typer.Option("--token-env")] = None, + max_tokens: Annotated[int, typer.Option("--max-tokens", min=1)] = 512, + temperature: Annotated[float, typer.Option("--temperature", min=0.0, max=2.0)] = 0.0, + timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=1.0)] = 120.0, + allow_insecure_http: Annotated[ + bool, typer.Option("--allow-insecure-http", help="Allow plaintext HTTP for a trusted Reader endpoint.") + ] = False, + max_questions: Annotated[int | None, typer.Option("--max-questions", min=1)] = None, + price_policy: Annotated[str | None, typer.Option("--price-policy")] = None, +) -> None: + """Call a configured Reader over prepared smoke prompts without scoring.""" + + resolved_price_policy = _parse_price_policy_option(price_policy, label="--price-policy") + try: + result = run_reader_smoke( + prepared_dir=prepared_dir, + output_dir=output_dir, + provider=provider, + model=model, + base_url=base_url, + base_url_env=base_url_env, + token_env=token_env, + max_tokens=max_tokens, + temperature=temperature, + timeout_seconds=timeout_seconds, + allow_insecure_http=allow_insecure_http, + max_questions=max_questions, + price_policy=resolved_price_policy, + ) + except ReaderSmokeError as error: + typer.echo(f"LongMemEval-V2 Reader failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset-reader-only", + "manifest": str(result.manifest_path), + "outputs": str(result.outputs_path), + "summary": str(result.summary_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("score-smoke") +def longmemeval_v2_score_smoke( + reader_dir: Annotated[Path, typer.Option("--reader-dir")], + data_root: Annotated[Path, typer.Option("--data-root")], + dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], + smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + judge_model: Annotated[str, typer.Option("--judge-model")] = SCORE_DEFAULT_DEEPSEEK_MODEL, + judge_token_env: Annotated[str, typer.Option("--judge-token-env")] = SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, + judge_base_url: Annotated[str, typer.Option("--judge-base-url")] = SCORE_DEFAULT_DEEPSEEK_BASE_URL, + judge_max_tokens: Annotated[int, typer.Option("--judge-max-tokens", min=1)] = 256, + judge_temperature: Annotated[float, typer.Option("--judge-temperature", min=0.0, max=2.0)] = 0.0, + judge_timeout_seconds: Annotated[float, typer.Option("--judge-timeout-seconds", min=1.0)] = 120.0, + judge_allow_insecure_http: Annotated[ + bool, typer.Option("--judge-allow-insecure-http", help="Allow plaintext HTTP for a trusted Judge endpoint.") + ] = False, + price_policy: Annotated[str | None, typer.Option("--judge-price-policy")] = None, +) -> None: + """Score Reader smoke outputs with pinned rules and DeepSeek only where upstream requires a judge.""" + + resolved_price_policy = _parse_price_policy_option(price_policy, label="--judge-price-policy") + try: + result = run_score_smoke( + reader_dir=reader_dir, + data_root=data_root, + dataset_lock=dataset_lock, + smoke_manifest=smoke_manifest, + harness_root=harness_root, + output_dir=output_dir, + judge_model=judge_model, + judge_token_env=judge_token_env, + judge_base_url=judge_base_url, + judge_max_tokens=judge_max_tokens, + judge_temperature=judge_temperature, + judge_timeout_seconds=judge_timeout_seconds, + judge_allow_insecure_http=judge_allow_insecure_http, + judge_price_policy=resolved_price_policy, + ) + except (ScoreSmokeError, ReaderSmokeError) as error: + typer.echo(f"LongMemEval-V2 scoring failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset-score-only", + "manifest": str(result.manifest_path), + "results": str(result.results_path), + "summary": str(result.summary_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("replay-score") +def longmemeval_v2_replay_score( + score_dir: Annotated[Path, typer.Option("--score-dir")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + output_dir: Annotated[Path, typer.Option("--output-dir")], +) -> None: + """Replay saved deterministic and Judge score decisions without model calls.""" + + try: + result = replay_score_smoke(score_dir=score_dir, harness_root=harness_root, output_dir=output_dir) + except ReplayScoreError as error: + typer.echo(f"LongMemEval-V2 score replay failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset-score-replay", + "manifest": str(result.manifest_path), + "results": str(result.results_path), + "summary": str(result.summary_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("run-smoke") +def longmemeval_v2_run_smoke( + data_root: Annotated[Path, typer.Option("--data-root")], + dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], + smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], + harness_root: Annotated[Path, typer.Option("--harness-root")], + harness_python: Annotated[Path, typer.Option("--harness-python")], + processor_revision: Annotated[str, typer.Option("--processor-revision")], + output_dir: Annotated[Path, typer.Option("--output-dir")], + powercontext_revision: Annotated[str, typer.Option("--powercontext-revision")], + integration_revision: Annotated[str, typer.Option("--integration-revision")], + run_id: Annotated[str | None, typer.Option("--run-id")] = None, + processor_model: Annotated[str, typer.Option("--processor-model")] = DEFAULT_PROCESSOR_MODEL, + memory_context_max_tokens: Annotated[int, typer.Option("--memory-context-max-tokens", min=1)] = 200_000, + powercontext_base_url: Annotated[str, typer.Option("--powercontext-base-url")] = "http://127.0.0.1:8000", + powercontext_token_env: Annotated[str, typer.Option("--powercontext-token-env")] = "POWERCONTEXT_TOKEN", + experiment_arm: Annotated[str, typer.Option("--experiment-arm")] = DEFAULT_EXPERIMENT_ARM_ID, + search_limit: Annotated[int, typer.Option("--search-limit", min=1, max=50)] = 10, + timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=0.1)] = 30.0, + reader_provider: Annotated[str, typer.Option("--reader-provider")] = "deepseek-openai", + reader_model: Annotated[str | None, typer.Option("--reader-model")] = None, + reader_base_url: Annotated[str | None, typer.Option("--reader-base-url")] = None, + reader_base_url_env: Annotated[str, typer.Option("--reader-base-url-env")] = DEFAULT_ANTHROPIC_BASE_URL_ENV, + reader_token_env: Annotated[str | None, typer.Option("--reader-token-env")] = None, + reader_max_tokens: Annotated[int, typer.Option("--reader-max-tokens", min=1)] = 512, + reader_temperature: Annotated[float, typer.Option("--reader-temperature", min=0.0, max=2.0)] = 0.0, + reader_timeout_seconds: Annotated[float, typer.Option("--reader-timeout-seconds", min=1.0)] = 120.0, + reader_allow_insecure_http: Annotated[ + bool, typer.Option("--reader-allow-insecure-http", help="Allow plaintext HTTP for a trusted Reader endpoint.") + ] = False, + judge_provider: Annotated[str, typer.Option("--judge-provider")] = "deepseek-openai", + judge_model: Annotated[str, typer.Option("--judge-model")] = SCORE_DEFAULT_DEEPSEEK_MODEL, + judge_token_env: Annotated[str, typer.Option("--judge-token-env")] = SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, + judge_base_url: Annotated[str, typer.Option("--judge-base-url")] = SCORE_DEFAULT_DEEPSEEK_BASE_URL, + judge_max_tokens: Annotated[int, typer.Option("--judge-max-tokens", min=1)] = 256, + judge_temperature: Annotated[float, typer.Option("--judge-temperature", min=0.0, max=2.0)] = 0.0, + judge_timeout_seconds: Annotated[float, typer.Option("--judge-timeout-seconds", min=1.0)] = 120.0, + judge_allow_insecure_http: Annotated[ + bool, typer.Option("--judge-allow-insecure-http", help="Allow plaintext HTTP for a trusted Judge endpoint.") + ] = False, + price_policy: Annotated[str | None, typer.Option("--price-policy")] = None, + judge_price_policy: Annotated[str | None, typer.Option("--judge-price-policy")] = None, + skip_reader: Annotated[bool, typer.Option("--skip-reader")] = False, + skip_score: Annotated[bool, typer.Option("--skip-score")] = False, +) -> None: + """Run the whole LongMemEval-V2 smoke workload into one fail-closed run directory.""" + + resolved_price_policy = _parse_price_policy_option(price_policy, label="--price-policy") + resolved_judge_price_policy = _parse_price_policy_option(judge_price_policy, label="--judge-price-policy") + try: + result = run_smoke( + data_root=data_root, + dataset_lock=dataset_lock, + smoke_manifest=smoke_manifest, + harness_root=harness_root, + harness_python=harness_python, + processor_revision=processor_revision, + output_dir=output_dir, + powercontext_revision=powercontext_revision, + integration_revision=integration_revision, + run_id=run_id, + processor_model=processor_model, + memory_context_max_tokens=memory_context_max_tokens, + powercontext_base_url=powercontext_base_url, + powercontext_token_env=powercontext_token_env, + experiment_arm=experiment_arm, + search_limit=search_limit, + timeout_seconds=timeout_seconds, + reader_provider=reader_provider, + reader_model=reader_model, + reader_base_url=reader_base_url, + reader_base_url_env=reader_base_url_env, + reader_token_env=reader_token_env, + reader_max_tokens=reader_max_tokens, + reader_temperature=reader_temperature, + reader_timeout_seconds=reader_timeout_seconds, + reader_allow_insecure_http=reader_allow_insecure_http, + judge_provider=judge_provider, + judge_model=judge_model, + judge_token_env=judge_token_env, + judge_base_url=judge_base_url, + judge_max_tokens=judge_max_tokens, + judge_temperature=judge_temperature, + judge_timeout_seconds=judge_timeout_seconds, + judge_allow_insecure_http=judge_allow_insecure_http, + reader_price_policy=resolved_price_policy, + judge_price_policy=resolved_judge_price_policy, + skip_reader=skip_reader, + skip_score=skip_score, + ) + except (RunSmokeError, LongMemEvalV2CatalogError, ExperimentArmError) as error: + typer.echo(f"LongMemEval-V2 smoke run failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset", + "status": result.status, + "manifest": str(result.manifest_path), + "summary": str(result.summary_path), + "completed_phases": list(result.completed_phases), + "skipped_phases": list(result.skipped_phases), + "failed_phase": result.failed_phase, + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + if result.status != "completed": + raise typer.Exit(code=1) + + +@app.command("report") +def longmemeval_v2_report( + run_dir: Annotated[Path, typer.Option("--run-dir")], + output_dir: Annotated[Path | None, typer.Option("--output-dir")] = None, +) -> None: + """Summarize one saved smoke run into a single unified report without a model or server.""" + + try: + result = build_report(run_dir=run_dir, output_dir=output_dir) + except ReportError as error: + typer.echo(f"LongMemEval-V2 report failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset", + "report": str(result.report_path), + "markdown": str(result.markdown_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +def main() -> None: + """Run the LongMemEval-V2 command-line application.""" + + app() + + +if __name__ == "__main__": + main() diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/costs.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/costs.py similarity index 95% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/costs.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/costs.py index 548562e868..e6793bad1e 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/costs.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/costs.py @@ -33,14 +33,14 @@ from collections.abc import Mapping from dataclasses import dataclass -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError # The amount fields are named ``*_usd`` and are summed across stages, so only a policy that # actually prices United States dollars may produce them. SUPPORTED_CURRENCY = "USD" -class CostPolicyError(PowerContextEvalError): +class CostPolicyError(LongMemEvalError): """An explicitly provided cost policy is missing, malformed, or unusable.""" @@ -117,10 +117,12 @@ def parse_cost_policy(value: object, *, label: str = "cost policy") -> ModelPric model=_nonblank(record["model"], f"{label} model"), currency=SUPPORTED_CURRENCY, input_cache_hit_price_per_million=_price( - record["input_cache_hit_price_per_million"], f"{label} input_cache_hit_price_per_million" + record["input_cache_hit_price_per_million"], + f"{label} input_cache_hit_price_per_million", ), input_cache_miss_price_per_million=_price( - record["input_cache_miss_price_per_million"], f"{label} input_cache_miss_price_per_million" + record["input_cache_miss_price_per_million"], + f"{label} input_cache_miss_price_per_million", ), output_price_per_million=_price(record["output_price_per_million"], f"{label} output_price_per_million"), price_policy_revision=_nonblank(record["price_policy_revision"], f"{label} price_policy_revision"), @@ -205,7 +207,12 @@ def usage_cost_block( assert policy is not None and hit is not None and miss is not None # narrowed by _unavailable_reason return { **usage, - "cost_usd": usage_cost_usd(policy, cache_hit_tokens=hit, cache_miss_tokens=miss, output_tokens=output_tokens), + "cost_usd": usage_cost_usd( + policy, + cache_hit_tokens=hit, + cache_miss_tokens=miss, + output_tokens=output_tokens, + ), "currency": policy.currency, "input_cache_hit_price_per_million": policy.input_cache_hit_price_per_million, "input_cache_miss_price_per_million": policy.input_cache_miss_price_per_million, diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/errors.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/errors.py new file mode 100644 index 0000000000..0998c6a304 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/errors.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Error base for LongMemEval-V2 evaluation failures.""" + + +class LongMemEvalError(Exception): + """Base error for LongMemEval-V2 evaluation failures.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/harness_bootstrap.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/harness_bootstrap.py similarity index 95% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/harness_bootstrap.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/harness_bootstrap.py index 828117f72f..79a33e205f 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/harness_bootstrap.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/harness_bootstrap.py @@ -26,8 +26,8 @@ from types import ModuleType from typing import Any -from powercontext_eval.benchmarks.longmemeval_v2.adapter import PowerContextMemory -from powercontext_eval.benchmarks.longmemeval_v2.catalog import validate_harness_checkout +from powercontext_eval_longmemeval_v2.adapter import PowerContextMemory +from powercontext_eval_longmemeval_v2.catalog import validate_harness_checkout class LongMemEvalV2HarnessBootstrapError(RuntimeError): diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_smoke.py similarity index 93% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_smoke.py index bbf90d8d53..deaa8480f0 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_smoke.py @@ -23,8 +23,12 @@ from dataclasses import dataclass from pathlib import Path -from powercontext_eval.benchmarks.longmemeval_v2.catalog import UPSTREAM_HARNESS_COMMIT, validate_harness_checkout -from powercontext_eval.errors import PowerContextEvalError +import powercontext_eval_longmemeval_v2 +from powercontext_eval_longmemeval_v2.catalog import ( + UPSTREAM_HARNESS_COMMIT, + validate_harness_checkout, +) +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError DEFAULT_PROCESSOR_MODEL = "Qwen/Qwen3.5-9B" PREPARE_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-prepare-run.v1" @@ -33,7 +37,7 @@ PREPARE_SUMMARY_SCHEMA = "powercontext.longmemeval-v2-prepare-summary.v1" -class PrepareSmokeError(PowerContextEvalError): +class PrepareSmokeError(LongMemEvalError): """The retrieval artifacts or pinned harness cannot produce Reader inputs.""" @@ -119,7 +123,7 @@ def prepare_reader_inputs_smoke( command = [ str(harness_python), "-m", - "powercontext_eval.benchmarks.longmemeval_v2.prepare_worker", + "powercontext_eval_longmemeval_v2.prepare_worker", "--harness-root", str(harness_root), "--input-path", @@ -138,8 +142,11 @@ def prepare_reader_inputs_smoke( str(memory_context_max_tokens), ] environment = dict(os.environ) - source_root = Path(__file__).parents[3] - environment["PYTHONPATH"] = _prepend_path(source_root, environment.get("PYTHONPATH")) + # The harness interpreter may not have this evaluation suite installed. + python_path = [str(Path(powercontext_eval_longmemeval_v2.__file__).resolve().parent.parent)] + if current_python_path := environment.get("PYTHONPATH"): + python_path.append(current_python_path) + environment["PYTHONPATH"] = os.pathsep.join(python_path) environment["PYTHONNOUSERSITE"] = "1" try: completed = subprocess.run( @@ -217,10 +224,6 @@ def _file_digest(path: Path) -> str: return hasher.hexdigest() -def _prepend_path(path: Path, current: str | None) -> str: - return str(path) if not current else f"{path}{os.pathsep}{current}" - - def _write_json_exclusive(path: Path, value: object) -> None: _write_text_exclusive(path, json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n") diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_worker.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_worker.py similarity index 99% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_worker.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_worker.py index 3da4b6ee0a..f9b81f8ec3 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/prepare_worker.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/prepare_worker.py @@ -27,7 +27,7 @@ from pathlib import Path from typing import Any, Protocol, cast -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import ( +from powercontext_eval_longmemeval_v2.prepare_smoke import ( PREPARE_FAILURE_SCHEMA, PREPARE_SUMMARY_SCHEMA, PREPARED_PROMPT_SCHEMA, diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/reader_smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/reader_smoke.py similarity index 91% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/reader_smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/reader_smoke.py index c513b49f30..e79c15d2c4 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/reader_smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/reader_smoke.py @@ -27,15 +27,15 @@ from typing import Any, Protocol from urllib.error import HTTPError, URLError from urllib.parse import urlsplit -from urllib.request import Request, urlopen +from urllib.request import HTTPRedirectHandler, Request, build_opener -from powercontext_eval.benchmarks.longmemeval_v2.costs import ( +from powercontext_eval_longmemeval_v2.costs import ( ModelPricePolicy, UsageAccount, cost_policy_record, usage_cost_block, ) -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError DEFAULT_ANTHROPIC_BASE_URL_ENV = "ANTHROPIC_BASE_URL" DEFAULT_ANTHROPIC_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN" @@ -49,7 +49,7 @@ READER_SUMMARY_SCHEMA = "powercontext.longmemeval-v2-reader-summary.v1" -class ReaderSmokeError(PowerContextEvalError): +class ReaderSmokeError(LongMemEvalError): """Prepared prompts cannot be sent safely to the configured Reader.""" @@ -79,6 +79,31 @@ class ReaderSmokeRun: summary_path: Path +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None: + # Redirects can forward the bearer token to another origin or downgrade HTTPS. + return None + + +def _validate_reader_base_url(base_url: str, *, allow_insecure_http: bool) -> None: + allowed_schemes = {"https", "http"} if allow_insecure_http else {"https"} + try: + parsed = urlsplit(base_url) + valid = ( + parsed.scheme in allowed_schemes + and bool(parsed.hostname) + and parsed.username is None + and parsed.password is None + and "?" not in base_url + and "#" not in base_url + ) + except ValueError: + valid = False + if not valid: + scheme_label = "HTTP or HTTPS" if allow_insecure_http else "HTTPS" + raise ReaderSmokeError(f"Reader base URL must be an {scheme_label} URL without credentials, query, or fragment") + + class AnthropicCompatibleReader: """Use bearer authentication without persisting the token or response headers.""" @@ -91,17 +116,9 @@ def __init__( max_tokens: int, temperature: float, timeout_seconds: float, + allow_insecure_http: bool = False, ) -> None: - parsed = urlsplit(base_url) - if ( - parsed.scheme != "https" - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.query - or parsed.fragment - ): - raise ReaderSmokeError("Reader base URL must be an HTTPS URL without credentials") + _validate_reader_base_url(base_url, allow_insecure_http=allow_insecure_http) if not token.strip(): raise ReaderSmokeError("Reader token is empty") if not model.strip(): @@ -118,6 +135,7 @@ def __init__( self._max_tokens = max_tokens self._temperature = temperature self._timeout_seconds = timeout_seconds + self._opener = build_opener(_RejectRedirects()) def complete(self, *, system: str, content: list[dict[str, object]]) -> Mapping[str, object]: request = Request( @@ -141,7 +159,7 @@ def complete(self, *, system: str, content: list[dict[str, object]]) -> Mapping[ method="POST", ) try: - with urlopen(request, timeout=self._timeout_seconds) as response: + with self._opener.open(request, timeout=self._timeout_seconds) as response: body = response.read() except HTTPError as error: raise ReaderSmokeError(f"Reader request returned HTTP {error.code}") from error @@ -168,17 +186,9 @@ def __init__( max_tokens: int, temperature: float, timeout_seconds: float, + allow_insecure_http: bool = False, ) -> None: - parsed = urlsplit(base_url) - if ( - parsed.scheme != "https" - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.query - or parsed.fragment - ): - raise ReaderSmokeError("Reader base URL must be an HTTPS URL without credentials") + _validate_reader_base_url(base_url, allow_insecure_http=allow_insecure_http) if not token.strip(): raise ReaderSmokeError("Reader token is empty") if not model.strip(): @@ -195,6 +205,7 @@ def __init__( self._max_tokens = max_tokens self._temperature = temperature self._timeout_seconds = timeout_seconds + self._opener = build_opener(_RejectRedirects()) def complete(self, *, system: str, content: list[dict[str, object]]) -> Mapping[str, object]: request = Request( @@ -213,11 +224,14 @@ def complete(self, *, system: str, content: list[dict[str, object]]) -> Mapping[ ensure_ascii=False, separators=(",", ":"), ).encode(), - headers={"Authorization": f"Bearer {self._token}", "Content-Type": "application/json"}, + headers={ + "Authorization": f"Bearer {self._token}", + "Content-Type": "application/json", + }, method="POST", ) try: - with urlopen(request, timeout=self._timeout_seconds) as response: + with self._opener.open(request, timeout=self._timeout_seconds) as response: body = response.read() except HTTPError as error: raise ReaderSmokeError(f"Reader request returned HTTP {error.code}") from error @@ -242,6 +256,7 @@ def run_reader_smoke( max_tokens: int = 512, temperature: float = 0.0, timeout_seconds: float = 120.0, + allow_insecure_http: bool = False, max_questions: int | None = None, price_policy: ModelPricePolicy | None = None, transport: ReaderTransport | None = None, @@ -253,7 +268,8 @@ def run_reader_smoke( if provider not in {"anthropic-compatible", "deepseek-openai"}: raise ReaderSmokeError("provider must be anthropic-compatible or deepseek-openai") normalized_model = _nonblank( - model or (DEFAULT_DEEPSEEK_MODEL if provider == "deepseek-openai" else DEFAULT_READER_MODEL), "model" + model or (DEFAULT_DEEPSEEK_MODEL if provider == "deepseek-openai" else DEFAULT_READER_MODEL), + "model", ) if max_questions is not None and (isinstance(max_questions, bool) or max_questions <= 0): raise ReaderSmokeError("max_questions must be null or positive") @@ -281,6 +297,7 @@ def run_reader_smoke( max_tokens=max_tokens, temperature=temperature, timeout_seconds=timeout_seconds, + allow_insecure_http=allow_insecure_http, ) else: active_transport = AnthropicCompatibleReader( @@ -290,6 +307,7 @@ def run_reader_smoke( max_tokens=max_tokens, temperature=temperature, timeout_seconds=timeout_seconds, + allow_insecure_http=allow_insecure_http, ) else: resolved_token_env = token_env or ( @@ -328,6 +346,7 @@ def run_reader_smoke( "max_tokens": max_tokens, "temperature": temperature, "timeout_seconds": timeout_seconds, + "allow_insecure_http": allow_insecure_http, }, "judge": None, "telemetry": "disabled-by-reader-runner", @@ -534,7 +553,10 @@ def _reader_usage_totals( ) -> dict[str, object]: """Report summed usage, keeping the cache split only when every response reported it.""" - totals: dict[str, object] = {"input_tokens": input_tokens, "output_tokens": output_tokens} + totals: dict[str, object] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } if cache_split_reported: totals["input_cache_hit_tokens"] = cache_hit_tokens totals["input_cache_miss_tokens"] = cache_miss_tokens @@ -588,7 +610,10 @@ def _deepseek_usage(usage: Mapping[Any, Any]) -> dict[str, object]: miss = _optional_nonnegative_int(usage.get("prompt_cache_miss_tokens")) total = _nonnegative_int(usage.get("prompt_tokens")) if hit is None or miss is None: - return {"input_tokens": total, "output_tokens": _nonnegative_int(usage.get("completion_tokens"))} + return { + "input_tokens": total, + "output_tokens": _nonnegative_int(usage.get("completion_tokens")), + } return { "input_tokens": total, "input_cache_hit_tokens": hit, diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/replay_score.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/replay_score.py similarity index 97% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/replay_score.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/replay_score.py index ed5bbfe121..d25f1ffee5 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/replay_score.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/replay_score.py @@ -27,16 +27,19 @@ from pathlib import Path from typing import Any, Protocol, cast -from powercontext_eval.benchmarks.longmemeval_v2.catalog import validate_harness_checkout -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import LLM_JUDGE_NAMES, PER_QUESTION_SCHEMA -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.catalog import validate_harness_checkout +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError +from powercontext_eval_longmemeval_v2.score_smoke import ( + LLM_JUDGE_NAMES, + PER_QUESTION_SCHEMA, +) REPLAY_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-score-replay.v1" REPLAY_FAILURE_SCHEMA = "powercontext.longmemeval-v2-score-replay-failure.v1" REPLAY_SUMMARY_SCHEMA = "powercontext.longmemeval-v2-score-replay-summary.v1" -class ReplayScoreError(PowerContextEvalError): +class ReplayScoreError(LongMemEvalError): """Saved scoring evidence cannot reproduce one deterministic score result.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/report.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/report.py similarity index 94% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/report.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/report.py index 6013b0a502..eef634ac96 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/report.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/report.py @@ -24,18 +24,25 @@ from pathlib import Path from typing import TypeAlias, TypeGuard, cast -from powercontext_eval.benchmarks.longmemeval_v2.costs import model_free_cost_block -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.costs import model_free_cost_block +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError REPORT_SCHEMA = "powercontext.longmemeval-v2-smoke-report.v1" REPORT_BANNER = "LongMemEval-V2 smoke subset — not a complete benchmark result." REPORT_BOUNDARY = ( "This report describes one fixed smoke subset over pinned upstream trajectories. " "It is not a complete benchmark result, not a product reliability claim, and not a substitute " - "for a full LongMemEval-V2 run. See evaluation/README.md and " - "evaluation/docs/longmemeval-v2-full-run.md for the recorded boundaries and the unexecuted full run." + "for a full LongMemEval-V2 run. See evaluation/memory/longmemeval_v2/README.md and " + "evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md for the recorded boundaries and the unexecuted full run." +) +ERROR_CLASSES = ( + "configuration", + "infrastructure", + "retrieval", + "generation", + "judge", + "integrity", ) -ERROR_CLASSES = ("configuration", "infrastructure", "retrieval", "generation", "judge", "integrity") _PHASES = ("preflight", "retrieval", "prepare", "reader", "score", "replay") _FAILURE_CLASS_BY_PHASE_FILE = { "prepare": "infrastructure", @@ -47,7 +54,7 @@ ReportStatus: TypeAlias = str -class ReportError(PowerContextEvalError): +class ReportError(LongMemEvalError): """Saved run artifacts cannot produce one unified report.""" @@ -62,7 +69,7 @@ class ReportRun: def build_report(*, run_dir: Path, output_dir: Path | None = None) -> ReportRun: """Read saved stage artifacts and write ``report.json`` and ``report.md`` fail-closed.""" - from powercontext_eval.benchmarks.longmemeval_v2.run_smoke import PHASE_DIRECTORIES + from powercontext_eval_longmemeval_v2.run_smoke import PHASE_DIRECTORIES run_root = run_dir.resolve() if not run_root.is_dir(): @@ -120,7 +127,12 @@ def _build(run_root: Path, directories: Mapping[str, str]) -> dict[str, object]: "experiment_arm": _experiment_arm(manifest, retrieval_manifest), "generated_at": datetime.now(UTC).isoformat(), "question_count": _question_count( - summary, score_summary, replay_summary, reader_summary, prepare_summary, retrieval_summary + summary, + score_summary, + replay_summary, + reader_summary, + prepare_summary, + retrieval_summary, ), "accuracy": _accuracy(score_summary, replay_summary), "latency_ms": { @@ -149,7 +161,11 @@ def _status( score_summary: dict[str, object] | None, artifacts: Mapping[str, object], ) -> ReportStatus: - if summary is not None and summary.get("status") in {"completed", "partial", "failed"}: + if summary is not None and summary.get("status") in { + "completed", + "partial", + "failed", + }: return str(summary["status"]) if score_summary is not None and score_summary.get("failed") == 0: return "completed" @@ -318,7 +334,12 @@ def _model_stage_state( def _nonnull_usage(cost: Mapping[str, object]) -> bool: """Report whether a cost block carries any non-zero token count.""" - for key in ("input_tokens", "input_cache_hit_tokens", "input_cache_miss_tokens", "output_tokens"): + for key in ( + "input_tokens", + "input_cache_hit_tokens", + "input_cache_miss_tokens", + "output_tokens", + ): value = cost.get(key) if isinstance(value, int) and not isinstance(value, bool) and value > 0: return True @@ -463,16 +484,28 @@ def _total_cost( if cost is not None and _cost_amount(cost) is not None ] if not costs: - return None, "no Reader or Judge model stage ran in this smoke run, so no model cost is reported" + return ( + None, + "no Reader or Judge model stage ran in this smoke run, so no model cost is reported", + ) currencies = {str(cost.get("currency")) for cost in costs} if len(currencies) != 1: - return None, "the recorded stage costs use different currencies, so no single total is reported" + return ( + None, + "the recorded stage costs use different currencies, so no single total is reported", + ) revisions = {str(cost.get("price_policy_revision")) for cost in costs} if len(revisions) != 1: - return None, "the recorded stage costs were priced under different price policy revisions" + return ( + None, + "the recorded stage costs were priced under different price policy revisions", + ) amounts = [_cost_amount(cost) for cost in costs] return ( - {"cost_usd": round(sum(amount for amount in amounts if amount is not None), 6), "currency": currencies.pop()}, + { + "cost_usd": round(sum(amount for amount in amounts if amount is not None), 6), + "currency": currencies.pop(), + }, "sum of the Reader and Judge usage costs recorded under the run's explicit price policy", ) diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/retrieval_smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/retrieval_smoke.py similarity index 98% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/retrieval_smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/retrieval_smoke.py index 9b9d356a32..da443d20ff 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/retrieval_smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/retrieval_smoke.py @@ -27,7 +27,7 @@ from pathlib import Path from typing import Literal, Protocol, cast -from powercontext_eval.benchmarks.longmemeval_v2.adapter import ( +from powercontext_eval_longmemeval_v2.adapter import ( DEFAULT_SOURCE_CHUNK_BYTES, MAX_MEMORY_TEXT_BYTES, PowerContextHTTPRuntime, @@ -36,19 +36,19 @@ PowerContextMemoryModeError, PowerContextRuntime, ) -from powercontext_eval.benchmarks.longmemeval_v2.arms import ( +from powercontext_eval_longmemeval_v2.arms import ( ExperimentArm, arm_manifest_record, resolve_experiment_arm, ) -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( +from powercontext_eval_longmemeval_v2.catalog import ( Ability, SmokeSelection, load_dataset_lock, load_smoke_manifest, ) -from powercontext_eval.benchmarks.longmemeval_v2.smoke import prepare_smoke_run -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError +from powercontext_eval_longmemeval_v2.smoke import prepare_smoke_run RETRIEVAL_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-retrieval-run.v1" RETRIEVAL_RESULT_SCHEMA = "powercontext.longmemeval-v2-retrieval-result.v1" @@ -56,7 +56,7 @@ RETRIEVAL_SUMMARY_SCHEMA = "powercontext.longmemeval-v2-retrieval-summary.v1" -class RetrievalSmokeError(PowerContextEvalError): +class RetrievalSmokeError(LongMemEvalError): """A retrieval-only smoke run could not preserve its evaluation contract.""" @@ -641,7 +641,9 @@ def _failure( } -def _failure_category(error: Exception) -> Literal["infrastructure", "integration", "integrity"]: +def _failure_category( + error: Exception, +) -> Literal["infrastructure", "integration", "integrity"]: if isinstance(error, PowerContextMemoryModeError): return "integrity" if isinstance(error, PowerContextMemoryAdapterError): diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/run_smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/run_smoke.py similarity index 91% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/run_smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/run_smoke.py index ea110fdbbf..4197890b47 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/run_smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/run_smoke.py @@ -27,32 +27,33 @@ from typing import Literal, TypeAlias from urllib.parse import urlsplit, urlunsplit -from powercontext_eval.benchmarks.longmemeval_v2.adapter import ( +from powercontext_eval_longmemeval_v2.adapter import ( PowerContextMemoryAdapterError, PowerContextMemoryModeError, ) -from powercontext_eval.benchmarks.longmemeval_v2.arms import ( +from powercontext_eval_longmemeval_v2.arms import ( ExperimentArm, arm_manifest_record, resolve_experiment_arm, ) -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( +from powercontext_eval_longmemeval_v2.catalog import ( UPSTREAM_HARNESS_COMMIT, LongMemEvalV2CatalogError, LongMemEvalV2EnvironmentError, LongMemEvalV2InputError, ) -from powercontext_eval.benchmarks.longmemeval_v2.costs import ( +from powercontext_eval_longmemeval_v2.costs import ( ModelPricePolicy, cost_policy_record, ) -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import ( +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError +from powercontext_eval_longmemeval_v2.prepare_smoke import ( DEFAULT_PROCESSOR_MODEL, PreparedPromptRun, PrepareSmokeError, prepare_reader_inputs_smoke, ) -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ( +from powercontext_eval_longmemeval_v2.reader_smoke import ( DEFAULT_ANTHROPIC_BASE_URL_ENV, DEFAULT_ANTHROPIC_TOKEN_ENV, DEFAULT_DEEPSEEK_BASE_URL, @@ -64,22 +65,25 @@ ReaderTransport, run_reader_smoke, ) -from powercontext_eval.benchmarks.longmemeval_v2.replay_score import ( +from powercontext_eval_longmemeval_v2.replay_score import ( ReplayScoreError, ReplayScoreRun, replay_score_smoke, ) -from powercontext_eval.benchmarks.longmemeval_v2.report import build_report -from powercontext_eval.benchmarks.longmemeval_v2.retrieval_smoke import ( +from powercontext_eval_longmemeval_v2.report import build_report +from powercontext_eval_longmemeval_v2.retrieval_smoke import ( RetrievalCapabilityError, RetrievalSmokeError, RetrievalSmokeRun, RetrievalSmokeRuntime, run_retrieval_smoke, ) -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ScoreSmokeError, ScoreSmokeRun, run_score_smoke -from powercontext_eval.benchmarks.longmemeval_v2.smoke import PreparedSmokeRun, prepare_smoke_run -from powercontext_eval.errors import PowerContextEvalError +from powercontext_eval_longmemeval_v2.score_smoke import ( + ScoreSmokeError, + ScoreSmokeRun, + run_score_smoke, +) +from powercontext_eval_longmemeval_v2.smoke import PreparedSmokeRun, prepare_smoke_run RUN_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-smoke-run.v1" RUN_SUMMARY_SCHEMA = "powercontext.longmemeval-v2-smoke-run-summary.v1" @@ -89,7 +93,14 @@ ErrorClass: TypeAlias = Literal["configuration", "infrastructure", "retrieval", "generation", "judge", "integrity"] RunStatus: TypeAlias = Literal["completed", "partial", "failed"] -PHASES: tuple[Phase, ...] = ("preflight", "retrieval", "prepare", "reader", "score", "replay") +PHASES: tuple[Phase, ...] = ( + "preflight", + "retrieval", + "prepare", + "reader", + "score", + "replay", +) PHASE_DIRECTORIES: Mapping[Phase, str] = { "preflight": "01-inputs", "retrieval": "02-retrieval", @@ -100,7 +111,7 @@ } -class RunSmokeError(PowerContextEvalError): +class RunSmokeError(LongMemEvalError): """The one-command smoke run cannot preserve its evaluation contract.""" @@ -187,6 +198,7 @@ def run_smoke( reader_max_tokens: int = 512, reader_temperature: float = 0.0, reader_timeout_seconds: float = 120.0, + reader_allow_insecure_http: bool = False, judge_provider: str = "deepseek-openai", judge_model: str = DEFAULT_DEEPSEEK_MODEL, judge_token_env: str = DEFAULT_DEEPSEEK_TOKEN_ENV, @@ -194,6 +206,7 @@ def run_smoke( judge_max_tokens: int = 256, judge_temperature: float = 0.0, judge_timeout_seconds: float = 120.0, + judge_allow_insecure_http: bool = False, reader_price_policy: ModelPricePolicy | None = None, judge_price_policy: ModelPricePolicy | None = None, skip_reader: bool = False, @@ -278,8 +291,14 @@ def run_smoke( "experiment_arm": arm_manifest_record(arm), "inputs": { "data_root": str(data_root.resolve()), - "dataset_lock": {"path": str(dataset_lock.resolve()), "content_sha256": lock_digest}, - "smoke_manifest": {"path": str(smoke_manifest.resolve()), "content_sha256": manifest_digest}, + "dataset_lock": { + "path": str(dataset_lock.resolve()), + "content_sha256": lock_digest, + }, + "smoke_manifest": { + "path": str(smoke_manifest.resolve()), + "content_sha256": manifest_digest, + }, "harness": { "root": str(harness_root.resolve()), "commit": UPSTREAM_HARNESS_COMMIT, @@ -305,6 +324,7 @@ def run_smoke( "max_tokens": reader_max_tokens, "temperature": reader_temperature, "timeout_seconds": reader_timeout_seconds, + "allow_insecure_http": reader_allow_insecure_http, }, "judge": None if skip_score @@ -316,8 +336,12 @@ def run_smoke( "max_tokens": judge_max_tokens, "temperature": judge_temperature, "timeout_seconds": judge_timeout_seconds, + "allow_insecure_http": judge_allow_insecure_http, + }, + "revisions": { + "powercontext": powercontext_ref, + "integration": integration_ref, }, - "revisions": {"powercontext": powercontext_ref, "integration": integration_ref}, "cost_policy": { "reader": cost_policy_record(reader_price_policy), "judge": cost_policy_record(judge_price_policy), @@ -402,7 +426,10 @@ def preflight() -> None: transport=reader_transport, ) _require_token( - enabled=not skip_score, token_env=resolved_judge_token_env, label="Judge", transport=judge_transport + enabled=not skip_score, + token_env=resolved_judge_token_env, + label="Judge", + transport=judge_transport, ) active.preflight( data_root=data_root, @@ -462,6 +489,7 @@ def preflight() -> None: max_tokens=reader_max_tokens, temperature=reader_temperature, timeout_seconds=reader_timeout_seconds, + allow_insecure_http=reader_allow_insecure_http, price_policy=reader_price_policy, transport=reader_transport, ), @@ -484,6 +512,7 @@ def preflight() -> None: judge_max_tokens=judge_max_tokens, judge_temperature=judge_temperature, judge_timeout_seconds=judge_timeout_seconds, + judge_allow_insecure_http=judge_allow_insecure_http, judge_price_policy=judge_price_policy, judge_transport=judge_transport, ), @@ -545,7 +574,11 @@ def _in_progress_status(completed: list[Phase], skip_reader: bool, skip_score: b def _artifacts(output_dir: Path) -> dict[str, str | None]: - names = {"run_manifest": "run-manifest.json", "run_summary": "run-summary.json", "failures": "failures.jsonl"} + names = { + "run_manifest": "run-manifest.json", + "run_summary": "run-summary.json", + "failures": "failures.jsonl", + } names.update({phase: PHASE_DIRECTORIES[phase] for phase in PHASES}) return {name: value if (output_dir / value).exists() else None for name, value in names.items()} @@ -610,7 +643,12 @@ def _require_reader_configuration( if not enabled or transport is not None: return if provider == "anthropic-compatible" and base_url is None: - _require_token(enabled=True, token_env=base_url_env, label="Reader base URL", transport=None) + _require_token( + enabled=True, + token_env=base_url_env, + label="Reader base URL", + transport=None, + ) _require_token(enabled=True, token_env=token_env, label="Reader", transport=None) diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/score_smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/score_smoke.py similarity index 96% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/score_smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/score_smoke.py index 8fcefb5808..2f8ca94449 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/score_smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/score_smoke.py @@ -28,20 +28,21 @@ from pathlib import Path from typing import Any, Protocol, cast -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( +from powercontext_eval_longmemeval_v2.catalog import ( LongMemEvalV2Catalog, SmokeSelection, load_dataset_lock, load_smoke_manifest, validate_harness_checkout, ) -from powercontext_eval.benchmarks.longmemeval_v2.costs import ( +from powercontext_eval_longmemeval_v2.costs import ( ModelPricePolicy, UsageAccount, cost_policy_record, usage_cost_block, ) -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ( +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError +from powercontext_eval_longmemeval_v2.reader_smoke import ( DEFAULT_DEEPSEEK_BASE_URL, DEFAULT_DEEPSEEK_MODEL, DEFAULT_DEEPSEEK_TOKEN_ENV, @@ -49,7 +50,6 @@ ReaderResponseError, ReaderTransport, ) -from powercontext_eval.errors import PowerContextEvalError SCORE_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-score-run.v1" SCORE_INPUT_SCHEMA = "powercontext.longmemeval-v2-score-input.v1" @@ -60,7 +60,7 @@ LLM_JUDGE_NAMES = {"llm_abstention_checker", "llm_gotchas_checker"} -class ScoreSmokeError(PowerContextEvalError): +class ScoreSmokeError(LongMemEvalError): """Reader outputs cannot be scored under the pinned LongMemEval-V2 contract.""" @@ -114,6 +114,7 @@ def run_score_smoke( judge_max_tokens: int = 256, judge_temperature: float = 0.0, judge_timeout_seconds: float = 120.0, + judge_allow_insecure_http: bool = False, judge_price_policy: ModelPricePolicy | None = None, judge_transport: ReaderTransport | None = None, ) -> ScoreSmokeRun: @@ -150,6 +151,7 @@ def run_score_smoke( max_tokens=judge_max_tokens, temperature=judge_temperature, timeout_seconds=judge_timeout_seconds, + allow_insecure_http=judge_allow_insecure_http, ) else: transport = judge_transport @@ -186,6 +188,7 @@ def run_score_smoke( "base_url": "configured-directly", "max_tokens": judge_max_tokens, "temperature": judge_temperature, + "allow_insecure_http": judge_allow_insecure_http, }, "cost_policy": cost_policy_record(judge_price_policy), }, @@ -281,7 +284,13 @@ def run_score_smoke( if failed: raise ScoreSmokeError(f"Scoring failed for {failed} question(s)") return ScoreSmokeRun( - output_dir, manifest_path, inputs_path, results_path, judge_outputs_path, failures_path, summary_path + output_dir, + manifest_path, + inputs_path, + results_path, + judge_outputs_path, + failures_path, + summary_path, ) @@ -312,7 +321,10 @@ def _judge_usage_totals( ) -> dict[str, object]: """Report summed Judge usage, keeping the cache split only when every call reported it.""" - totals: dict[str, object] = {"input_tokens": input_tokens, "output_tokens": output_tokens} + totals: dict[str, object] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } if cache_split_reported: totals["input_cache_hit_tokens"] = cache_hit_tokens totals["input_cache_miss_tokens"] = cache_miss_tokens diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/smoke.py similarity index 95% rename from evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py rename to evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/smoke.py index e3601e3c96..09b1fa67b0 100644 --- a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/smoke.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from pathlib import Path -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( +from powercontext_eval_longmemeval_v2.catalog import ( RUN_INPUT_MANIFEST_SCHEMA, UPSTREAM_HARNESS_COMMIT, UPSTREAM_REPOSITORY, @@ -99,7 +99,10 @@ def prepare_smoke_run( def _write_json(path: Path, value: object) -> None: try: - path.write_text(json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", encoding="utf-8") + path.write_text( + json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) except OSError as error: raise LongMemEvalV2EnvironmentError(f"Cannot write smoke artifact: {path}") from error diff --git a/evaluation/memory/longmemeval_v2/tests/conftest.py b/evaluation/memory/longmemeval_v2/tests/conftest.py new file mode 100644 index 0000000000..084bc1f7b2 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/tests/conftest.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from types import SimpleNamespace + +import pytest + + +@contextmanager +def _model_server() -> Iterator[SimpleNamespace]: + state = SimpleNamespace(calls=[], redirect_url=None, redirect_status=302, text="answer") + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + state.calls.append({"path": self.path, "authorization": self.headers.get("Authorization"), "body": payload}) + if state.redirect_url: + self.send_response(state.redirect_status) + self.send_header("Location", state.redirect_url) + self.end_headers() + return + value = ( + {"content": [{"type": "text", "text": state.text}], "usage": {"input_tokens": 3, "output_tokens": 1}} + if self.path.endswith("/messages") + else { + "choices": [{"message": {"content": state.text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}, + } + ) + body = json.dumps(value).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + state.calls.append({"path": self.path, "authorization": self.headers.get("Authorization")}) + self.send_response(200) + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + state.url = f"http://127.0.0.1:{server.server_port}" + thread = Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) + thread.start() + try: + yield state + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.fixture +def model_http_server(monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[], SimpleNamespace]]: + monkeypatch.setenv("NO_PROXY", "127.0.0.1") + with ExitStack() as stack: + yield lambda: stack.enter_context(_model_server()) diff --git a/evaluation/memory/longmemeval_v2/tests/test_cli_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_cli_smoke.py new file mode 100644 index 0000000000..e206674d45 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/tests/test_cli_smoke.py @@ -0,0 +1,126 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys +from pathlib import Path + +from typer.testing import CliRunner + +from powercontext_eval_longmemeval_v2.catalog import ( + LongMemEvalV2EnvironmentError, + LongMemEvalV2InputError, +) +from powercontext_eval_longmemeval_v2.cli import app +from powercontext_eval_longmemeval_v2.smoke import PreparedSmokeRun + + +def test_longmemeval_v2_smoke_prepares_input_artifacts_without_a_model(monkeypatch, tmp_path: Path) -> None: + def prepare(**kwargs: object) -> PreparedSmokeRun: + assert kwargs == { + "data_root": Path("/data"), + "dataset_lock": Path("/dataset-lock.json"), + "harness_root": Path("/harness"), + "smoke_manifest": Path("/smoke.json"), + "output_dir": Path("/output"), + } + return PreparedSmokeRun( + output_dir=tmp_path / "output", + manifest_path=tmp_path / "output" / "manifest.json", + subset_path=tmp_path / "output" / "subset.json", + ) + + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.prepare_smoke_run", prepare) + result = CliRunner().invoke( + app, + [ + "smoke", + "--data-root", + "/data", + "--dataset-lock", + "/dataset-lock.json", + "--harness-root", + "/harness", + "--smoke-manifest", + "/smoke.json", + "--output-dir", + "/output", + ], + ) + + assert result.exit_code == 0, result.output + assert '"classification": "smoke-subset"' in result.output + + +def test_longmemeval_v2_smoke_reports_invalid_configuration_as_bad_parameter( + monkeypatch, +) -> None: + def prepare(**_kwargs: object) -> PreparedSmokeRun: + raise LongMemEvalV2InputError("Smoke manifest schema is unsupported") + + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.prepare_smoke_run", prepare) + result = CliRunner().invoke(app, ["smoke", *_longmemeval_smoke_arguments()]) + + assert result.exit_code == 2 + assert "Invalid value" in result.output + assert "Smoke manifest schema is unsupported" in result.output + + +def test_longmemeval_v2_smoke_reports_environment_failure_with_exit_one( + monkeypatch, +) -> None: + def prepare(**_kwargs: object) -> PreparedSmokeRun: + raise LongMemEvalV2EnvironmentError("LongMemEval-V2 SHA-256 mismatch for questions.jsonl") + + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.prepare_smoke_run", prepare) + result = CliRunner().invoke(app, ["smoke", *_longmemeval_smoke_arguments()]) + + assert result.exit_code == 1 + assert "LongMemEval-V2 smoke failed" in result.output + assert "SHA-256 mismatch" in result.output + assert "Invalid value" not in result.output + + +def _longmemeval_smoke_arguments() -> list[str]: + return [ + "--data-root", + "/data", + "--dataset-lock", + "/dataset-lock.json", + "--harness-root", + "/harness", + "--smoke-manifest", + "/smoke.json", + "--output-dir", + "/output", + ] + + +def test_cli_help_describes_the_evaluation_runner() -> None: + result = CliRunner().invoke(app, ["--help"]) + + assert result.exit_code == 0 + assert "LongMemEval-V2" in result.output + assert not isinstance(result.exception, RuntimeError) + + +def test_cli_module_is_directly_executable() -> None: + result = subprocess.run( + [sys.executable, "-m", "powercontext_eval_longmemeval_v2.cli", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert "smoke" in result.stdout diff --git a/evaluation/tests/unit/test_longmemeval_v2_adapter.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_adapter.py similarity index 94% rename from evaluation/tests/unit/test_longmemeval_v2_adapter.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_adapter.py index 4197cb53a5..bbc1c40292 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_adapter.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_adapter.py @@ -21,7 +21,7 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2.adapter import ( +from powercontext_eval_longmemeval_v2.adapter import ( AUDIT_SCHEMA, PowerContextHTTPRuntime, PowerContextMemory, @@ -53,7 +53,11 @@ def remember_memory(self, payload: Mapping[str, object]) -> Mapping[str, object] "memory": {"family": "memory", "artifact_id": "memory", "revision": index}, "entry": { "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": index}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": index, + }, "entry_id": f"entry-{index}", "entry_version_id": f"entry-{index}-v1", } @@ -70,7 +74,11 @@ def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: "score": 0.9, "matched_by": ["fts"], "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": 2}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": 2, + }, "entry_id": "entry-2", "entry_version_id": "entry-2-v1", }, @@ -138,7 +146,9 @@ def audit_events(tmp_path: Path) -> list[dict[str, Any]]: return [json.loads(line) for line in (tmp_path / "adapter-audit.jsonl").read_text(encoding="utf-8").splitlines()] -def test_insert_uses_public_source_and_memory_operations_with_utf8_safe_chunks(tmp_path: Path) -> None: +def test_insert_uses_public_source_and_memory_operations_with_utf8_safe_chunks( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter(tmp_path, runtime, source_chunk_bytes=512) @@ -150,7 +160,15 @@ def test_insert_uses_public_source_and_memory_operations_with_utf8_safe_chunks(t assert all(len(str(request["content"]).encode()) <= 512 for request in runtime.captures) restored = "".join(str(request["content"]) for request in runtime.captures) projected = json.loads(restored) - assert set(projected) == {"id", "domain", "environment", "goal", "outcome", "start_url", "states"} + assert set(projected) == { + "id", + "domain", + "environment", + "goal", + "outcome", + "start_url", + "states", + } assert "question_type" not in restored assert "gold_answer" not in restored assert len(str(runtime.memories[0]["text"]).encode()) <= 8192 @@ -169,7 +187,9 @@ def test_insert_uses_public_source_and_memory_operations_with_utf8_safe_chunks(t assert set(event["timings_ms"]) == {"source_capture", "memory_remember", "total"} -def test_l0_l1_projection_writes_two_bounded_memory_entries_without_schema_changes(tmp_path: Path) -> None: +def test_l0_l1_projection_writes_two_bounded_memory_entries_without_schema_changes( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter(tmp_path, runtime, memory_projection="deterministic-l0-l1-v1") @@ -184,7 +204,9 @@ def test_l0_l1_projection_writes_two_bounded_memory_entries_without_schema_chang assert audit["memory_entry_count"] == 2 -def test_l0_l1_projection_keeps_the_byte_limit_when_a_trajectory_reaches_it(tmp_path: Path) -> None: +def test_l0_l1_projection_keeps_the_byte_limit_when_a_trajectory_reaches_it( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter(tmp_path, runtime, memory_projection="deterministic-l0-l1-v1") long_trajectory = trajectory() @@ -210,7 +232,9 @@ def test_l0_l1_projection_keeps_the_byte_limit_when_a_trajectory_reaches_it(tmp_ assert l1_text.startswith("LongMemEval-V2 deterministic L1 trajectory summary") -def test_query_returns_upstream_text_items_and_records_citations(tmp_path: Path) -> None: +def test_query_returns_upstream_text_items_and_records_citations( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter(tmp_path, runtime, search_mode="fts", search_limit=4) memory.set_query_context(query_invocation_id="query-7") @@ -252,7 +276,9 @@ def test_query_returns_upstream_text_items_and_records_citations(tmp_path: Path) } -def test_failed_query_is_audited_without_question_or_image_content(tmp_path: Path) -> None: +def test_failed_query_is_audited_without_question_or_image_content( + tmp_path: Path, +) -> None: class FailingRuntime(FakeRuntime): def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: raise PowerContextMemoryAdapterError("unavailable") @@ -299,7 +325,9 @@ def test_hybrid_query_requests_and_records_the_executed_mode(tmp_path: Path) -> assert metadata["actual_mode"] == "hybrid" -def test_query_time_compact_uses_public_prepared_context_and_records_strategy(tmp_path: Path) -> None: +def test_query_time_compact_uses_public_prepared_context_and_records_strategy( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter( tmp_path, @@ -325,7 +353,9 @@ def test_query_time_compact_uses_public_prepared_context_and_records_strategy(tm assert event["retrieval_strategy"] == "prepared-context" -def test_query_time_compact_rejects_a_response_over_its_byte_budget(tmp_path: Path) -> None: +def test_query_time_compact_rejects_a_response_over_its_byte_budget( + tmp_path: Path, +) -> None: class OversizedPreparedContextRuntime(FakeRuntime): def prepare_context(self, payload: Mapping[str, object]) -> Mapping[str, object]: return { @@ -346,7 +376,9 @@ def prepare_context(self, payload: Mapping[str, object]) -> Mapping[str, object] memory.query("Which assignment group should I use?") -def test_task_lens_projects_only_question_keywords_into_the_public_search(tmp_path: Path) -> None: +def test_task_lens_projects_only_question_keywords_into_the_public_search( + tmp_path: Path, +) -> None: runtime = FakeRuntime() memory = adapter(tmp_path, runtime, search_mode="fts", task_lens="question-keywords-v1") @@ -358,7 +390,9 @@ def test_task_lens_projects_only_question_keywords_into_the_public_search(tmp_pa assert event["query_sha256"] != event["retrieval_query_sha256"] -def test_fts_query_fails_closed_when_the_server_executed_a_different_mode(tmp_path: Path) -> None: +def test_fts_query_fails_closed_when_the_server_executed_a_different_mode( + tmp_path: Path, +) -> None: class MisreportingHybridRuntime(FakeRuntime): def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: response = dict(super().search_memory(payload)) @@ -377,7 +411,9 @@ def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: assert event["actual_mode"] == "hybrid" -def test_hybrid_query_fails_closed_when_the_server_executed_a_different_mode(tmp_path: Path) -> None: +def test_hybrid_query_fails_closed_when_the_server_executed_a_different_mode( + tmp_path: Path, +) -> None: class SilentFtsFallbackRuntime(FakeRuntime): def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: response = dict(super().search_memory(payload)) @@ -396,7 +432,9 @@ def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: assert event["actual_mode"] == "fts" -def test_an_explicit_mode_fails_closed_when_the_server_reports_no_mode(tmp_path: Path) -> None: +def test_an_explicit_mode_fails_closed_when_the_server_reports_no_mode( + tmp_path: Path, +) -> None: class UnreportedModeRuntime(FakeRuntime): def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: response = dict(super().search_memory(payload)) @@ -445,7 +483,9 @@ def test_duplicate_insert_fails_closed_and_records_the_attempt(tmp_path: Path) - assert len(runtime.captures) == 1 -def test_partial_ingest_failure_preserves_completed_chunk_citations(tmp_path: Path) -> None: +def test_partial_ingest_failure_preserves_completed_chunk_citations( + tmp_path: Path, +) -> None: class PartiallyFailingRuntime(FakeRuntime): def remember_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: raise PowerContextMemoryAdapterError("memory unavailable") diff --git a/evaluation/tests/unit/test_longmemeval_v2_arms.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_arms.py similarity index 88% rename from evaluation/tests/unit/test_longmemeval_v2_arms.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_arms.py index 401451ee92..7112d39862 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_arms.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_arms.py @@ -17,7 +17,7 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2.arms import ( +from powercontext_eval_longmemeval_v2.arms import ( CURRENT_MEMORY_FTS, CURRENT_MEMORY_HYBRID, DEFAULT_EXPERIMENT_ARM_ID, @@ -73,7 +73,9 @@ def test_get_experiment_arm_returns_the_registered_arm() -> None: @pytest.mark.parametrize("arm_id", ["", " ", "l0-persistent-v1", "temporal-recency-v1", "task-lens-v1"]) -def test_get_experiment_arm_rejects_unknown_ids_with_the_supported_list(arm_id: str) -> None: +def test_get_experiment_arm_rejects_unknown_ids_with_the_supported_list( + arm_id: str, +) -> None: with pytest.raises(ExperimentArmError, match="supported arms"): get_experiment_arm(arm_id) @@ -137,9 +139,19 @@ def comparable_manifest(arm: ExperimentArm, **overrides: object) -> dict[str, ob "experiment_arm": arm_manifest_record(arm), "inputs": { "data_root": "D:/data", - "dataset_lock": {"path": "D:/locks/a.json", "content_sha256": "lock-digest"}, - "smoke_manifest": {"path": "D:/locks/b.json", "content_sha256": "smoke-digest"}, - "harness": {"root": "D:/harness", "commit": "2cc8c540", "python": "D:/python"}, + "dataset_lock": { + "path": "D:/locks/a.json", + "content_sha256": "lock-digest", + }, + "smoke_manifest": { + "path": "D:/locks/b.json", + "content_sha256": "smoke-digest", + }, + "harness": { + "root": "D:/harness", + "commit": "2cc8c540", + "python": "D:/python", + }, }, "processor": {"model": "Qwen/Qwen3.5-9B", "revision": "processor-sha"}, "memory_context_max_tokens": 200_000, @@ -191,9 +203,17 @@ def test_runs_without_a_registered_arm_record_are_refused() -> None: with pytest.raises(ExperimentArmError, match="has no experiment_arm record"): ensure_comparable_experiment_runs(manifest_a, without_arm) - tampered_arm = dict(cast("dict[str, object]", comparable_manifest(CURRENT_MEMORY_HYBRID)["experiment_arm"])) + tampered_arm = dict( + cast( + "dict[str, object]", + comparable_manifest(CURRENT_MEMORY_HYBRID)["experiment_arm"], + ) + ) tampered_arm["search_mode"] = "vector" - manifest_b = {**comparable_manifest(CURRENT_MEMORY_HYBRID), "experiment_arm": tampered_arm} + manifest_b = { + **comparable_manifest(CURRENT_MEMORY_HYBRID), + "experiment_arm": tampered_arm, + } with pytest.raises(ExperimentArmError, match="unregistered experiment_arm record"): ensure_comparable_experiment_runs(manifest_a, manifest_b) @@ -203,7 +223,10 @@ def test_runs_that_differ_beyond_the_arm_are_refused() -> None: manifest_b = comparable_manifest(CURRENT_MEMORY_HYBRID) manifest_b["inputs"] = { "data_root": "D:/data", - "dataset_lock": {"path": "D:/locks/other.json", "content_sha256": "other-lock-digest"}, + "dataset_lock": { + "path": "D:/locks/other.json", + "content_sha256": "other-lock-digest", + }, "smoke_manifest": {"path": "D:/locks/b.json", "content_sha256": "smoke-digest"}, "harness": {"root": "D:/harness", "commit": "2cc8c540", "python": "D:/python"}, } @@ -230,13 +253,15 @@ def test_a_different_search_limit_or_processor_refuses_the_comparison() -> None: ensure_comparable_experiment_runs(manifest_a, other_limit) other_reader = comparable_manifest( - CURRENT_MEMORY_HYBRID, reader={"provider": "deepseek-openai", "model": "deepseek-chat"} + CURRENT_MEMORY_HYBRID, + reader={"provider": "deepseek-openai", "model": "deepseek-chat"}, ) with pytest.raises(ExperimentArmError, match="reader"): ensure_comparable_experiment_runs(manifest_a, other_reader) other_powercontext_revision = comparable_manifest( - CURRENT_MEMORY_HYBRID, revisions={"powercontext": "other-sha", "integration": "integration-sha"} + CURRENT_MEMORY_HYBRID, + revisions={"powercontext": "other-sha", "integration": "integration-sha"}, ) with pytest.raises(ExperimentArmError, match=r"revisions\.powercontext"): ensure_comparable_experiment_runs(manifest_a, other_powercontext_revision) diff --git a/evaluation/tests/unit/test_longmemeval_v2_catalog.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_catalog.py similarity index 87% rename from evaluation/tests/unit/test_longmemeval_v2_catalog.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_catalog.py index a8d6147ea1..267abe0b09 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_catalog.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_catalog.py @@ -21,9 +21,9 @@ import pytest -import powercontext_eval.benchmarks.longmemeval_v2.catalog as longmemeval_catalog -import powercontext_eval.benchmarks.longmemeval_v2.smoke as longmemeval_smoke -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( +import powercontext_eval_longmemeval_v2.catalog as longmemeval_catalog +import powercontext_eval_longmemeval_v2.smoke as longmemeval_smoke +from powercontext_eval_longmemeval_v2.catalog import ( RUN_INPUT_MANIFEST_SCHEMA, RUN_SUBSET_SCHEMA, SMOKE_MANIFEST_SCHEMA, @@ -38,9 +38,9 @@ load_smoke_manifest, validate_harness_checkout, ) -from powercontext_eval.benchmarks.longmemeval_v2.smoke import prepare_smoke_run +from powercontext_eval_longmemeval_v2.smoke import prepare_smoke_run -LOCKS = Path(__file__).parents[2] / "locks" +LOCKS = Path(__file__).parents[1] / "locks" def _write_jsonl(path: Path, values: list[dict[str, object]]) -> None: @@ -53,11 +53,36 @@ def data_root(tmp_path: Path) -> Path: _write_jsonl( root / "questions.jsonl", [ - {"id": "q-static", "domain": "web", "question": "static", "question_type": "static-environment"}, - {"id": "q-dynamic", "domain": "web", "question": "dynamic", "question_type": "dynamic-environment"}, - {"id": "q-workflow", "domain": "enterprise", "question": "workflow", "question_type": "procedure"}, - {"id": "q-gotcha", "domain": "enterprise", "question": "gotcha", "question_type": "errors-gotchas"}, - {"id": "q-premise", "domain": "web", "question": "premise", "question_type": "static-environment-abs"}, + { + "id": "q-static", + "domain": "web", + "question": "static", + "question_type": "static-environment", + }, + { + "id": "q-dynamic", + "domain": "web", + "question": "dynamic", + "question_type": "dynamic-environment", + }, + { + "id": "q-workflow", + "domain": "enterprise", + "question": "workflow", + "question_type": "procedure", + }, + { + "id": "q-gotcha", + "domain": "enterprise", + "question": "gotcha", + "question_type": "errors-gotchas", + }, + { + "id": "q-premise", + "domain": "web", + "question": "premise", + "question_type": "static-environment-abs", + }, ], ) _write_jsonl( @@ -131,12 +156,19 @@ def harness_checkout(tmp_path: Path) -> tuple[Path, str]: (harness / "evaluation").mkdir(parents=True) harness_file = harness / "evaluation" / "harness.py" harness_file.write_text("original = True\n", encoding="utf-8") - for arguments in (("init", "-q"), ("config", "user.email", "tests@example.com"), ("config", "user.name", "Tests")): + for arguments in ( + ("init", "-q"), + ("config", "user.email", "tests@example.com"), + ("config", "user.name", "Tests"), + ): subprocess.run(("git", "-C", str(harness), *arguments), check=True) subprocess.run(("git", "-C", str(harness), "add", "evaluation/harness.py"), check=True) subprocess.run(("git", "-C", str(harness), "commit", "-qm", "fixture harness"), check=True) revision = subprocess.run( - ("git", "-C", str(harness), "rev-parse", "HEAD"), check=True, capture_output=True, text=True + ("git", "-C", str(harness), "rev-parse", "HEAD"), + check=True, + capture_output=True, + text=True, ).stdout.strip() return harness, revision @@ -144,7 +176,13 @@ def harness_checkout(tmp_path: Path) -> tuple[Path, str]: def test_catalog_validates_all_upstream_input_relationships(tmp_path: Path) -> None: catalog = LongMemEvalV2Catalog.load(data_root(tmp_path), tier="small") - assert catalog.source_question_ids == ("q-static", "q-dynamic", "q-workflow", "q-gotcha", "q-premise") + assert catalog.source_question_ids == ( + "q-static", + "q-dynamic", + "q-workflow", + "q-gotcha", + "q-premise", + ) assert catalog.questions["q-workflow"].ability == "workflow_knowledge" assert set(catalog.input_digests) == { "questions.jsonl", @@ -187,7 +225,9 @@ def test_smoke_selection_requires_upstream_question_order(tmp_path: Path) -> Non catalog.select_smoke((cases[1], cases[0], *cases[2:])) -def test_catalog_skips_blank_jsonl_rows_like_the_upstream_loader(tmp_path: Path) -> None: +def test_catalog_skips_blank_jsonl_rows_like_the_upstream_loader( + tmp_path: Path, +) -> None: root = data_root(tmp_path) questions = root / "questions.jsonl" questions.write_text("\n" + questions.read_text(encoding="utf-8") + "\n", encoding="utf-8") @@ -210,7 +250,9 @@ def read_bytes(*_args: object, **_kwargs: object) -> bytes: assert len(catalog.questions) == 5 -def test_smoke_manifest_is_fixed_and_validated_against_the_catalog(tmp_path: Path) -> None: +def test_smoke_manifest_is_fixed_and_validated_against_the_catalog( + tmp_path: Path, +) -> None: root = data_root(tmp_path) manifest = smoke_manifest(tmp_path / "smoke.json") selection = load_smoke_manifest(manifest) diff --git a/evaluation/tests/unit/test_longmemeval_v2_cli.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_cli.py similarity index 89% rename from evaluation/tests/unit/test_longmemeval_v2_cli.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_cli.py index 1f5d239714..34a2345a74 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_cli.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_cli.py @@ -19,18 +19,18 @@ import pytest from typer.testing import CliRunner -from powercontext_eval.benchmarks.longmemeval_v2.costs import ModelPricePolicy -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import PreparedPromptRun -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ReaderSmokeRun -from powercontext_eval.benchmarks.longmemeval_v2.replay_score import ReplayScoreRun -from powercontext_eval.benchmarks.longmemeval_v2.report import ReportError, ReportRun -from powercontext_eval.benchmarks.longmemeval_v2.retrieval_smoke import ( +from powercontext_eval_longmemeval_v2.cli import app +from powercontext_eval_longmemeval_v2.costs import ModelPricePolicy +from powercontext_eval_longmemeval_v2.prepare_smoke import PreparedPromptRun +from powercontext_eval_longmemeval_v2.reader_smoke import ReaderSmokeRun +from powercontext_eval_longmemeval_v2.replay_score import ReplayScoreRun +from powercontext_eval_longmemeval_v2.report import ReportError, ReportRun +from powercontext_eval_longmemeval_v2.retrieval_smoke import ( RetrievalCapabilityError, RetrievalSmokeRun, ) -from powercontext_eval.benchmarks.longmemeval_v2.run_smoke import RunSmokeError, SmokeRunResult -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ScoreSmokeRun -from powercontext_eval.cli import app +from powercontext_eval_longmemeval_v2.run_smoke import RunSmokeError, SmokeRunResult +from powercontext_eval_longmemeval_v2.score_smoke import ScoreSmokeRun def test_longmemeval_v2_retrieval_smoke_runs_without_reader_or_judge( @@ -50,11 +50,10 @@ def run(**kwargs: object) -> RetrievalSmokeRun: audit_path=output / "adapter-audit.jsonl", ) - monkeypatch.setattr("powercontext_eval.cli.run_retrieval_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_retrieval_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "retrieval-smoke", "--data-root", "/data", @@ -112,11 +111,10 @@ def prepare(**kwargs: object) -> PreparedPromptRun: summary_path=output / "prepare-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.prepare_reader_inputs_smoke", prepare) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.prepare_reader_inputs_smoke", prepare) result = CliRunner().invoke( app, [ - "longmemeval-v2", "prepare-smoke", "--retrieval-dir", "/retrieval", @@ -162,11 +160,10 @@ def run(**kwargs: object) -> ReaderSmokeRun: summary_path=output / "reader-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.run_reader_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_reader_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "reader-smoke", "--prepared-dir", "/prepared", @@ -189,6 +186,7 @@ def run(**kwargs: object) -> ReaderSmokeRun: "max_tokens": 512, "temperature": 0.0, "timeout_seconds": 120.0, + "allow_insecure_http": False, "max_questions": None, "price_policy": None, } @@ -213,11 +211,10 @@ def run(**kwargs: object) -> ScoreSmokeRun: summary_path=output / "score-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.run_score_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_score_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "score-smoke", "--reader-dir", "/reader", @@ -250,6 +247,7 @@ def run(**kwargs: object) -> ScoreSmokeRun: "judge_max_tokens": 256, "judge_temperature": 0.0, "judge_timeout_seconds": 120.0, + "judge_allow_insecure_http": False, "judge_price_policy": None, } ] @@ -271,11 +269,10 @@ def replay(**kwargs: object) -> ReplayScoreRun: summary_path=output / "replay-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.replay_score_smoke", replay) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.replay_score_smoke", replay) result = CliRunner().invoke( app, [ - "longmemeval-v2", "replay-score", "--score-dir", "/score", @@ -288,10 +285,21 @@ def replay(**kwargs: object) -> ReplayScoreRun: assert result.exit_code == 0, result.output assert '"classification": "smoke-subset-score-replay"' in result.output - assert calls == [{"score_dir": Path("/score"), "harness_root": Path("/harness"), "output_dir": Path("/output")}] + assert calls == [ + { + "score_dir": Path("/score"), + "harness_root": Path("/harness"), + "output_dir": Path("/output"), + } + ] -def test_longmemeval_v2_run_smoke_forwards_every_stage_option(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +@pytest.mark.parametrize("allow_http", [False, True]) +def test_longmemeval_v2_run_smoke_forwards_every_stage_option( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + allow_http: bool, +) -> None: calls: list[dict[str, object]] = [] def run(**kwargs: object) -> SmokeRunResult: @@ -303,16 +311,22 @@ def run(**kwargs: object) -> SmokeRunResult: summary_path=output / "run-summary.json", failures_path=output / "failures.jsonl", status="completed", - completed_phases=("preflight", "retrieval", "prepare", "reader", "score", "replay"), + completed_phases=( + "preflight", + "retrieval", + "prepare", + "reader", + "score", + "replay", + ), skipped_phases=(), failed_phase=None, ) - monkeypatch.setattr("powercontext_eval.cli.run_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -332,6 +346,7 @@ def run(**kwargs: object) -> SmokeRunResult: "pc-sha", "--integration-revision", "integration-sha", + *(["--reader-allow-insecure-http", "--judge-allow-insecure-http"] if allow_http else []), ], ) @@ -364,6 +379,7 @@ def run(**kwargs: object) -> SmokeRunResult: "reader_max_tokens": 512, "reader_temperature": 0.0, "reader_timeout_seconds": 120.0, + "reader_allow_insecure_http": allow_http, "judge_provider": "deepseek-openai", "judge_model": "deepseek-flash", "judge_token_env": "DEEPSEEK_API_KEY", @@ -371,6 +387,7 @@ def run(**kwargs: object) -> SmokeRunResult: "judge_max_tokens": 256, "judge_temperature": 0.0, "judge_timeout_seconds": 120.0, + "judge_allow_insecure_http": allow_http, "reader_price_policy": None, "judge_price_policy": None, "skip_reader": False, @@ -395,11 +412,10 @@ def run(**kwargs: object) -> SmokeRunResult: failed_phase="prepare", ) - monkeypatch.setattr("powercontext_eval.cli.run_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -434,11 +450,10 @@ def test_longmemeval_v2_run_smoke_reports_a_refused_output_directory( def run(**kwargs: object) -> SmokeRunResult: raise RunSmokeError("Refusing to overwrite smoke run artifacts: /output") - monkeypatch.setattr("powercontext_eval.cli.run_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -484,11 +499,10 @@ def run(**kwargs: object) -> SmokeRunResult: failed_phase=None, ) - monkeypatch.setattr("powercontext_eval.cli.run_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -522,7 +536,6 @@ def test_longmemeval_v2_run_smoke_rejects_an_unknown_experiment_arm() -> None: result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -557,7 +570,6 @@ def test_longmemeval_v2_retrieval_smoke_rejects_an_unknown_experiment_arm() -> N result = CliRunner().invoke( app, [ - "longmemeval-v2", "retrieval-smoke", "--data-root", "/data", @@ -593,11 +605,10 @@ def run(**kwargs: object) -> RetrievalSmokeRun: "required by experiment arm current-memory-hybrid-v1" ) - monkeypatch.setattr("powercontext_eval.cli.run_retrieval_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_retrieval_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "retrieval-smoke", "--data-root", "/data", @@ -632,20 +643,25 @@ def report(**kwargs: object) -> ReportRun: calls.append(kwargs) return ReportRun(report_path=tmp_path / "report.json", markdown_path=tmp_path / "report.md") - monkeypatch.setattr("powercontext_eval.cli.build_report", report) - result = CliRunner().invoke(app, ["longmemeval-v2", "report", "--run-dir", "/run"]) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.build_report", report) + result = CliRunner().invoke(app, ["report", "--run-dir", "/run"]) assert result.exit_code == 0, result.output assert '"classification": "smoke-subset"' in result.output assert calls == [{"run_dir": Path("/run"), "output_dir": None}] -def test_longmemeval_v2_report_exits_nonzero_for_a_refused_target(monkeypatch: pytest.MonkeyPatch) -> None: +def test_longmemeval_v2_report_exits_nonzero_for_a_refused_target( + monkeypatch: pytest.MonkeyPatch, +) -> None: def report(**kwargs: object) -> ReportRun: raise ReportError("Refusing to overwrite report artifacts: /report") - monkeypatch.setattr("powercontext_eval.cli.build_report", report) - result = CliRunner().invoke(app, ["longmemeval-v2", "report", "--run-dir", "/run", "--output-dir", "/report"]) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.build_report", report) + result = CliRunner().invoke( + app, + ["report", "--run-dir", "/run", "--output-dir", "/report"], + ) assert result.exit_code == 1 assert "Refusing to overwrite report artifacts" in result.output @@ -680,11 +696,10 @@ def run(**kwargs: object) -> ReaderSmokeRun: summary_path=output / "reader-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.run_reader_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_reader_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "reader-smoke", "--prepared-dir", "/prepared", @@ -726,11 +741,10 @@ def run(**kwargs: object) -> ScoreSmokeRun: summary_path=output / "score-summary.json", ) - monkeypatch.setattr("powercontext_eval.cli.run_score_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_score_smoke", run) result = CliRunner().invoke( app, [ - "longmemeval-v2", "score-smoke", "--reader-dir", "/reader", @@ -776,17 +790,23 @@ def run(**kwargs: object) -> SmokeRunResult: summary_path=output / "run-summary.json", failures_path=output / "failures.jsonl", status="completed", - completed_phases=("preflight", "retrieval", "prepare", "reader", "score", "replay"), + completed_phases=( + "preflight", + "retrieval", + "prepare", + "reader", + "score", + "replay", + ), skipped_phases=(), failed_phase=None, ) - monkeypatch.setattr("powercontext_eval.cli.run_smoke", run) + monkeypatch.setattr("powercontext_eval_longmemeval_v2.cli.run_smoke", run) judge_policy_json = json.dumps({**json.loads(PRICE_POLICY_JSON), "model": "deepseek-v4-pro"}) result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -827,7 +847,6 @@ def test_longmemeval_v2_run_smoke_rejects_a_malformed_price_policy() -> None: result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -860,7 +879,6 @@ def test_longmemeval_v2_run_smoke_rejects_a_non_usd_price_policy() -> None: result = CliRunner().invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", @@ -898,7 +916,6 @@ def test_longmemeval_v2_run_smoke_rejects_a_non_json_price_policy() -> None: result = CliRunner(env={"COLUMNS": "200"}).invoke( app, [ - "longmemeval-v2", "run-smoke", "--data-root", "/data", diff --git a/evaluation/tests/unit/test_longmemeval_v2_costs.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_costs.py similarity index 89% rename from evaluation/tests/unit/test_longmemeval_v2_costs.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_costs.py index f632dd8822..d12a585419 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_costs.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_costs.py @@ -23,8 +23,8 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2.catalog import SmokeSelection -from powercontext_eval.benchmarks.longmemeval_v2.costs import ( +from powercontext_eval_longmemeval_v2.catalog import SmokeSelection +from powercontext_eval_longmemeval_v2.costs import ( CostPolicyError, ModelPricePolicy, cost_policy_record, @@ -114,7 +114,12 @@ def test_usage_cost_usd_prices_cache_hit_and_miss_input_separately() -> None: assert policy is not None # 1M cached input at 0.006/M, 1M uncached input at 0.3/M, 1M output at 1.2/M. - priced = usage_cost_usd(policy, cache_hit_tokens=1_000_000, cache_miss_tokens=1_000_000, output_tokens=1_000_000) + priced = usage_cost_usd( + policy, + cache_hit_tokens=1_000_000, + cache_miss_tokens=1_000_000, + output_tokens=1_000_000, + ) assert priced == 1.506 # Pricing every input token at the cache-miss rate would overcharge this by 0.294 USD. blended = usage_cost_usd(policy, cache_hit_tokens=0, cache_miss_tokens=2_000_000, output_tokens=1_000_000) @@ -253,8 +258,12 @@ def test_usage_cost_block_refuses_a_cache_split_that_does_not_match_input_total( {"prompt_tokens": 100, "completion_tokens": 1, "prompt_cache_miss_tokens": 100}, ], ) -def test_deepseek_normalization_drops_a_partial_cache_split(usage: dict[str, int]) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import _normalize_deepseek_response +def test_deepseek_normalization_drops_a_partial_cache_split( + usage: dict[str, int], +) -> None: + from powercontext_eval_longmemeval_v2.reader_smoke import ( + _normalize_deepseek_response, + ) response = _normalize_deepseek_response( { @@ -326,7 +335,11 @@ def _prepared_artifacts(root: Path) -> Path: ) _write_json( prepared / "prepare-summary.json", - {"classification": "smoke-subset-prepare-only", "question_count": 10, "failed": 0}, + { + "classification": "smoke-subset-prepare-only", + "question_count": 10, + "failed": 0, + }, ) with (prepared / "prepared-prompts.jsonl").open("w", encoding="utf-8") as stream: for index in range(10): @@ -337,7 +350,10 @@ def _prepared_artifacts(root: Path) -> Path: "sequence": index + 1, "messages": [ {"role": "system", "content": "system"}, - {"role": "user", "content": [{"type": "text", "text": "question"}]}, + { + "role": "user", + "content": [{"type": "text", "text": "question"}], + }, ], } ) @@ -373,7 +389,9 @@ def complete(self, *, system: str, content: list[dict[str, object]]) -> dict[str def test_deepseek_response_normalization_keeps_the_native_cache_split() -> None: """DeepSeek's ``prompt_cache_*_tokens`` fields must survive into normalized usage.""" - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import _normalize_deepseek_response + from powercontext_eval_longmemeval_v2.reader_smoke import ( + _normalize_deepseek_response, + ) normalized = _normalize_deepseek_response( { @@ -397,7 +415,9 @@ def test_deepseek_response_normalization_keeps_the_native_cache_split() -> None: def test_deepseek_response_normalization_omits_an_absent_cache_split() -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import _normalize_deepseek_response + from powercontext_eval_longmemeval_v2.reader_smoke import ( + _normalize_deepseek_response, + ) normalized = _normalize_deepseek_response( { @@ -410,8 +430,10 @@ def test_deepseek_response_normalization_omits_an_absent_cache_split() -> None: assert normalized["usage"] == {"input_tokens": 10, "output_tokens": 2} -def test_reader_keeps_the_deepseek_cache_split_in_its_outputs_and_summary(tmp_path: Path) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke +def test_reader_keeps_the_deepseek_cache_split_in_its_outputs_and_summary( + tmp_path: Path, +) -> None: + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke result = run_reader_smoke( prepared_dir=_prepared_artifacts(tmp_path), @@ -438,8 +460,10 @@ def test_reader_keeps_the_deepseek_cache_split_in_its_outputs_and_summary(tmp_pa } -def test_reader_summary_prices_cache_hit_and_miss_usage_separately(tmp_path: Path) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke +def test_reader_summary_prices_cache_hit_and_miss_usage_separately( + tmp_path: Path, +) -> None: + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke policy = parse_cost_policy(PRICE_POLICY) transport = _CountingReader() @@ -464,10 +488,12 @@ def test_reader_summary_prices_cache_hit_and_miss_usage_separately(tmp_path: Pat assert manifest["cost_policy"] == PRICE_POLICY -def test_reader_summary_keeps_cost_null_when_a_response_omits_the_cache_split(tmp_path: Path) -> None: +def test_reader_summary_keeps_cost_null_when_a_response_omits_the_cache_split( + tmp_path: Path, +) -> None: """One response without the split makes the summed split incomplete, so cost stays null.""" - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke policy = parse_cost_policy(PRICE_POLICY) result = run_reader_smoke( @@ -486,8 +512,10 @@ def test_reader_summary_keeps_cost_null_when_a_response_omits_the_cache_split(tm assert "input_cache_hit_tokens" not in summary["usage"] -def test_reader_summary_keeps_cost_null_when_the_policy_prices_another_model(tmp_path: Path) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke +def test_reader_summary_keeps_cost_null_when_the_policy_prices_another_model( + tmp_path: Path, +) -> None: + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke policy = parse_cost_policy({**PRICE_POLICY, "model": "deepseek-v4-pro"}) result = run_reader_smoke( @@ -506,7 +534,7 @@ def test_reader_summary_keeps_cost_null_when_the_policy_prices_another_model(tmp def test_reader_summary_keeps_cost_null_without_a_policy(tmp_path: Path) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke result = run_reader_smoke( prepared_dir=_prepared_artifacts(tmp_path), @@ -524,8 +552,10 @@ def test_reader_summary_keeps_cost_null_without_a_policy(tmp_path: Path) -> None assert manifest["cost_policy"] is None -def test_reader_summary_never_writes_zero_for_an_unconfigured_cost(tmp_path: Path) -> None: - from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import run_reader_smoke +def test_reader_summary_never_writes_zero_for_an_unconfigured_cost( + tmp_path: Path, +) -> None: + from powercontext_eval_longmemeval_v2.reader_smoke import run_reader_smoke result = run_reader_smoke( prepared_dir=_prepared_artifacts(tmp_path), @@ -546,10 +576,17 @@ def _score_fixture(root: Path) -> tuple[Path, Path, Path]: data = root / "data" reader.mkdir() data.mkdir() - _write_json(reader / "reader-manifest.json", {"classification": "smoke-subset-reader-only", "judge": None}) + _write_json( + reader / "reader-manifest.json", + {"classification": "smoke-subset-reader-only", "judge": None}, + ) _write_json( reader / "reader-summary.json", - {"classification": "smoke-subset-reader-only", "question_count": 10, "failed": 0}, + { + "classification": "smoke-subset-reader-only", + "question_count": 10, + "failed": 0, + }, ) cases = [] with ( @@ -574,7 +611,14 @@ def _score_fixture(root: Path) -> tuple[Path, Path, Path]: ) cases.append({"question_id": question_id, "ability": "static_state"}) manifest = root / "smoke.json" - _write_json(manifest, {"schema": "powercontext.longmemeval-v2-smoke.v1", "tier": "small", "cases": cases}) + _write_json( + manifest, + { + "schema": "powercontext.longmemeval-v2-smoke.v1", + "tier": "small", + "cases": cases, + }, + ) return reader, data, manifest @@ -619,23 +663,33 @@ def score_to_bool(self, value: Any) -> bool: return bool(value) def _build_abstention_judge_messages(self, **kwargs: Any) -> list[dict[str, str]]: - return [{"role": "system", "content": "abstention"}, {"role": "user", "content": "question"}] + return [ + {"role": "system", "content": "abstention"}, + {"role": "user", "content": "question"}, + ] def _build_gotchas_judge_messages(self, **kwargs: Any) -> list[dict[str, str]]: - return [{"role": "system", "content": "gotchas"}, {"role": "user", "content": "question"}] + return [ + {"role": "system", "content": "gotchas"}, + {"role": "user", "content": "question"}, + ] def _parse_llm_binary_judgement(self, text: str) -> tuple[int, str]: return 1, "accepted" def _run_score(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, policy: Any, judge: Any) -> Any: - from powercontext_eval.benchmarks.longmemeval_v2 import score_smoke - from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import run_score_smoke + from powercontext_eval_longmemeval_v2 import score_smoke + from powercontext_eval_longmemeval_v2.score_smoke import run_score_smoke reader, data, manifest = _score_fixture(tmp_path) monkeypatch.setattr(score_smoke, "validate_harness_checkout", lambda root: None) monkeypatch.setattr(score_smoke, "_load_metrics", lambda root: _FakeMetrics()) - monkeypatch.setattr(score_smoke, "load_dataset_lock", lambda path: SimpleNamespace(tier="small", file_digests={})) + monkeypatch.setattr( + score_smoke, + "load_dataset_lock", + lambda path: SimpleNamespace(tier="small", file_digests={}), + ) monkeypatch.setattr( score_smoke, "LongMemEvalV2Catalog", diff --git a/evaluation/tests/unit/test_longmemeval_v2_harness_bootstrap.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_harness_bootstrap.py similarity index 86% rename from evaluation/tests/unit/test_longmemeval_v2_harness_bootstrap.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_harness_bootstrap.py index e8dcebda65..7f4af3420b 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_harness_bootstrap.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_harness_bootstrap.py @@ -20,8 +20,8 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2 import harness_bootstrap -from powercontext_eval.benchmarks.longmemeval_v2.adapter import AUDIT_SCHEMA, PowerContextMemory +from powercontext_eval_longmemeval_v2 import harness_bootstrap +from powercontext_eval_longmemeval_v2.adapter import AUDIT_SCHEMA, PowerContextMemory _MEMORY_CONTRACT = """ MEMORY_TYPES = {} @@ -84,7 +84,11 @@ def remember_memory(self, payload: Mapping[str, object]) -> Mapping[str, object] return { "entry": { "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": 1}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": 1, + }, "entry_id": "entry-1", "entry_version_id": "entry-1-v1", } @@ -98,7 +102,11 @@ def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: { "text": "The remembered assignment procedure.", "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": 1}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": 1, + }, "entry_id": "entry-1", "entry_version_id": "entry-1-v1", }, @@ -137,7 +145,10 @@ def test_bootstrap_registers_adapter_and_runs_harness_insert_query_chain( { "memory_config": { "memory_type": "powercontext", - "memory_params": {"scope_id": "smoke-scope", "audit_path": str(audit_path)}, + "memory_params": { + "scope_id": "smoke-scope", + "audit_path": str(audit_path), + }, }, "trajectory": { "id": "trajectory-1", @@ -170,7 +181,11 @@ def test_bootstrap_registers_adapter_and_runs_harness_insert_query_chain( validated: list[Path] = [] fake = FakePowerContext() - monkeypatch.setattr(harness_bootstrap, "validate_harness_checkout", lambda root: validated.append(root)) + monkeypatch.setattr( + harness_bootstrap, + "validate_harness_checkout", + lambda root: validated.append(root), + ) monkeypatch.setattr(PowerContextMemory, "_http_runtime", lambda self: fake) harness_bootstrap.run_pinned_harness(harness_root, [str(input_path), str(output_path)]) @@ -179,7 +194,12 @@ def test_bootstrap_registers_adapter_and_runs_harness_insert_query_chain( assert len(fake.captures) == 1 assert len(fake.memories) == 1 assert fake.searches == [ - {"scope_id": "smoke-scope", "query": "Which assignment group should be used?", "limit": 10, "mode": "auto"} + { + "scope_id": "smoke-scope", + "query": "Which assignment group should be used?", + "limit": 10, + "mode": "auto", + } ] output = json.loads(output_path.read_text(encoding="utf-8")) assert output["context"] == [{"type": "text", "value": "The remembered assignment procedure."}] diff --git a/evaluation/tests/unit/test_longmemeval_v2_prepare_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_smoke.py similarity index 67% rename from evaluation/tests/unit/test_longmemeval_v2_prepare_smoke.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_smoke.py index 4231aed8f5..5c380cb16e 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_prepare_smoke.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_smoke.py @@ -21,10 +21,14 @@ from pathlib import Path from typing import Any +import pydantic import pytest -from powercontext_eval.benchmarks.longmemeval_v2 import prepare_smoke -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import PrepareSmokeError, prepare_reader_inputs_smoke +from powercontext_eval_longmemeval_v2 import prepare_smoke +from powercontext_eval_longmemeval_v2.prepare_smoke import ( + PrepareSmokeError, + prepare_reader_inputs_smoke, +) def retrieval_artifacts(tmp_path: Path) -> Path: @@ -40,7 +44,13 @@ def retrieval_artifacts(tmp_path: Path) -> Path: encoding="utf-8", ) (root / "summary.json").write_text( - json.dumps({"classification": "smoke-subset-retrieval-only", "question_count": 10, "failed": 0}), + json.dumps( + { + "classification": "smoke-subset-retrieval-only", + "question_count": 10, + "failed": 0, + } + ), encoding="utf-8", ) (root / "retrieval-results.jsonl").write_text("{}\n", encoding="utf-8") @@ -82,21 +92,24 @@ def fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[s ) manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) - assert manifest["processor"] == {"model": "test/processor", "revision": "processor-sha"} + assert manifest["processor"] == { + "model": "test/processor", + "revision": "processor-sha", + } assert manifest["memory_context_max_tokens"] == 123 assert manifest["reader"] is None assert manifest["judge"] is None assert len(calls) == 1 command, kwargs = calls[0] - assert command[1:3] == ["-m", "powercontext_eval.benchmarks.longmemeval_v2.prepare_worker"] + assert command[1:3] == ["-m", "powercontext_eval_longmemeval_v2.prepare_worker"] assert command[command.index("--processor-revision") + 1] == "processor-sha" assert kwargs["cwd"] == harness assert kwargs["env"]["PYTHONNOUSERSITE"] == "1" - source_root = Path(prepare_smoke.__file__).parents[3] - assert Path(kwargs["env"]["PYTHONPATH"].split(os.pathsep, 1)[0]).resolve() == source_root.resolve() -def test_prepare_smoke_refuses_to_overwrite_before_validating_harness(tmp_path: Path) -> None: +def test_prepare_smoke_refuses_to_overwrite_before_validating_harness( + tmp_path: Path, +) -> None: output = tmp_path / "prepared" output.mkdir() @@ -162,37 +175,72 @@ def test_prepare_smoke_launches_the_requested_virtualenv_interpreter( venv_root = tmp_path / "harness-venv" venv.create(venv_root, with_pip=False) interpreter_suffix = "Scripts/python.exe" if os.name == "nt" else "bin/python" - retrieval_artifacts(tmp_path) - harness = tmp_path / "harness" - harness.mkdir() - monkeypatch.chdir(tmp_path) - real_run = subprocess.run - launches: list[list[str]] = [] - - def probe_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - # Launch the constructed interpreter for real: the prepare stage depends on the - # requested virtualenv (whose POSIX python is a symlink), not the base one. - launches.append(list(command)) - probe = real_run( - [command[0], "-c", "import sys; print(sys.prefix)"], - capture_output=True, - text=True, - check=False, + interpreter = venv_root / interpreter_suffix + retrieval = retrieval_artifacts(tmp_path) + (retrieval / "retrieval-results.jsonl").write_text( + json.dumps( + { + "question_id": "q-1", + "domain": "enterprise", + "question": {"text": "Where is the evidence?", "image": None}, + "memory_context": [{"type": "text", "value": "The saved evidence."}], + } ) - assert probe.returncode == 0, probe.stderr - assert Path(probe.stdout.strip()).resolve() == venv_root.resolve() - prompts_path = Path(command[command.index("--prompts-path") + 1]) - failures_path = Path(command[command.index("--failures-path") + 1]) - summary_path = Path(command[command.index("--summary-path") + 1]) - prompts_path.write_text("{}\n", encoding="utf-8") - failures_path.write_text("", encoding="utf-8") - summary_path.write_text("{}\n", encoding="utf-8") - return subprocess.CompletedProcess(command, 0, probe.stdout, "") + + "\n", + encoding="utf-8", + ) + harness = tmp_path / "harness" + (harness / "evaluation").mkdir(parents=True) + (harness / "evaluation" / "__init__.py").write_text("", encoding="utf-8") + (harness / "evaluation" / "harness.py").write_text( + """ +import sys +from types import SimpleNamespace +MEMORY_CONTEXT_PROCESSOR_LOCAL = SimpleNamespace() +def get_system_prompt(domain): + return sys.prefix + +def validate_memory_context_items(memory_context, *, question_id): + return memory_context + +def truncate_memory_context(memory_context, *, max_tokens, question_id): + return memory_context, 3, 3 + +def build_messages(*, system_prompt, question_text, image_path, memory_context): + messages = [{"role": "system", "content": system_prompt}, {"role": "user", "content": question_text}] + return messages, messages +""", + encoding="utf-8", + ) + (harness / "transformers.py").write_text( + """ +class AutoProcessor: + @staticmethod + def from_pretrained(model, *, revision): + return object() +""", + encoding="utf-8", + ) + monkeypatch.delenv("PYTHONPATH", raising=False) + monkeypatch.chdir(tmp_path) + probe = subprocess.run( + [ + str(interpreter), + "-c", + ("import importlib.util; assert importlib.util.find_spec('powercontext_eval_longmemeval_v2') is None"), + ], + capture_output=True, + text=True, + check=False, + ) + assert probe.returncode == 0, probe.stderr + # Supply runtime dependencies without installing anything into the harness venv. + dependency_path = str(Path(pydantic.__file__).resolve().parent.parent) + monkeypatch.setenv("PYTHONPATH", dependency_path) monkeypatch.setattr(prepare_smoke, "validate_harness_checkout", lambda root: None) - monkeypatch.setattr(prepare_smoke.subprocess, "run", probe_run) - prepare_reader_inputs_smoke( + result = prepare_reader_inputs_smoke( retrieval_dir=Path("retrieval"), harness_root=Path("harness"), harness_python=Path("harness-venv") / interpreter_suffix, @@ -200,6 +248,9 @@ def probe_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[ processor_revision="processor-sha", ) - [command] = launches - assert Path(command[0]).is_absolute() - assert command[0].replace("\\", "/").endswith(f"harness-venv/{interpreter_suffix}") + prepared = json.loads(result.prompts_path.read_text(encoding="utf-8")) + assert Path(prepared["system_prompt"]).resolve() == venv_root.resolve() + assert prepared["memory_context"] == [{"type": "text", "value": "The saved evidence."}] + summary = json.loads(result.summary_path.read_text(encoding="utf-8")) + assert summary["succeeded"] == 1 + assert summary["failed"] == 0 diff --git a/evaluation/tests/unit/test_longmemeval_v2_prepare_worker.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_worker.py similarity index 88% rename from evaluation/tests/unit/test_longmemeval_v2_prepare_worker.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_worker.py index 9664cf1ae5..394e64098d 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_prepare_worker.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_prepare_worker.py @@ -16,7 +16,7 @@ from typing import Any -from powercontext_eval.benchmarks.longmemeval_v2.prepare_worker import _prepare_record +from powercontext_eval_longmemeval_v2.prepare_worker import _prepare_record class FakeHarness: @@ -52,7 +52,10 @@ def build_messages( assert question_text == "Where is the evidence?" assert image_path is None assert memory_context == [{"type": "text", "value": "memory-a"}] - messages = [{"role": "system", "content": system_prompt}, {"role": "user", "content": question_text}] + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": question_text}, + ] return messages, messages @@ -81,4 +84,11 @@ def test_prepare_record_uses_harness_budget_and_emits_replayable_prompt() -> Non assert prepared["memory_context_max_tokens"] == 20 assert prepared["prompt_sha256"] assert prepared["messages"] == prepared["prompt_messages"] - assert not {"answer", "gold_answer", "question_type", "judge", "scorer", "eval_function"} & set(prepared) + assert not { + "answer", + "gold_answer", + "question_type", + "judge", + "scorer", + "eval_function", + } & set(prepared) diff --git a/evaluation/tests/unit/test_longmemeval_v2_reader_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_reader_smoke.py similarity index 63% rename from evaluation/tests/unit/test_longmemeval_v2_reader_smoke.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_reader_smoke.py index 0038875ac6..c1a338370c 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_reader_smoke.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_reader_smoke.py @@ -18,12 +18,14 @@ from collections.abc import Mapping from pathlib import Path from typing import Any, Self +from urllib.request import OpenerDirector import pytest +from typer.testing import CliRunner -from powercontext_eval.benchmarks.longmemeval_v2 import reader_smoke -from powercontext_eval.benchmarks.longmemeval_v2.costs import ModelPricePolicy -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ( +from powercontext_eval_longmemeval_v2.cli import app +from powercontext_eval_longmemeval_v2.costs import ModelPricePolicy +from powercontext_eval_longmemeval_v2.reader_smoke import ( AnthropicCompatibleReader, DeepSeekOpenAIReader, ReaderSmokeError, @@ -35,7 +37,14 @@ @pytest.mark.parametrize("url", ["https://provider.example/?token=secret", "https://provider.example/#token"]) def test_reader_rejects_query_and_fragment_urls(reader_type: type[object], url: str) -> None: with pytest.raises(ReaderSmokeError, match="without credentials"): - reader_type(url, token="token", model="model", max_tokens=1, temperature=0.0, timeout_seconds=1) # type: ignore[operator] + reader_type( + url, + token="token", + model="model", + max_tokens=1, + temperature=0.0, + timeout_seconds=1, + ) # type: ignore[operator] class FakeReader: @@ -56,11 +65,23 @@ def prepared_artifacts(tmp_path: Path) -> Path: root = tmp_path / "prepared" root.mkdir() (root / "prepare-manifest.json").write_text( - json.dumps({"classification": "smoke-subset-prepare-only", "reader": None, "judge": None}), + json.dumps( + { + "classification": "smoke-subset-prepare-only", + "reader": None, + "judge": None, + } + ), encoding="utf-8", ) (root / "prepare-summary.json").write_text( - json.dumps({"classification": "smoke-subset-prepare-only", "question_count": 10, "failed": 0}), + json.dumps( + { + "classification": "smoke-subset-prepare-only", + "question_count": 10, + "failed": 0, + } + ), encoding="utf-8", ) with (root / "prepared-prompts.jsonl").open("w", encoding="utf-8") as stream: @@ -77,7 +98,10 @@ def prepared_artifacts(tmp_path: Path) -> Path: "gold_answer": "must-not-be-copied", "messages": [ {"role": "system", "content": "system"}, - {"role": "user", "content": [{"type": "text", "text": "question"}]}, + { + "role": "user", + "content": [{"type": "text", "text": "question"}], + }, ], } ) @@ -117,7 +141,9 @@ def test_reader_smoke_writes_responses_and_never_persists_credentials_or_gold( assert summary["cost"]["unavailable_reason"] == "no price policy was configured for this run" -def test_reader_smoke_refuses_to_overwrite_before_loading_prepared_artifacts(tmp_path: Path) -> None: +def test_reader_smoke_refuses_to_overwrite_before_loading_prepared_artifacts( + tmp_path: Path, +) -> None: output = tmp_path / "reader" output.mkdir() @@ -129,7 +155,9 @@ def test_reader_smoke_refuses_to_overwrite_before_loading_prepared_artifacts(tmp ) -def test_deepseek_reader_uses_openai_messages_and_disables_thinking(monkeypatch: pytest.MonkeyPatch) -> None: +def test_deepseek_reader_uses_openai_messages_and_disables_thinking( + monkeypatch: pytest.MonkeyPatch, +) -> None: requests: list[Any] = [] class Response: @@ -145,7 +173,12 @@ def read(self) -> bytes: return json.dumps( { "model": "deepseek-flash", - "choices": [{"finish_reason": "stop", "message": {"content": "\\boxed{answer}"}}], + "choices": [ + { + "finish_reason": "stop", + "message": {"content": "\\boxed{answer}"}, + } + ], "usage": {"prompt_tokens": 12, "completion_tokens": 3}, } ).encode() @@ -155,7 +188,7 @@ def fake_urlopen(request: Any, *, timeout: float) -> Response: requests.append(request) return Response() - monkeypatch.setattr(reader_smoke, "urlopen", fake_urlopen) + monkeypatch.setattr(OpenerDirector, "open", lambda self, *args, **kwargs: fake_urlopen(*args, **kwargs)) reader = DeepSeekOpenAIReader( "https://api.deepseek.com", token="super-secret-token", @@ -210,7 +243,7 @@ def read(self) -> bytes: def fake_urlopen(request: Any, *, timeout: float) -> NoTextResponse: return NoTextResponse() - monkeypatch.setattr(reader_smoke, "urlopen", fake_urlopen) + monkeypatch.setattr(OpenerDirector, "open", lambda self, *args, **kwargs: fake_urlopen(*args, **kwargs)) reader = DeepSeekOpenAIReader( "https://api.deepseek.com", token="token", @@ -256,8 +289,104 @@ def fake_urlopen(request: Any, *, timeout: float) -> NoTextResponse: assert all(failure["error_type"] == "ReaderResponseError" for failure in failures) assert all( failure["usage"] - == {"input_tokens": 100, "output_tokens": 20, "input_cache_hit_tokens": 40, "input_cache_miss_tokens": 60} + == { + "input_tokens": 100, + "output_tokens": 20, + "input_cache_hit_tokens": 40, + "input_cache_miss_tokens": 60, + } for failure in failures ) assert all(isinstance(failure["reader_latency_ms"], (int, float)) for failure in failures) assert (output / "reader-outputs.jsonl").read_text(encoding="utf-8") == "" + + +@pytest.mark.parametrize("reader_type", [AnthropicCompatibleReader, DeepSeekOpenAIReader]) +@pytest.mark.parametrize( + ("url", "allow_http"), + [ + ("http://127.0.0.1:1", False), + ("http://user:secret@127.0.0.1:1", True), + ("http://@127.0.0.1:1", True), + ("http://127.0.0.1:1/?token=secret", True), + ("http://127.0.0.1:1/#token", True), + ("file:///tmp/model", True), + ], +) +def test_reader_http_opt_in_does_not_accept_unsafe_urls(reader_type: Any, url: str, allow_http: bool) -> None: + with pytest.raises(ReaderSmokeError, match="without credentials"): + reader_type( + url, + token="secret", + model="model", + max_tokens=10, + temperature=0, + timeout_seconds=1, + allow_insecure_http=allow_http, + ) + + +@pytest.mark.parametrize("provider", ["anthropic-compatible", "deepseek-openai"]) +def test_reader_cli_requires_http_opt_in_and_records_it( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + model_http_server: Any, + provider: str, +) -> None: + server = model_http_server() + output = tmp_path / "reader" + monkeypatch.setenv("MODEL_TEST_TOKEN", "local-test-token") + arguments = [ + "reader-smoke", + "--prepared-dir", + str(prepared_artifacts(tmp_path)), + "--output-dir", + str(output), + "--provider", + provider, + "--base-url", + server.url, + "--token-env", + "MODEL_TEST_TOKEN", + "--max-questions", + "1", + ] + rejected = CliRunner().invoke(app, arguments) + assert rejected.exit_code == 1, rejected.output + assert "HTTPS" in rejected.output + assert not output.exists() + assert not server.calls + + accepted = CliRunner().invoke(app, [*arguments, "--allow-insecure-http"]) + assert accepted.exit_code == 0, accepted.output + assert len(server.calls) == 1 + assert server.calls[0]["authorization"] == "Bearer local-test-token" + expected_path = "/v1/messages" if provider == "anthropic-compatible" else "/chat/completions" + assert server.calls[0]["path"] == expected_path + manifest = json.loads((output / "reader-manifest.json").read_text()) + assert manifest["reader"]["allow_insecure_http"] is True + summary = json.loads((output / "reader-summary.json").read_text()) + assert summary["succeeded"] == 1 + assert summary["failed"] == 0 + assert all("local-test-token" not in path.read_text() for path in output.iterdir()) + + +@pytest.mark.parametrize("reader_type", [AnthropicCompatibleReader, DeepSeekOpenAIReader]) +@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) +def test_reader_does_not_forward_credentials_on_redirect(reader_type: Any, status: int, model_http_server: Any) -> None: + origin, destination = model_http_server(), model_http_server() + origin.redirect_url = destination.url + "/steal-token" + origin.redirect_status = status + reader = reader_type( + origin.url, + token="local-test-token", + model="model", + max_tokens=10, + temperature=0, + timeout_seconds=2, + allow_insecure_http=True, + ) + with pytest.raises(ReaderSmokeError, match=f"HTTP {status}"): + reader.complete(system="system", content=[{"type": "text", "text": "question"}]) + assert len(origin.calls) == 1 + assert not destination.calls diff --git a/evaluation/tests/unit/test_longmemeval_v2_replay_score.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_replay_score.py similarity index 87% rename from evaluation/tests/unit/test_longmemeval_v2_replay_score.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_replay_score.py index 3329dc6398..2e774df654 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_replay_score.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_replay_score.py @@ -20,8 +20,11 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2 import replay_score -from powercontext_eval.benchmarks.longmemeval_v2.replay_score import ReplayScoreError, replay_score_smoke +from powercontext_eval_longmemeval_v2 import replay_score +from powercontext_eval_longmemeval_v2.replay_score import ( + ReplayScoreError, + replay_score_smoke, +) class FakeMetrics: @@ -86,13 +89,22 @@ def test_replay_score_uses_saved_judge_label_without_network(monkeypatch: pytest assert summary["reader_calls"] == 0 assert summary["judge_calls"] == 0 results = [json.loads(line) for line in result.results_path.read_text(encoding="utf-8").splitlines()] - assert [row["score_mode"] for row in results] == ["deterministic_replay", "llm_judge_replay"] + assert [row["score_mode"] for row in results] == [ + "deterministic_replay", + "llm_judge_replay", + ] assert all("reference_answer" not in row for row in results) -def test_replay_score_refuses_to_overwrite_before_loading_inputs(tmp_path: Path) -> None: +def test_replay_score_refuses_to_overwrite_before_loading_inputs( + tmp_path: Path, +) -> None: output = tmp_path / "replay" output.mkdir() with pytest.raises(ReplayScoreError, match="Refusing to overwrite"): - replay_score_smoke(score_dir=tmp_path / "missing", harness_root=tmp_path / "harness", output_dir=output) + replay_score_smoke( + score_dir=tmp_path / "missing", + harness_root=tmp_path / "harness", + output_dir=output, + ) diff --git a/evaluation/tests/unit/test_longmemeval_v2_report.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_report.py similarity index 86% rename from evaluation/tests/unit/test_longmemeval_v2_report.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_report.py index 5f32b363ae..2efba87858 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_report.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_report.py @@ -20,12 +20,29 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2.report import REPORT_BANNER, ReportError, build_report +from powercontext_eval_longmemeval_v2.report import ( + REPORT_BANNER, + ReportError, + build_report, +) GOLD_SENTINEL = "the gold reference answer that must never reach a report" -SECRET_PATTERNS = (re.compile(r"sk-[A-Za-z0-9_-]{8,}"), re.compile(r"eyJ[A-Za-z0-9_-]{10,}")) +SECRET_PATTERNS = ( + re.compile(r"sk-[A-Za-z0-9_-]{8,}"), + re.compile(r"eyJ[A-Za-z0-9_-]{10,}"), +) FORBIDDEN_KEYS = frozenset( - {"reference_answer", "gold", "gold_answer", "api_key", "auth_token", "token", "token_env", "password", "secret"} + { + "reference_answer", + "gold", + "gold_answer", + "api_key", + "auth_token", + "token", + "token_env", + "password", + "secret", + } ) @@ -85,7 +102,10 @@ def _successful_run(root: Path) -> Path: ) _write_jsonl( run / "02-retrieval" / "adapter-audit.jsonl", - [{"operation": "ingest", "timings_ms": {"total": 4.0}}, {"operation": "query", "timings_ms": {"total": 9.0}}], + [ + {"operation": "ingest", "timings_ms": {"total": 4.0}}, + {"operation": "query", "timings_ms": {"total": 9.0}}, + ], ) _write_json( run / "02-retrieval" / "summary.json", @@ -99,9 +119,17 @@ def _successful_run(root: Path) -> Path: ) _write_json( run / "03-prepare" / "prepare-summary.json", - {"question_count": 10, "failed": 0, "memory_context_tokens": 1234, "elapsed_ms": 7.0}, + { + "question_count": 10, + "failed": 0, + "memory_context_tokens": 1234, + "elapsed_ms": 7.0, + }, + ) + _write_jsonl( + run / "04-reader" / "reader-outputs.jsonl", + [{"question_id": "q0", "reader_latency_ms": 5.0}] * 10, ) - _write_jsonl(run / "04-reader" / "reader-outputs.jsonl", [{"question_id": "q0", "reader_latency_ms": 5.0}] * 10) _write_json( run / "04-reader" / "reader-summary.json", { @@ -126,7 +154,12 @@ def _successful_run(root: Path) -> Path: "label": 0, "judge_latency_ms": 4.0, }, - {"question_id": "q2", "evaluator": "llm_gotchas_checker", "label": 1, "judge_latency_ms": 5.0}, + { + "question_id": "q2", + "evaluator": "llm_gotchas_checker", + "label": 1, + "judge_latency_ms": 5.0, + }, ], ) _write_json( @@ -143,17 +176,31 @@ def _successful_run(root: Path) -> Path: ) _write_json( run / "06-replay" / "replay-summary.json", - {"question_count": 10, "correct": 4, "incorrect": 6, "failed": 0, "accuracy": 0.4}, + { + "question_count": 10, + "correct": 4, + "incorrect": 6, + "failed": 0, + "accuracy": 0.4, + }, ) # The score stage keeps local reference answers for replay. A report must never read or copy them. _write_jsonl( run / "05-score" / "scoring-inputs.local.jsonl", - [{"question_id": "q0", "reference_answer": GOLD_SENTINEL, "api_key": "sk-abcdefghijklmnop"}], + [ + { + "question_id": "q0", + "reference_answer": GOLD_SENTINEL, + "api_key": "sk-abcdefghijklmnop", + } + ], ) return run -def test_report_summarizes_a_successful_run_from_saved_artifacts(tmp_path: Path) -> None: +def test_report_summarizes_a_successful_run_from_saved_artifacts( + tmp_path: Path, +) -> None: run = _successful_run(tmp_path) result = build_report(run_dir=run) @@ -170,7 +217,12 @@ def test_report_summarizes_a_successful_run_from_saved_artifacts(tmp_path: Path) "task_lens": None, } assert report["question_count"] == 10 - assert report["accuracy"] == {"correct": 4, "incorrect": 6, "failed": 0, "value": 0.4} + assert report["accuracy"] == { + "correct": 4, + "incorrect": 6, + "failed": 0, + "value": 0.4, + } assert report["latency_ms"] == { "ingest": 4.0, "retrieval": 20.0, @@ -178,7 +230,12 @@ def test_report_summarizes_a_successful_run_from_saved_artifacts(tmp_path: Path) "reader": 50.0, "judge": 12.0, } - assert report["context"] == {"items": 30, "bytes": 900, "tokens": 1234, "citations_available": 12} + assert report["context"] == { + "items": 30, + "bytes": 900, + "tokens": 1234, + "citations_available": 12, + } assert report["usage"] == { "ingestion_tokens": 0, "ingestion_tokens_note": ( @@ -208,7 +265,12 @@ def test_report_summarizes_a_successful_run_from_saved_artifacts(tmp_path: Path) "the Judge made 1 model call(s) but recorded no priced cost" ), } - assert report["abstention"] == {"count": 2, "correct": 1, "incorrect": 1, "unavailable": False} + assert report["abstention"] == { + "count": 2, + "correct": 1, + "incorrect": 1, + "unavailable": False, + } assert report["failures"] == { "configuration": 0, "infrastructure": 0, @@ -225,7 +287,7 @@ def test_report_summarizes_a_successful_run_from_saved_artifacts(tmp_path: Path) assert "Accuracy: 4/10 (0.4)" in markdown assert "Arm: current-memory-hybrid-v1" in markdown assert "not a complete benchmark result" in markdown - assert "evaluation/docs/longmemeval-v2-full-run.md" in markdown + assert "evaluation/memory/longmemeval_v2/docs/longmemeval-v2-full-run.md" in markdown def test_report_uses_relative_latency_from_saved_artifacts_only(tmp_path: Path) -> None: @@ -237,16 +299,25 @@ def test_report_uses_relative_latency_from_saved_artifacts_only(tmp_path: Path) assert report["latency_ms"]["retrieval"] == 20.0 # ten saved queries at 2 ms each -def test_report_counts_failures_by_class_from_run_and_stage_artifacts(tmp_path: Path) -> None: +def test_report_counts_failures_by_class_from_run_and_stage_artifacts( + tmp_path: Path, +) -> None: run = _successful_run(tmp_path) _write_jsonl( run / "failures.jsonl", [ - {"phase": "reader", "error_class": "configuration", "summary": "missing key"}, + { + "phase": "reader", + "error_class": "configuration", + "summary": "missing key", + }, {"phase": "reader", "error_class": "generation", "summary": "http 500"}, ], ) - _write_jsonl(run / "04-reader" / "reader-failures.jsonl", [{"question_id": "q1", "phase": "reader"}]) + _write_jsonl( + run / "04-reader" / "reader-failures.jsonl", + [{"question_id": "q1", "phase": "reader"}], + ) _write_jsonl( run / "02-retrieval" / "failures.jsonl", [ @@ -305,7 +376,10 @@ def _costed_run(tmp_path: Path, *, reader_cost: float | None, judge_cost: float manifest["modes"] = {"reader": True, "score": True} manifest["reader"] = {"provider": "deepseek-openai", "model": "deepseek-flash"} manifest["judge"] = {"provider": "deepseek-openai", "model": "deepseek-flash"} - manifest["cost_policy"] = {"reader": PRICE_POLICY_RECORD, "judge": PRICE_POLICY_RECORD} + manifest["cost_policy"] = { + "reader": PRICE_POLICY_RECORD, + "judge": PRICE_POLICY_RECORD, + } _write_json(run / "run-manifest.json", manifest) reader_summary = json.loads((run / "04-reader" / "reader-summary.json").read_text(encoding="utf-8")) reader_summary["cost"] = _stage_cost_record(cost_usd=reader_cost) @@ -317,7 +391,9 @@ def _costed_run(tmp_path: Path, *, reader_cost: float | None, judge_cost: float return run -def test_report_sums_stage_costs_recorded_under_one_price_policy(tmp_path: Path) -> None: +def test_report_sums_stage_costs_recorded_under_one_price_policy( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) result = build_report(run_dir=run) @@ -333,7 +409,9 @@ def test_report_sums_stage_costs_recorded_under_one_price_policy(tmp_path: Path) assert "total_usd=0.000637" in markdown -def test_report_keeps_the_total_null_when_an_executed_judge_cost_block_is_missing(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_an_executed_judge_cost_block_is_missing( + tmp_path: Path, +) -> None: """A Judge that ran without a recorded cost block must not be silently excluded.""" run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) @@ -348,7 +426,9 @@ def test_report_keeps_the_total_null_when_an_executed_judge_cost_block_is_missin assert "Judge made 3 model call(s) but recorded no priced cost" in report["usage"]["estimated_cost_note"] -def test_report_keeps_the_total_null_when_an_executed_reader_cost_block_is_missing(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_an_executed_reader_cost_block_is_missing( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) reader_summary = json.loads((run / "04-reader" / "reader-summary.json").read_text(encoding="utf-8")) del reader_summary["cost"] @@ -360,7 +440,9 @@ def test_report_keeps_the_total_null_when_an_executed_reader_cost_block_is_missi assert "Reader stage ran but its summary recorded no cost block" in report["usage"]["estimated_cost_note"] -def test_report_keeps_the_total_null_when_a_configured_reader_summary_is_absent(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_a_configured_reader_summary_is_absent( + tmp_path: Path, +) -> None: """A manifest that configured the Reader but kept no summary must not be totalled.""" run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) @@ -374,7 +456,9 @@ def test_report_keeps_the_total_null_when_a_configured_reader_summary_is_absent( ) -def test_report_keeps_the_total_null_when_an_executed_stage_was_unpriced(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_an_executed_stage_was_unpriced( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=None, judge_cost=0.000147) result = build_report(run_dir=run) @@ -385,7 +469,9 @@ def test_report_keeps_the_total_null_when_an_executed_stage_was_unpriced(tmp_pat assert "the Reader recorded non-zero usage without a priced cost" in report["usage"]["estimated_cost_note"] -def test_report_keeps_the_total_null_when_stage_costs_use_different_policy_revisions(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_stage_costs_use_different_policy_revisions( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) score_summary = json.loads((run / "05-score" / "score-summary.json").read_text(encoding="utf-8")) score_summary["judge_cost"]["price_policy_revision"] = "deepseek-public-list-2026-10" @@ -397,7 +483,9 @@ def test_report_keeps_the_total_null_when_stage_costs_use_different_policy_revis assert "configured price policy field price_policy_revision" in report["usage"]["estimated_cost_note"] -def test_report_keeps_the_total_null_when_the_recorded_model_is_not_the_configured_model(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_the_recorded_model_is_not_the_configured_model( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) score_summary = json.loads((run / "05-score" / "score-summary.json").read_text(encoding="utf-8")) score_summary["judge_cost"]["model"] = "deepseek-v4-pro" @@ -409,7 +497,9 @@ def test_report_keeps_the_total_null_when_the_recorded_model_is_not_the_configur assert "not the configured 'deepseek-flash'" in report["usage"]["estimated_cost_note"] -def test_report_totals_a_judge_that_made_no_model_call_as_zero_judge_cost(tmp_path: Path) -> None: +def test_report_totals_a_judge_that_made_no_model_call_as_zero_judge_cost( + tmp_path: Path, +) -> None: """A score stage with no LLM-judge questions owes no Judge cost and must not block the total.""" run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=None) @@ -434,7 +524,9 @@ def test_report_totals_a_judge_that_made_no_model_call_as_zero_judge_cost(tmp_pa ) -def test_report_ignores_a_zero_call_zero_usage_judge_cost_when_its_policy_differs(tmp_path: Path) -> None: +def test_report_ignores_a_zero_call_zero_usage_judge_cost_when_its_policy_differs( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.0) score_summary = json.loads((run / "05-score" / "score-summary.json").read_text(encoding="utf-8")) score_summary["judge_calls"] = 0 @@ -456,7 +548,9 @@ def test_report_ignores_a_zero_call_zero_usage_judge_cost_when_its_policy_differ assert report["usage"]["estimated_cost_usd"] == 0.00049 -def test_report_rejects_a_zero_call_judge_with_a_priced_nonzero_usage(tmp_path: Path) -> None: +def test_report_rejects_a_zero_call_judge_with_a_priced_nonzero_usage( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) score_summary = json.loads((run / "05-score" / "score-summary.json").read_text(encoding="utf-8")) score_summary["judge_calls"] = 0 @@ -469,7 +563,9 @@ def test_report_rejects_a_zero_call_judge_with_a_priced_nonzero_usage(tmp_path: assert "despite zero recorded model calls" in report["usage"]["estimated_cost_note"] -def test_report_rejects_a_priced_cost_that_does_not_match_its_configured_provider(tmp_path: Path) -> None: +def test_report_rejects_a_priced_cost_that_does_not_match_its_configured_provider( + tmp_path: Path, +) -> None: run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=0.000147) reader_summary = json.loads((run / "04-reader" / "reader-summary.json").read_text(encoding="utf-8")) reader_summary["cost"]["provider"] = "anthropic-compatible" @@ -482,7 +578,9 @@ def test_report_rejects_a_priced_cost_that_does_not_match_its_configured_provide assert "not the configured 'deepseek-openai'" in report["usage"]["estimated_cost_note"] -def test_report_keeps_the_total_null_when_a_judge_without_calls_still_recorded_usage(tmp_path: Path) -> None: +def test_report_keeps_the_total_null_when_a_judge_without_calls_still_recorded_usage( + tmp_path: Path, +) -> None: """A block claiming no Judge call but non-zero usage is self-contradictory, so it blocks the total.""" run = _costed_run(tmp_path, reader_cost=0.00049, judge_cost=None) @@ -531,21 +629,39 @@ def test_report_covers_a_run_that_stopped_after_a_failed_phase(tmp_path: Path) - assert report["accuracy"] is None assert report["usage"]["reader_input_tokens"] is None assert report["latency_ms"]["reader"] is None - assert report["abstention"] == {"count": 0, "correct": None, "incorrect": None, "unavailable": True} + assert report["abstention"] == { + "count": 0, + "correct": None, + "incorrect": None, + "unavailable": True, + } assert report["artifacts"]["score"] is None markdown = result.markdown_path.read_text(encoding="utf-8") assert "Accuracy: unavailable" in markdown assert markdown.splitlines()[0] == REPORT_BANNER -def test_report_covers_a_model_free_run_without_inventing_accuracy(tmp_path: Path) -> None: +def test_report_covers_a_model_free_run_without_inventing_accuracy( + tmp_path: Path, +) -> None: run = tmp_path / "model-free" - _write_json(run / "run-manifest.json", {"classification": "smoke-subset", "run_id": "model-free"}) + _write_json( + run / "run-manifest.json", + {"classification": "smoke-subset", "run_id": "model-free"}, + ) _write_json( run / "run-summary.json", - {"classification": "smoke-subset", "status": "partial", "question_count": 10, "accuracy": None}, + { + "classification": "smoke-subset", + "status": "partial", + "question_count": 10, + "accuracy": None, + }, + ) + _write_json( + run / "02-retrieval" / "summary.json", + {"question_count": 10, "failed": 0, "context_bytes": 10}, ) - _write_json(run / "02-retrieval" / "summary.json", {"question_count": 10, "failed": 0, "context_bytes": 10}) result = build_report(run_dir=run) report = json.loads(result.report_path.read_text(encoding="utf-8")) @@ -600,10 +716,18 @@ def test_report_rejects_a_missing_run_directory(tmp_path: Path) -> None: def test_report_of_a_run_without_an_arm_records_no_arm(tmp_path: Path) -> None: run = tmp_path / "legacy-run" - _write_json(run / "run-manifest.json", {"classification": "smoke-subset", "run_id": "legacy"}) + _write_json( + run / "run-manifest.json", + {"classification": "smoke-subset", "run_id": "legacy"}, + ) _write_json( run / "run-summary.json", - {"classification": "smoke-subset", "status": "partial", "question_count": 10, "accuracy": None}, + { + "classification": "smoke-subset", + "status": "partial", + "question_count": 10, + "accuracy": None, + }, ) result = build_report(run_dir=run) @@ -613,7 +737,9 @@ def test_report_of_a_run_without_an_arm_records_no_arm(tmp_path: Path) -> None: assert "Arm: unavailable" in markdown -def test_report_counts_hybrid_mode_mismatch_as_integrity_failures(tmp_path: Path) -> None: +def test_report_counts_hybrid_mode_mismatch_as_integrity_failures( + tmp_path: Path, +) -> None: run = _successful_run(tmp_path) _write_jsonl( run / "02-retrieval" / "failures.jsonl", diff --git a/evaluation/tests/unit/test_longmemeval_v2_retrieval_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_retrieval_smoke.py similarity index 95% rename from evaluation/tests/unit/test_longmemeval_v2_retrieval_smoke.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_retrieval_smoke.py index 024170ede1..a1a801739c 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_retrieval_smoke.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_retrieval_smoke.py @@ -23,14 +23,14 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2 import retrieval_smoke -from powercontext_eval.benchmarks.longmemeval_v2.arms import ExperimentArmError -from powercontext_eval.benchmarks.longmemeval_v2.retrieval_smoke import ( +from powercontext_eval_longmemeval_v2 import retrieval_smoke +from powercontext_eval_longmemeval_v2.arms import ExperimentArmError +from powercontext_eval_longmemeval_v2.retrieval_smoke import ( RetrievalCapabilityError, RetrievalSmokeError, run_retrieval_smoke, ) -from powercontext_eval.benchmarks.longmemeval_v2.smoke import PreparedSmokeRun +from powercontext_eval_longmemeval_v2.smoke import PreparedSmokeRun class FakeRetrievalRuntime: @@ -73,7 +73,11 @@ def remember_memory(self, payload: Mapping[str, object]) -> Mapping[str, object] return { "entry": { "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": index}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": index, + }, "entry_id": f"entry-{index}", "entry_version_id": f"entry-{index}-v1", } @@ -91,7 +95,11 @@ def search_memory(self, payload: Mapping[str, object]) -> Mapping[str, object]: { "text": text, "citation": { - "memory_ref": {"family": "memory", "artifact_id": "memory", "revision": 1}, + "memory_ref": { + "family": "memory", + "artifact_id": "memory", + "revision": 1, + }, "entry_id": f"hit-{scope_id}", "entry_version_id": f"hit-{scope_id}-v1", }, @@ -222,7 +230,11 @@ def run( (data_root / "haystacks" / "lme_v2_small.json").read_bytes() ).hexdigest(), } - monkeypatch.setattr(retrieval_smoke, "load_dataset_lock", lambda path: SimpleNamespace(file_digests=digests)) + monkeypatch.setattr( + retrieval_smoke, + "load_dataset_lock", + lambda path: SimpleNamespace(file_digests=digests), + ) arguments: dict[str, Any] = { "data_root": data_root, "dataset_lock": tmp_path / "dataset-lock.json", @@ -260,7 +272,10 @@ def guarded_read_text(path: Path, encoding: str | None = None, errors: str | Non assert len(runtime.scopes) == 3 assert runtime.scopes[1]["parent_scope_id"] == "scope-1" assert runtime.scopes[2]["parent_scope_id"] == "scope-1" - assert {request["scope_id"] for request in runtime.captures} == {"scope-2", "scope-3"} + assert {request["scope_id"] for request in runtime.captures} == { + "scope-2", + "scope-3", + } assert all("decoy-evidence" not in str(request["content"]) for request in runtime.captures) results = read_jsonl(output_dir / "retrieval-results.jsonl") assert [result["question_id"] for result in results] == ["question-a", "question-b"] @@ -321,7 +336,11 @@ def test_repeated_runs_with_the_same_run_id_and_haystack_get_isolated_scopes( (data_root / "haystacks" / "lme_v2_small.json").read_bytes() ).hexdigest(), } - monkeypatch.setattr(retrieval_smoke, "load_dataset_lock", lambda path: SimpleNamespace(file_digests=digests)) + monkeypatch.setattr( + retrieval_smoke, + "load_dataset_lock", + lambda path: SimpleNamespace(file_digests=digests), + ) manifests = [] for arm, directory in ( ("current-memory-fts-v1", "output-fts"), diff --git a/evaluation/tests/unit/test_longmemeval_v2_run_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_run_smoke.py similarity index 85% rename from evaluation/tests/unit/test_longmemeval_v2_run_smoke.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_run_smoke.py index 069898c314..9b98b9db9c 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_run_smoke.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_run_smoke.py @@ -19,23 +19,37 @@ import pytest -from powercontext_eval.benchmarks.longmemeval_v2.adapter import ( +from powercontext_eval_longmemeval_v2.adapter import ( PowerContextMemoryAdapterError, PowerContextMemoryModeError, ) -from powercontext_eval.benchmarks.longmemeval_v2.arms import CURRENT_MEMORY_HYBRID, ExperimentArmError -from powercontext_eval.benchmarks.longmemeval_v2.costs import parse_cost_policy -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import PreparedPromptRun, PrepareSmokeError -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ReaderSmokeError, ReaderSmokeRun -from powercontext_eval.benchmarks.longmemeval_v2.replay_score import ReplayScoreRun -from powercontext_eval.benchmarks.longmemeval_v2.retrieval_smoke import ( +from powercontext_eval_longmemeval_v2.arms import ( + CURRENT_MEMORY_HYBRID, + ExperimentArmError, +) +from powercontext_eval_longmemeval_v2.costs import parse_cost_policy +from powercontext_eval_longmemeval_v2.prepare_smoke import ( + PreparedPromptRun, + PrepareSmokeError, +) +from powercontext_eval_longmemeval_v2.reader_smoke import ( + ReaderSmokeError, + ReaderSmokeRun, +) +from powercontext_eval_longmemeval_v2.replay_score import ReplayScoreRun +from powercontext_eval_longmemeval_v2.retrieval_smoke import ( RetrievalCapabilityError, RetrievalSmokeError, RetrievalSmokeRun, ) -from powercontext_eval.benchmarks.longmemeval_v2.run_smoke import RunSmokeError, SmokeStages, classify_error, run_smoke -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ScoreSmokeRun -from powercontext_eval.benchmarks.longmemeval_v2.smoke import PreparedSmokeRun +from powercontext_eval_longmemeval_v2.run_smoke import ( + RunSmokeError, + SmokeStages, + classify_error, + run_smoke, +) +from powercontext_eval_longmemeval_v2.score_smoke import ScoreSmokeRun +from powercontext_eval_longmemeval_v2.smoke import PreparedSmokeRun SECRET = "sk-smoke-test-secret-value" READER_PRICE_POLICY = { @@ -85,7 +99,11 @@ def preflight(self, **arguments: Any) -> PreparedSmokeRun: output = self._record("preflight", arguments) manifest = output / "manifest.json" manifest.write_text("{}\n", encoding="utf-8") - return PreparedSmokeRun(output_dir=output, manifest_path=manifest, subset_path=output / "subset.json") + return PreparedSmokeRun( + output_dir=output, + manifest_path=manifest, + subset_path=output / "subset.json", + ) def retrieval(self, **arguments: Any) -> RetrievalSmokeRun: output = self._record("retrieval", arguments) @@ -95,7 +113,10 @@ def retrieval(self, **arguments: Any) -> RetrievalSmokeRun: { "question_id": f"q{index}", "status": "succeeded", - "memory_context": [{"type": "text", "value": "a"}, {"type": "text", "value": "b"}], + "memory_context": [ + {"type": "text", "value": "a"}, + {"type": "text", "value": "b"}, + ], "timings_ms": {"search": 1.0, "format": 0.5, "total": 2.0}, } for index in range(10) @@ -103,7 +124,17 @@ def retrieval(self, **arguments: Any) -> RetrievalSmokeRun: ) _write( output / "adapter-audit.jsonl", - [{"operation": "ingest", "timings_ms": {"source_capture": 1.0, "memory_remember": 1.0, "total": 3.0}}] * 4, + [ + { + "operation": "ingest", + "timings_ms": { + "source_capture": 1.0, + "memory_remember": 1.0, + "total": 3.0, + }, + } + ] + * 4, ) _write_json( output / "summary.json", @@ -147,7 +178,10 @@ def prepare(self, **arguments: Any) -> PreparedPromptRun: def reader(self, **arguments: Any) -> ReaderSmokeRun: output = self._record("reader", arguments) - _write(output / "reader-outputs.jsonl", [{"question_id": "q0", "reader_latency_ms": 5.0}] * 10) + _write( + output / "reader-outputs.jsonl", + [{"question_id": "q0", "reader_latency_ms": 5.0}] * 10, + ) _write_json( output / "reader-summary.json", { @@ -267,22 +301,50 @@ def _run(tmp_path: Path, harness: _Harness, **overrides: Any) -> Any: return run_smoke(**arguments) -def test_run_smoke_runs_every_phase_in_order_and_writes_a_report(tmp_path: Path) -> None: +def test_run_smoke_runs_every_phase_in_order_and_writes_a_report( + tmp_path: Path, +) -> None: harness = _Harness() result = _run(tmp_path, harness) assert result.status == "completed" - assert harness.calls == ["preflight", "retrieval", "prepare", "reader", "score", "replay"] - assert result.completed_phases == ("preflight", "retrieval", "prepare", "reader", "score", "replay") + assert harness.calls == [ + "preflight", + "retrieval", + "prepare", + "reader", + "score", + "replay", + ] + assert result.completed_phases == ( + "preflight", + "retrieval", + "prepare", + "reader", + "score", + "replay", + ) assert result.skipped_phases == () assert result.failed_phase is None - for name in ("01-inputs", "02-retrieval", "03-prepare", "04-reader", "05-score", "06-replay"): + for name in ( + "01-inputs", + "02-retrieval", + "03-prepare", + "04-reader", + "05-score", + "06-replay", + ): assert (tmp_path / "run" / name).is_dir() summary = json.loads(result.summary_path.read_text(encoding="utf-8")) assert summary["classification"] == "smoke-subset" assert summary["status"] == "completed" - assert summary["accuracy"] == {"correct": 4, "incorrect": 6, "failed": 0, "value": 0.4} + assert summary["accuracy"] == { + "correct": 4, + "incorrect": 6, + "failed": 0, + "value": 0.4, + } assert summary["question_count"] == 10 assert summary["artifacts"]["retrieval"] == "02-retrieval" assert (tmp_path / "run" / "report.json").is_file() @@ -311,7 +373,10 @@ def test_run_smoke_records_the_manifest_before_any_stage_runs(tmp_path: Path) -> assert manifest["reader"]["model"] == "deepseek-flash" assert manifest["reader"]["token_env"] == "DEEPSEEK_API_KEY" assert manifest["judge"]["token_env"] == "DEEPSEEK_API_KEY" - assert manifest["revisions"] == {"powercontext": "pc-sha", "integration": "integration-sha"} + assert manifest["revisions"] == { + "powercontext": "pc-sha", + "integration": "integration-sha", + } assert manifest["phases"]["replay"] == "06-replay" assert manifest["privacy"]["credentials"] == "resolved-from-environment-at-runtime-never-recorded" assert manifest["privacy"]["reference_answers"] == "never-read-by-the-adapter-retrieval-prepare-or-reader-stages" @@ -321,7 +386,9 @@ def test_run_smoke_records_the_manifest_before_any_stage_runs(tmp_path: Path) -> assert harness.retrieval_arguments["experiment_arm"].arm_id == "current-memory-fts-v1" -def test_run_smoke_refuses_an_existing_output_directory_before_any_stage(tmp_path: Path) -> None: +def test_run_smoke_refuses_an_existing_output_directory_before_any_stage( + tmp_path: Path, +) -> None: harness = _Harness() (tmp_path / "run").mkdir() @@ -341,7 +408,9 @@ def test_run_smoke_rejects_a_base_url_that_carries_credentials(tmp_path: Path) - assert not (tmp_path / "run").exists() -def test_run_smoke_stops_after_a_failed_phase_and_keeps_prior_artifacts(tmp_path: Path) -> None: +def test_run_smoke_stops_after_a_failed_phase_and_keeps_prior_artifacts( + tmp_path: Path, +) -> None: harness = _Harness(fail_on="prepare") result = _run(tmp_path, harness) @@ -379,7 +448,9 @@ def test_run_smoke_classifies_a_reader_failure_as_generation(tmp_path: Path) -> assert result.completed_phases == ("preflight", "retrieval", "prepare") -def test_run_smoke_skips_model_phases_without_inventing_accuracy(tmp_path: Path) -> None: +def test_run_smoke_skips_model_phases_without_inventing_accuracy( + tmp_path: Path, +) -> None: harness = _Harness() result = _run(tmp_path, harness, skip_reader=True) @@ -466,7 +537,11 @@ def test_run_smoke_does_not_publish_completed_while_retrieval_is_still_pending( harness = _Harness() observed: list[str] = [] - monkeypatch.setattr(harness, "retrieval", _observing_stage(harness.retrieval, observed, tmp_path / "run")) + monkeypatch.setattr( + harness, + "retrieval", + _observing_stage(harness.retrieval, observed, tmp_path / "run"), + ) result = _run(tmp_path, harness, skip_reader=True) assert observed == ["partial"] @@ -512,7 +587,9 @@ def wrapper(**arguments: Any) -> Any: return wrapper -def test_run_smoke_forwards_the_selected_experiment_arm_to_retrieval(tmp_path: Path) -> None: +def test_run_smoke_forwards_the_selected_experiment_arm_to_retrieval( + tmp_path: Path, +) -> None: harness = _Harness() result = _run(tmp_path, harness, skip_reader=True, experiment_arm="current-memory-hybrid-v1") @@ -543,14 +620,21 @@ def test_run_smoke_classifies_mode_and_capability_errors(tmp_path: Path) -> None assert classify_error(PowerContextMemoryAdapterError("transport failed")) == "infrastructure" -def test_run_smoke_records_and_forwards_separate_reader_and_judge_price_policies(tmp_path: Path) -> None: +def test_run_smoke_records_and_forwards_separate_reader_and_judge_price_policies( + tmp_path: Path, +) -> None: """Reader and Judge may run different models, so each stage carries its own policy.""" harness = _Harness() reader_policy = parse_cost_policy(READER_PRICE_POLICY) judge_policy = parse_cost_policy({**READER_PRICE_POLICY, "model": "deepseek-v4-pro"}) assert reader_policy is not None and judge_policy is not None - result = _run(tmp_path, harness, reader_price_policy=reader_policy, judge_price_policy=judge_policy) + result = _run( + tmp_path, + harness, + reader_price_policy=reader_policy, + judge_price_policy=judge_policy, + ) manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) assert manifest["cost_policy"] == { @@ -563,7 +647,9 @@ def test_run_smoke_records_and_forwards_separate_reader_and_judge_price_policies assert harness.score_arguments["judge_price_policy"] is judge_policy -def test_run_smoke_keeps_both_cost_policies_null_when_no_prices_are_configured(tmp_path: Path) -> None: +def test_run_smoke_keeps_both_cost_policies_null_when_no_prices_are_configured( + tmp_path: Path, +) -> None: harness = _Harness() result = _run(tmp_path, harness) @@ -573,3 +659,20 @@ def test_run_smoke_keeps_both_cost_policies_null_when_no_prices_are_configured(t assert harness.reader_arguments["price_policy"] is None assert harness.score_arguments is not None assert harness.score_arguments["judge_price_policy"] is None + + +@pytest.mark.parametrize("allow_http", [False, True]) +def test_run_smoke_records_and_passes_http_opt_in_for_each_role(tmp_path: Path, allow_http: bool) -> None: + harness = _Harness() + result = _run( + tmp_path, + harness, + reader_allow_insecure_http=allow_http, + judge_allow_insecure_http=allow_http, + ) + assert result.status == "completed" + manifest = json.loads(result.manifest_path.read_text()) + assert manifest["reader"]["allow_insecure_http"] is allow_http + assert manifest["judge"]["allow_insecure_http"] is allow_http + assert harness.reader_arguments["allow_insecure_http"] is allow_http + assert harness.score_arguments["judge_allow_insecure_http"] is allow_http diff --git a/evaluation/tests/unit/test_longmemeval_v2_score_smoke.py b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_score_smoke.py similarity index 77% rename from evaluation/tests/unit/test_longmemeval_v2_score_smoke.py rename to evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_score_smoke.py index 809a25ae57..0bd18fdc1d 100644 --- a/evaluation/tests/unit/test_longmemeval_v2_score_smoke.py +++ b/evaluation/memory/longmemeval_v2/tests/test_longmemeval_v2_score_smoke.py @@ -21,12 +21,17 @@ from typing import Any import pytest +from typer.testing import CliRunner -from powercontext_eval.benchmarks.longmemeval_v2 import score_smoke -from powercontext_eval.benchmarks.longmemeval_v2.catalog import SmokeSelection -from powercontext_eval.benchmarks.longmemeval_v2.costs import ModelPricePolicy -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ReaderResponseError -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ScoreSmokeError, run_score_smoke +from powercontext_eval_longmemeval_v2 import score_smoke +from powercontext_eval_longmemeval_v2.catalog import SmokeSelection +from powercontext_eval_longmemeval_v2.cli import app +from powercontext_eval_longmemeval_v2.costs import ModelPricePolicy +from powercontext_eval_longmemeval_v2.reader_smoke import ReaderResponseError +from powercontext_eval_longmemeval_v2.score_smoke import ( + ScoreSmokeError, + run_score_smoke, +) class FakeMetrics: @@ -43,10 +48,16 @@ def score_to_bool(self, value: Any) -> bool: return bool(value) def _build_abstention_judge_messages(self, **kwargs: Any) -> list[dict[str, str]]: - return [{"role": "system", "content": "abstention"}, {"role": "user", "content": kwargs["reference_answer"]}] + return [ + {"role": "system", "content": "abstention"}, + {"role": "user", "content": kwargs["reference_answer"]}, + ] def _build_gotchas_judge_messages(self, **kwargs: Any) -> list[dict[str, str]]: - return [{"role": "system", "content": "gotchas"}, {"role": "user", "content": kwargs["reference_answer"]}] + return [ + {"role": "system", "content": "gotchas"}, + {"role": "user", "content": kwargs["reference_answer"]}, + ] def _parse_llm_binary_judgement(self, text: str) -> tuple[int, str]: assert text == '{"label":1}' @@ -71,7 +82,11 @@ def score_fixture(tmp_path: Path) -> tuple[Path, Path, Path]: reader.mkdir() data.mkdir() reader_manifest = {"classification": "smoke-subset-reader-only", "judge": None} - reader_summary = {"classification": "smoke-subset-reader-only", "question_count": 10, "failed": 0} + reader_summary = { + "classification": "smoke-subset-reader-only", + "question_count": 10, + "failed": 0, + } (reader / "reader-manifest.json").write_text(json.dumps(reader_manifest), encoding="utf-8") (reader / "reader-summary.json").write_text(json.dumps(reader_summary), encoding="utf-8") cases = [] @@ -99,14 +114,24 @@ def score_fixture(tmp_path: Path) -> tuple[Path, Path, Path]: ) manifest = tmp_path / "smoke.json" manifest.write_text( - json.dumps({"schema": "powercontext.longmemeval-v2-smoke.v1", "tier": "small", "cases": cases}), + json.dumps( + { + "schema": "powercontext.longmemeval-v2-smoke.v1", + "tier": "small", + "cases": cases, + } + ), encoding="utf-8", ) return reader, data, manifest def allow_validated_catalog(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(score_smoke, "load_dataset_lock", lambda path: SimpleNamespace(tier="small", file_digests={})) + monkeypatch.setattr( + score_smoke, + "load_dataset_lock", + lambda path: SimpleNamespace(tier="small", file_digests={}), + ) monkeypatch.setattr( score_smoke, "LongMemEvalV2Catalog", @@ -238,7 +263,12 @@ def test_score_smoke_preserves_judge_usage_when_the_judgement_cannot_be_parsed( assert all(failure["error_type"] == "JudgeJudgementError" for failure in failures) assert all( failure["judge_usage"] - == {"input_tokens": 10, "output_tokens": 2, "input_cache_hit_tokens": 4, "input_cache_miss_tokens": 6} + == { + "input_tokens": 10, + "output_tokens": 2, + "input_cache_hit_tokens": 4, + "input_cache_miss_tokens": 6, + } for failure in failures ) assert all(isinstance(failure["judge_latency_ms"], (int, float)) for failure in failures) @@ -302,7 +332,60 @@ def complete(self, *, system: str, content: list[dict[str, object]]) -> Mapping[ assert all(failure["error_type"] == "JudgeJudgementError" for failure in failures) assert all( failure["judge_usage"] - == {"input_tokens": 10, "output_tokens": 2, "input_cache_hit_tokens": 4, "input_cache_miss_tokens": 6} + == { + "input_tokens": 10, + "output_tokens": 2, + "input_cache_hit_tokens": 4, + "input_cache_miss_tokens": 6, + } for failure in failures ) assert all(isinstance(failure["judge_latency_ms"], (int, float)) for failure in failures) + + +def test_score_cli_requires_judge_http_opt_in_and_records_it( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + model_http_server: Any, +) -> None: + reader, data, manifest = score_fixture(tmp_path) + server = model_http_server() + server.text = '{"label":1}' + output = tmp_path / "score" + monkeypatch.setenv("JUDGE_TEST_TOKEN", "local-test-token") + monkeypatch.setattr(score_smoke, "validate_harness_checkout", lambda root: None) + monkeypatch.setattr(score_smoke, "_load_metrics", lambda root: FakeMetrics()) + allow_validated_catalog(monkeypatch) + arguments = [ + "score-smoke", + "--reader-dir", + str(reader), + "--data-root", + str(data), + "--dataset-lock", + str(tmp_path / "lock"), + "--smoke-manifest", + str(manifest), + "--harness-root", + str(tmp_path / "harness"), + "--output-dir", + str(output), + "--judge-base-url", + server.url, + "--judge-token-env", + "JUDGE_TEST_TOKEN", + ] + rejected = CliRunner().invoke(app, arguments) + assert rejected.exit_code == 1, rejected.output + assert "HTTPS" in rejected.output + assert not server.calls + assert not output.exists() + accepted = CliRunner().invoke(app, [*arguments, "--judge-allow-insecure-http"]) + assert accepted.exit_code == 0, accepted.output + assert len(server.calls) == 4 + assert all(call["authorization"] == "Bearer local-test-token" for call in server.calls) + summary = json.loads((output / "score-summary.json").read_text()) + assert summary["failed"] == 0 + assert summary["judge_calls"] == 4 + saved_manifest = json.loads((output / "score-manifest.json").read_text()) + assert saved_manifest["judge"]["allow_insecure_http"] is True diff --git a/evaluation/performance/README.md b/evaluation/performance/README.md new file mode 100644 index 0000000000..5de0452ea8 --- /dev/null +++ b/evaluation/performance/README.md @@ -0,0 +1,11 @@ +# Performance and capacity benchmarks + +These suites measure runtime latency, throughput, storage costs, and capacity limits. +Run their commands from the repository root with the root Python environment. + +| Suite | Purpose | +| --- | --- | +| [Memory capacity](memory_capacity/README.md) | Append latency, storage growth, compaction, and search identity preservation without model calls. | + +Add load, stress, and capacity suites here, keeping each workload in its own directory with its setup and measurement +contract. Put labeled-dataset quality evaluation in [memory](../memory/README.md). diff --git a/evaluation/performance/__init__.py b/evaluation/performance/__init__.py new file mode 100644 index 0000000000..80da555606 --- /dev/null +++ b/evaluation/performance/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Performance and capacity benchmark suites.""" diff --git a/benchmark/memory_capacity/README.md b/evaluation/performance/memory_capacity/README.md similarity index 94% rename from benchmark/memory_capacity/README.md rename to evaluation/performance/memory_capacity/README.md index 76d2f67909..d4c30211fa 100644 --- a/benchmark/memory_capacity/README.md +++ b/evaluation/performance/memory_capacity/README.md @@ -6,8 +6,8 @@ canonical bytes, database bytes, mean append latency, the final 100 appends, aff of FTS hit identities across compaction. It retains historical manifests and entry bodies. ```bash -uv run python -m benchmark.memory_capacity --output .artifacts/memory-capacity/run-01/sqlite.json -uv run python -m benchmark.memory_capacity --backend oceanbase --output .artifacts/memory-capacity/run-01/oceanbase.json +uv run python -m evaluation.performance.memory_capacity --output .artifacts/memory-capacity/run-01/sqlite.json +uv run python -m evaluation.performance.memory_capacity --backend oceanbase --output .artifacts/memory-capacity/run-01/oceanbase.json ``` Choose a new run directory for each measurement. Raw JSON, logs, and databases are local acceptance artifacts under the diff --git a/benchmark/memory_capacity/__init__.py b/evaluation/performance/memory_capacity/__init__.py similarity index 100% rename from benchmark/memory_capacity/__init__.py rename to evaluation/performance/memory_capacity/__init__.py diff --git a/benchmark/memory_capacity/__main__.py b/evaluation/performance/memory_capacity/__main__.py similarity index 100% rename from benchmark/memory_capacity/__main__.py rename to evaluation/performance/memory_capacity/__main__.py diff --git a/evaluation/pyproject.toml b/evaluation/pyproject.toml index 86ee0932db..f89f83cb8a 100644 --- a/evaluation/pyproject.toml +++ b/evaluation/pyproject.toml @@ -28,7 +28,9 @@ dependencies = [ ] [project.scripts] -powercontext-eval = "powercontext_eval.cli:main" +swebench-pro = "powercontext_eval_swebench_pro.cli:main" +longmemeval-v2 = "powercontext_eval_longmemeval_v2.cli:main" +work-continuity = "powercontext_eval_work_continuity.cli:main" [dependency-groups] dev = [ @@ -39,18 +41,26 @@ dev = [ ] [tool.hatch.build.targets.wheel] -packages = ["src/powercontext_eval"] +packages = [ + "coding/swebench_pro/src/powercontext_eval_swebench_pro", + "coding/work_continuity/src/powercontext_eval_work_continuity", + "memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2", +] + +[tool.hatch.build.targets.sdist] +include = ["/coding/swebench_pro", "/coding/work_continuity", "/memory/longmemeval_v2", "/README.md", "/pyproject.toml", "/uv.lock"] +exclude = ["**/.venv", "**/node_modules", "**/dist", "**/__pycache__", "**/.ruff_cache", "**/.pytest_cache", "**/*.tsbuildinfo", "**/test-results", "**/playwright-report"] [tool.ty.environment] python = "./.venv" python-version = "3.11" +[tool.ty.src] +include = ["coding/swebench_pro/src", "coding/work_continuity/src", "memory/longmemeval_v2/src"] + [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["coding/swebench_pro/tests", "coding/work_continuity/tests", "memory/longmemeval_v2/tests"] +addopts = ["--import-mode=importlib"] markers = [ "live: tests that require external services", ] - -[tool.ruff] -target-version = "py311" -line-length = 120 diff --git a/evaluation/skills/README.md b/evaluation/skills/README.md new file mode 100644 index 0000000000..705f44bf36 --- /dev/null +++ b/evaluation/skills/README.md @@ -0,0 +1,11 @@ +# Skill evaluations + +Evaluate how agent Skills affect instruction following and tool selection. Keep each +suite's host setup, fixtures, and evidence requirements in its own directory. + +| Suite | Measures | Entry point | +| --- | --- | --- | +| [skill-up](skill-up/README.md) | Claude Code routing and tool selection with and without the packaged PowerContext Skill, using mocked MCP replies. | `cd evaluation/skills/skill-up` then follow its README. | + +These results describe Skill guidance behavior. They do not establish live backend +acceptance or contribute to coding-task and memory-quality benchmark scores. diff --git a/evaluation/skill-up/.gitattributes b/evaluation/skills/skill-up/.gitattributes similarity index 100% rename from evaluation/skill-up/.gitattributes rename to evaluation/skills/skill-up/.gitattributes diff --git a/evaluation/skill-up/.gitignore b/evaluation/skills/skill-up/.gitignore similarity index 100% rename from evaluation/skill-up/.gitignore rename to evaluation/skills/skill-up/.gitignore diff --git a/evaluation/skill-up/README.md b/evaluation/skills/skill-up/README.md similarity index 98% rename from evaluation/skill-up/README.md rename to evaluation/skills/skill-up/README.md index 7dfc9031dc..bca9033e79 100644 --- a/evaluation/skill-up/README.md +++ b/evaluation/skills/skill-up/README.md @@ -5,7 +5,7 @@ This project evaluates the packaged `powercontext-project-context` Skill with `dd2ff3250074505204b969405bef2cdce4bae428`. It measures **Claude Code + MCP instruction/tool-selection behavior** with deterministic `rule_based` assertions and controlled replies. Model behavior can still vary between runs. -It complements the [integration guidance qualification record](../../docs/en/development/integration-guidance-evaluation.md). +It complements the [integration guidance qualification record](../../../docs/en/development/integration-guidance-evaluation.md). It neither replaces those multi-surface observations nor contributes to the SWE-bench Pro or LoCoMo scores. Offline validation and synthetic tests do not establish real model coverage; retain authenticated runs' full evidence. @@ -17,9 +17,10 @@ or WSL; native Windows also supports model runs with the Bash/PATH setup below. runtime invokes host commands and is not an OS sandbox. Use a disposable evaluation account/workspace: the runner disables hooks and bypasses Claude Code permission prompts. A paid Claude model run executes both arms (12 user turns). -From this directory on Linux x86-64 (Python setup also works on macOS/WSL): +From the repository root on Linux x86-64 (Python setup also works on macOS/WSL): ```bash +cd evaluation/skills/skill-up python3 -m venv .venv . .venv/bin/activate python3 -m pip install -r requirements-dev.txt diff --git a/evaluation/skill-up/evals/cases/empty-search.yaml b/evaluation/skills/skill-up/evals/cases/empty-search.yaml similarity index 100% rename from evaluation/skill-up/evals/cases/empty-search.yaml rename to evaluation/skills/skill-up/evals/cases/empty-search.yaml diff --git a/evaluation/skill-up/evals/cases/explicit-save.yaml b/evaluation/skills/skill-up/evals/cases/explicit-save.yaml similarity index 100% rename from evaluation/skill-up/evals/cases/explicit-save.yaml rename to evaluation/skills/skill-up/evals/cases/explicit-save.yaml diff --git a/evaluation/skill-up/evals/cases/failed-save.yaml b/evaluation/skills/skill-up/evals/cases/failed-save.yaml similarity index 100% rename from evaluation/skill-up/evals/cases/failed-save.yaml rename to evaluation/skills/skill-up/evals/cases/failed-save.yaml diff --git a/evaluation/skill-up/evals/cases/inspect-candidates.yaml b/evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml similarity index 100% rename from evaluation/skill-up/evals/cases/inspect-candidates.yaml rename to evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml diff --git a/evaluation/skill-up/evals/cases/ordinary-coding.yaml b/evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml similarity index 100% rename from evaluation/skill-up/evals/cases/ordinary-coding.yaml rename to evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml diff --git a/evaluation/skill-up/evals/eval.yaml b/evaluation/skills/skill-up/evals/eval.yaml similarity index 100% rename from evaluation/skill-up/evals/eval.yaml rename to evaluation/skills/skill-up/evals/eval.yaml diff --git a/evaluation/skill-up/evals/fixtures/mcp/powercontext-default.json b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json similarity index 100% rename from evaluation/skill-up/evals/fixtures/mcp/powercontext-default.json rename to evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json diff --git a/evaluation/skill-up/evals/fixtures/mcp/powercontext-failed-save.json b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json similarity index 100% rename from evaluation/skill-up/evals/fixtures/mcp/powercontext-failed-save.json rename to evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json diff --git a/evaluation/skill-up/evals/fixtures/mcp/powercontext-http.example.yaml b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-http.example.yaml similarity index 100% rename from evaluation/skill-up/evals/fixtures/mcp/powercontext-http.example.yaml rename to evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-http.example.yaml diff --git a/evaluation/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md similarity index 100% rename from evaluation/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md rename to evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md diff --git a/evaluation/skill-up/harness/plugin-root/scripts/workspace_scope.py b/evaluation/skills/skill-up/harness/plugin-root/scripts/workspace_scope.py similarity index 100% rename from evaluation/skill-up/harness/plugin-root/scripts/workspace_scope.py rename to evaluation/skills/skill-up/harness/plugin-root/scripts/workspace_scope.py diff --git a/evaluation/skill-up/report.py b/evaluation/skills/skill-up/report.py similarity index 100% rename from evaluation/skill-up/report.py rename to evaluation/skills/skill-up/report.py diff --git a/evaluation/skill-up/requirements-dev.txt b/evaluation/skills/skill-up/requirements-dev.txt similarity index 100% rename from evaluation/skill-up/requirements-dev.txt rename to evaluation/skills/skill-up/requirements-dev.txt diff --git a/evaluation/skills/skill-up/ruff.toml b/evaluation/skills/skill-up/ruff.toml new file mode 100644 index 0000000000..2c6e969e7e --- /dev/null +++ b/evaluation/skills/skill-up/ruff.toml @@ -0,0 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +target-version = "py311" +line-length = 120 diff --git a/evaluation/skill-up/run.py b/evaluation/skills/skill-up/run.py similarity index 100% rename from evaluation/skill-up/run.py rename to evaluation/skills/skill-up/run.py diff --git a/evaluation/skill-up/skill-lock.json b/evaluation/skills/skill-up/skill-lock.json similarity index 100% rename from evaluation/skill-up/skill-lock.json rename to evaluation/skills/skill-up/skill-lock.json diff --git a/evaluation/skill-up/sync_skill.py b/evaluation/skills/skill-up/sync_skill.py similarity index 99% rename from evaluation/skill-up/sync_skill.py rename to evaluation/skills/skill-up/sync_skill.py index edf036c6a0..51eb178b00 100644 --- a/evaluation/skill-up/sync_skill.py +++ b/evaluation/skills/skill-up/sync_skill.py @@ -24,7 +24,7 @@ from pathlib import Path PROJECT = Path(__file__).resolve().parent -REPOSITORY = PROJECT.parents[1] +REPOSITORY = PROJECT.parents[2] SOURCE = "integrations/claude-code/plugins/powercontext/skills/powercontext-project-context" VENDOR = PROJECT / "vendor/powercontext-project-context" LOCK = PROJECT / "skill-lock.json" diff --git a/evaluation/skill-up/tests/test_report.py b/evaluation/skills/skill-up/tests/test_report.py similarity index 100% rename from evaluation/skill-up/tests/test_report.py rename to evaluation/skills/skill-up/tests/test_report.py diff --git a/evaluation/skill-up/tests/test_suite.py b/evaluation/skills/skill-up/tests/test_suite.py similarity index 100% rename from evaluation/skill-up/tests/test_suite.py rename to evaluation/skills/skill-up/tests/test_suite.py diff --git a/evaluation/skill-up/validate_suite.py b/evaluation/skills/skill-up/validate_suite.py similarity index 100% rename from evaluation/skill-up/validate_suite.py rename to evaluation/skills/skill-up/validate_suite.py diff --git a/evaluation/skill-up/vendor/powercontext-project-context/SKILL.md b/evaluation/skills/skill-up/vendor/powercontext-project-context/SKILL.md similarity index 100% rename from evaluation/skill-up/vendor/powercontext-project-context/SKILL.md rename to evaluation/skills/skill-up/vendor/powercontext-project-context/SKILL.md diff --git a/evaluation/skill-up/vendor/powercontext-project-context/references/review-publication.md b/evaluation/skills/skill-up/vendor/powercontext-project-context/references/review-publication.md similarity index 100% rename from evaluation/skill-up/vendor/powercontext-project-context/references/review-publication.md rename to evaluation/skills/skill-up/vendor/powercontext-project-context/references/review-publication.md diff --git a/evaluation/skill-up/vendor/powercontext-project-context/references/scope-memory.md b/evaluation/skills/skill-up/vendor/powercontext-project-context/references/scope-memory.md similarity index 100% rename from evaluation/skill-up/vendor/powercontext-project-context/references/scope-memory.md rename to evaluation/skills/skill-up/vendor/powercontext-project-context/references/scope-memory.md diff --git a/evaluation/skill-up/vendor/powercontext-project-context/references/work-handoff.md b/evaluation/skills/skill-up/vendor/powercontext-project-context/references/work-handoff.md similarity index 100% rename from evaluation/skill-up/vendor/powercontext-project-context/references/work-handoff.md rename to evaluation/skills/skill-up/vendor/powercontext-project-context/references/work-handoff.md diff --git a/evaluation/src/powercontext_eval/cli.py b/evaluation/src/powercontext_eval/cli.py deleted file mode 100644 index 9d8bb94767..0000000000 --- a/evaluation/src/powercontext_eval/cli.py +++ /dev/null @@ -1,1009 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line entry point for the evaluation runner.""" - -from __future__ import annotations - -import json -import os -import signal -from collections.abc import Iterator -from contextlib import contextmanager -from datetime import UTC, datetime, timedelta -from pathlib import Path -from types import FrameType -from typing import TYPE_CHECKING, Annotated, Any, Protocol, cast -from urllib.error import HTTPError, URLError -from urllib.parse import urlsplit -from urllib.request import Request, urlopen - -import typer -from pydantic import ValidationError - -from powercontext_eval.benchmarks.longmemeval_v2.adapter import PowerContextMemoryAdapterError -from powercontext_eval.benchmarks.longmemeval_v2.arms import DEFAULT_EXPERIMENT_ARM_ID, ExperimentArmError -from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( - LongMemEvalV2CatalogError, - LongMemEvalV2EnvironmentError, - LongMemEvalV2InputError, -) -from powercontext_eval.benchmarks.longmemeval_v2.costs import ( - CostPolicyError, - ModelPricePolicy, - parse_cost_policy, -) -from powercontext_eval.benchmarks.longmemeval_v2.prepare_smoke import ( - DEFAULT_PROCESSOR_MODEL, - PrepareSmokeError, - prepare_reader_inputs_smoke, -) -from powercontext_eval.benchmarks.longmemeval_v2.reader_smoke import ( - DEFAULT_ANTHROPIC_BASE_URL_ENV, - ReaderSmokeError, - run_reader_smoke, -) -from powercontext_eval.benchmarks.longmemeval_v2.replay_score import ReplayScoreError, replay_score_smoke -from powercontext_eval.benchmarks.longmemeval_v2.report import ReportError, build_report -from powercontext_eval.benchmarks.longmemeval_v2.retrieval_smoke import RetrievalSmokeError, run_retrieval_smoke -from powercontext_eval.benchmarks.longmemeval_v2.run_smoke import RunSmokeError, run_smoke -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ( - DEFAULT_DEEPSEEK_BASE_URL as SCORE_DEFAULT_DEEPSEEK_BASE_URL, -) -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ( - DEFAULT_DEEPSEEK_MODEL as SCORE_DEFAULT_DEEPSEEK_MODEL, -) -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ( - DEFAULT_DEEPSEEK_TOKEN_ENV as SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, -) -from powercontext_eval.benchmarks.longmemeval_v2.score_smoke import ( - ScoreSmokeError, - run_score_smoke, -) -from powercontext_eval.benchmarks.longmemeval_v2.smoke import prepare_smoke_run -from powercontext_eval.benchmarks.swebench_pro.catalog import PUBLIC_V2_TASK_SET, SweBenchProCatalog, TaskSet -from powercontext_eval.benchmarks.work_continuity.arms import ( - DEFAULT_ASSEMBLY_MAX_BYTES, - ContinuationArmError, - declared_run_arm_ids, - ensure_comparable_work_continuity_runs, - supported_continuation_arm_ids, -) -from powercontext_eval.benchmarks.work_continuity.attempts import ( - AttemptInputError, - load_attempts, -) -from powercontext_eval.benchmarks.work_continuity.catalog import ( - TaskCatalog, - WorkContinuityCatalogError, - WorkContinuityInputError, -) -from powercontext_eval.benchmarks.work_continuity.report import ( - ReportError as WorkContinuityReportError, -) -from powercontext_eval.benchmarks.work_continuity.report import ( - build_report as build_work_continuity_report, -) -from powercontext_eval.benchmarks.work_continuity.runner import ( - WorkContinuityRunError, - execution_configuration, - require_recording_bindings, - run_summary, - run_work_continuity, - write_run_artifacts, -) -from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT -from powercontext_eval.models import TreatmentMode - -if TYPE_CHECKING: - from powercontext_eval.web.config import WebConfig - -app = typer.Typer(no_args_is_help=True, help="PowerContext evaluation runner.") -swebench_pro_app = typer.Typer(no_args_is_help=True, help="Pinned SWE-bench Pro evaluation.") -longmemeval_v2_app = typer.Typer(no_args_is_help=True, help="Pinned LongMemEval-V2 evaluation.") -work_continuity_app = typer.Typer( - no_args_is_help=True, - help="Continuation-method evaluation for Rollover Handoff and its alternatives.", -) -app.add_typer(swebench_pro_app, name="swebench-pro") -app.add_typer(longmemeval_v2_app, name="longmemeval-v2") -app.add_typer(work_continuity_app, name="work-continuity") -DEFAULT_DOCKER_NETWORK_POOL = "172.30.0.0/15" - - -@app.callback() -def root() -> None: - """Run reproducible PowerContext evaluations.""" - - -class _Stoppable(Protocol): - def stop(self) -> None: ... - - -def run_codex_contract_smoke(**kwargs: Any) -> Any: - """Load the platform-specific contract runner only when its command is used.""" - - from powercontext_eval.powercontext_sut import run_codex_contract_smoke as implementation - - return implementation(**kwargs) - - -def run_swebench_pro_instance(*args: Any, **kwargs: Any) -> Any: - """Load the platform-specific SWE-bench runner only when its command is used.""" - - from powercontext_eval.runner import run_swebench_pro_instance as implementation - - return implementation(*args, **kwargs) - - -def _parse_price_policy_option(value: str | None, *, label: str) -> ModelPricePolicy | None: - """Parse an explicitly passed JSON price policy, or keep costs unconfigured as null.""" - - if value is None: - return None - try: - parsed = json.loads(value) - except json.JSONDecodeError as error: - raise typer.BadParameter(f"{label} must be a JSON object: {error}") from None - try: - return parse_cost_policy(parsed, label=label) - except CostPolicyError as error: - raise typer.BadParameter(str(error)) from None - - -def _request_worker_stop(worker: _Stoppable, _signum: int, _frame: FrameType | None) -> None: - """Request that a worker exit after its current task finishes.""" - worker.stop() - - -def _web_config(root_path: Path | None) -> WebConfig: - from powercontext_eval.web.config import WebConfig - - try: - environ = dict(os.environ) - if root_path is not None: - environ["POWERCONTEXT_EVAL_ROOT"] = os.fspath(root_path) - return WebConfig.from_environment(environ) - except (KeyError, TypeError, ValueError, ValidationError): - raise typer.BadParameter("Invalid evaluation configuration.", param_hint="--root") from None - - -@contextmanager -def _worker_signal_handlers(worker: _Stoppable) -> Iterator[None]: - previous: dict[signal.Signals, Any] = {} - stop_requested = False - - def handler(signum: int, frame: FrameType | None) -> None: - nonlocal stop_requested - if stop_requested: - return - stop_requested = True - _request_worker_stop(worker, signum, frame) - - try: - for signum in (signal.SIGTERM, signal.SIGINT): - previous[signum] = signal.getsignal(signum) - signal.signal(signum, handler) - yield - finally: - for signum, prior in previous.items(): - signal.signal(signum, prior) - - -@app.command("web") -def web(root_path: Annotated[Path | None, typer.Option("--root")] = None) -> None: - """Serve the evaluation console API and frontend.""" - import uvicorn - - from powercontext_eval.web.api import create_app - - config = _web_config(root_path) - uvicorn.run(create_app(config), host=config.host, port=config.port) - - -@app.command("worker") -def worker(root_path: Annotated[Path | None, typer.Option("--root")] = None) -> None: - """Run queued task pairs at configured parallelism until shutdown is requested.""" - from powercontext_eval.web.store import TaskStore - from powercontext_eval.web.usage import CodexUsageProbe - from powercontext_eval.web.worker import EvaluationWorker - - config = _web_config(root_path) - store = TaskStore( - config.database_path, - lease_duration=timedelta(seconds=config.lease_seconds), - max_attempts=config.max_attempts, - ) - store.initialize() - service = EvaluationWorker( - config, - store, - usage_probe=CodexUsageProbe( - codex_binary=config.codex_binary, - auth_json=config.auth_json, - codex_config=config.codex_config, - proxy_url=config.proxy_url, - timeout_seconds=config.usage_probe_timeout_seconds, - ), - ) - with _worker_signal_handlers(service): - service.run_forever() - - -@app.command("codex-contract-smoke") -def codex_contract_smoke( - run_root: str = typer.Option(...), - task_image: str = typer.Option(...), - codex_bin: str = typer.Option(...), - tokensflow_bin: str = typer.Option(...), - tokensflow_user_home: str = typer.Option(...), - tokensflow_egress_network: str = typer.Option(...), - uv_bin: str = typer.Option(...), - powercontext_source: str = typer.Option(...), - powercontext_sha: str = typer.Option(...), - auth_json: str = typer.Option(...), - proxy_url: str = typer.Option(...), - prompt: str = typer.Option("Reply with exactly OK."), -) -> None: - """Run OFF/ON identity, daemon, bounded-drain, and Codex contract checks.""" - - outcome = run_codex_contract_smoke( - run_root=run_root, - task_image=task_image, - codex_bin=codex_bin, - tokensflow_bin=tokensflow_bin, - tokensflow_user_home=tokensflow_user_home, - tokensflow_egress_network=tokensflow_egress_network, - uv_bin=uv_bin, - powercontext_source=powercontext_source, - powercontext_sha=powercontext_sha, - auth_json=auth_json, - proxy_url=proxy_url, - prompt=prompt, - ) - typer.echo(json.dumps(outcome, ensure_ascii=False, sort_keys=True)) - - -@longmemeval_v2_app.command("smoke") -def longmemeval_v2_smoke( - data_root: Annotated[Path, typer.Option("--data-root")], - dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], - output_dir: Annotated[Path, typer.Option("--output-dir")], -) -> None: - """Validate fixed LongMemEval-V2 inputs and write smoke artifacts without calling a model.""" - - try: - prepared = prepare_smoke_run( - data_root=data_root, - dataset_lock=dataset_lock, - harness_root=harness_root, - smoke_manifest=smoke_manifest, - output_dir=output_dir, - ) - except LongMemEvalV2InputError as error: - raise typer.BadParameter(str(error)) from None - except LongMemEvalV2EnvironmentError as error: - typer.echo(f"LongMemEval-V2 smoke failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset", - "manifest": str(prepared.manifest_path), - "subset": str(prepared.subset_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("retrieval-smoke") -def longmemeval_v2_retrieval_smoke( - data_root: Annotated[Path, typer.Option("--data-root")], - dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - run_id: Annotated[str, typer.Option("--run-id")], - powercontext_revision: Annotated[str, typer.Option("--powercontext-revision")], - integration_revision: Annotated[str, typer.Option("--integration-revision")], - base_url: Annotated[str, typer.Option("--base-url")] = "http://127.0.0.1:8000", - token_env: Annotated[str, typer.Option("--token-env")] = "POWERCONTEXT_TOKEN", - experiment_arm: Annotated[str, typer.Option("--experiment-arm")] = DEFAULT_EXPERIMENT_ARM_ID, - search_limit: Annotated[int, typer.Option("--search-limit", min=1, max=50)] = 10, - timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=0.1)] = 30.0, -) -> None: - """Run the fixed LongMemEval-V2 subset through Memory retrieval without a model.""" - - try: - result = run_retrieval_smoke( - data_root=data_root, - dataset_lock=dataset_lock, - harness_root=harness_root, - smoke_manifest=smoke_manifest, - output_dir=output_dir, - run_id=run_id, - powercontext_revision=powercontext_revision, - integration_revision=integration_revision, - base_url=base_url, - token_env=token_env, - experiment_arm=experiment_arm, - search_limit=search_limit, - timeout_seconds=timeout_seconds, - ) - except ExperimentArmError as error: - raise typer.BadParameter(str(error)) from None - except (LongMemEvalV2CatalogError, RetrievalSmokeError, PowerContextMemoryAdapterError) as error: - typer.echo(f"LongMemEval-V2 retrieval smoke failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset-retrieval-only", - "manifest": str(result.manifest_path), - "results": str(result.results_path), - "summary": str(result.summary_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("prepare-smoke") -def longmemeval_v2_prepare_smoke( - retrieval_dir: Annotated[Path, typer.Option("--retrieval-dir")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - harness_python: Annotated[Path, typer.Option("--harness-python")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - processor_revision: Annotated[str, typer.Option("--processor-revision")], - processor_model: Annotated[str, typer.Option("--processor-model")] = DEFAULT_PROCESSOR_MODEL, - memory_context_max_tokens: Annotated[int, typer.Option("--memory-context-max-tokens", min=1)] = 200_000, -) -> None: - """Build pinned, bounded Reader inputs from retrieval-only smoke artifacts.""" - - try: - result = prepare_reader_inputs_smoke( - retrieval_dir=retrieval_dir, - harness_root=harness_root, - harness_python=harness_python, - output_dir=output_dir, - processor_model=processor_model, - processor_revision=processor_revision, - memory_context_max_tokens=memory_context_max_tokens, - ) - except PrepareSmokeError as error: - typer.echo(f"LongMemEval-V2 prompt preparation failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset-prepare-only", - "manifest": str(result.manifest_path), - "prompts": str(result.prompts_path), - "summary": str(result.summary_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("reader-smoke") -def longmemeval_v2_reader_smoke( - prepared_dir: Annotated[Path, typer.Option("--prepared-dir")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - provider: Annotated[str, typer.Option("--provider")] = "anthropic-compatible", - model: Annotated[str | None, typer.Option("--model")] = None, - base_url: Annotated[str | None, typer.Option("--base-url")] = None, - base_url_env: Annotated[str, typer.Option("--base-url-env")] = DEFAULT_ANTHROPIC_BASE_URL_ENV, - token_env: Annotated[str | None, typer.Option("--token-env")] = None, - max_tokens: Annotated[int, typer.Option("--max-tokens", min=1)] = 512, - temperature: Annotated[float, typer.Option("--temperature", min=0.0, max=2.0)] = 0.0, - timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=1.0)] = 120.0, - max_questions: Annotated[int | None, typer.Option("--max-questions", min=1)] = None, - price_policy: Annotated[str | None, typer.Option("--price-policy")] = None, -) -> None: - """Call a configured Reader over prepared smoke prompts without scoring.""" - - resolved_price_policy = _parse_price_policy_option(price_policy, label="--price-policy") - try: - result = run_reader_smoke( - prepared_dir=prepared_dir, - output_dir=output_dir, - provider=provider, - model=model, - base_url=base_url, - base_url_env=base_url_env, - token_env=token_env, - max_tokens=max_tokens, - temperature=temperature, - timeout_seconds=timeout_seconds, - max_questions=max_questions, - price_policy=resolved_price_policy, - ) - except ReaderSmokeError as error: - typer.echo(f"LongMemEval-V2 Reader failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset-reader-only", - "manifest": str(result.manifest_path), - "outputs": str(result.outputs_path), - "summary": str(result.summary_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("score-smoke") -def longmemeval_v2_score_smoke( - reader_dir: Annotated[Path, typer.Option("--reader-dir")], - data_root: Annotated[Path, typer.Option("--data-root")], - dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], - smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - judge_model: Annotated[str, typer.Option("--judge-model")] = SCORE_DEFAULT_DEEPSEEK_MODEL, - judge_token_env: Annotated[str, typer.Option("--judge-token-env")] = SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, - judge_base_url: Annotated[str, typer.Option("--judge-base-url")] = SCORE_DEFAULT_DEEPSEEK_BASE_URL, - judge_max_tokens: Annotated[int, typer.Option("--judge-max-tokens", min=1)] = 256, - judge_temperature: Annotated[float, typer.Option("--judge-temperature", min=0.0, max=2.0)] = 0.0, - judge_timeout_seconds: Annotated[float, typer.Option("--judge-timeout-seconds", min=1.0)] = 120.0, - price_policy: Annotated[str | None, typer.Option("--judge-price-policy")] = None, -) -> None: - """Score Reader smoke outputs with pinned rules and DeepSeek only where upstream requires a judge.""" - - resolved_price_policy = _parse_price_policy_option(price_policy, label="--judge-price-policy") - try: - result = run_score_smoke( - reader_dir=reader_dir, - data_root=data_root, - dataset_lock=dataset_lock, - smoke_manifest=smoke_manifest, - harness_root=harness_root, - output_dir=output_dir, - judge_model=judge_model, - judge_token_env=judge_token_env, - judge_base_url=judge_base_url, - judge_max_tokens=judge_max_tokens, - judge_temperature=judge_temperature, - judge_timeout_seconds=judge_timeout_seconds, - judge_price_policy=resolved_price_policy, - ) - except ScoreSmokeError as error: - typer.echo(f"LongMemEval-V2 scoring failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset-score-only", - "manifest": str(result.manifest_path), - "results": str(result.results_path), - "summary": str(result.summary_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("replay-score") -def longmemeval_v2_replay_score( - score_dir: Annotated[Path, typer.Option("--score-dir")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - output_dir: Annotated[Path, typer.Option("--output-dir")], -) -> None: - """Replay saved deterministic and Judge score decisions without model calls.""" - - try: - result = replay_score_smoke(score_dir=score_dir, harness_root=harness_root, output_dir=output_dir) - except ReplayScoreError as error: - typer.echo(f"LongMemEval-V2 score replay failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset-score-replay", - "manifest": str(result.manifest_path), - "results": str(result.results_path), - "summary": str(result.summary_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@longmemeval_v2_app.command("run-smoke") -def longmemeval_v2_run_smoke( - data_root: Annotated[Path, typer.Option("--data-root")], - dataset_lock: Annotated[Path, typer.Option("--dataset-lock")], - smoke_manifest: Annotated[Path, typer.Option("--smoke-manifest")], - harness_root: Annotated[Path, typer.Option("--harness-root")], - harness_python: Annotated[Path, typer.Option("--harness-python")], - processor_revision: Annotated[str, typer.Option("--processor-revision")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - powercontext_revision: Annotated[str, typer.Option("--powercontext-revision")], - integration_revision: Annotated[str, typer.Option("--integration-revision")], - run_id: Annotated[str | None, typer.Option("--run-id")] = None, - processor_model: Annotated[str, typer.Option("--processor-model")] = DEFAULT_PROCESSOR_MODEL, - memory_context_max_tokens: Annotated[int, typer.Option("--memory-context-max-tokens", min=1)] = 200_000, - powercontext_base_url: Annotated[str, typer.Option("--powercontext-base-url")] = "http://127.0.0.1:8000", - powercontext_token_env: Annotated[str, typer.Option("--powercontext-token-env")] = "POWERCONTEXT_TOKEN", - experiment_arm: Annotated[str, typer.Option("--experiment-arm")] = DEFAULT_EXPERIMENT_ARM_ID, - search_limit: Annotated[int, typer.Option("--search-limit", min=1, max=50)] = 10, - timeout_seconds: Annotated[float, typer.Option("--timeout-seconds", min=0.1)] = 30.0, - reader_provider: Annotated[str, typer.Option("--reader-provider")] = "deepseek-openai", - reader_model: Annotated[str | None, typer.Option("--reader-model")] = None, - reader_base_url: Annotated[str | None, typer.Option("--reader-base-url")] = None, - reader_base_url_env: Annotated[str, typer.Option("--reader-base-url-env")] = DEFAULT_ANTHROPIC_BASE_URL_ENV, - reader_token_env: Annotated[str | None, typer.Option("--reader-token-env")] = None, - reader_max_tokens: Annotated[int, typer.Option("--reader-max-tokens", min=1)] = 512, - reader_temperature: Annotated[float, typer.Option("--reader-temperature", min=0.0, max=2.0)] = 0.0, - reader_timeout_seconds: Annotated[float, typer.Option("--reader-timeout-seconds", min=1.0)] = 120.0, - judge_provider: Annotated[str, typer.Option("--judge-provider")] = "deepseek-openai", - judge_model: Annotated[str, typer.Option("--judge-model")] = SCORE_DEFAULT_DEEPSEEK_MODEL, - judge_token_env: Annotated[str, typer.Option("--judge-token-env")] = SCORE_DEFAULT_DEEPSEEK_TOKEN_ENV, - judge_base_url: Annotated[str, typer.Option("--judge-base-url")] = SCORE_DEFAULT_DEEPSEEK_BASE_URL, - judge_max_tokens: Annotated[int, typer.Option("--judge-max-tokens", min=1)] = 256, - judge_temperature: Annotated[float, typer.Option("--judge-temperature", min=0.0, max=2.0)] = 0.0, - judge_timeout_seconds: Annotated[float, typer.Option("--judge-timeout-seconds", min=1.0)] = 120.0, - price_policy: Annotated[str | None, typer.Option("--price-policy")] = None, - judge_price_policy: Annotated[str | None, typer.Option("--judge-price-policy")] = None, - skip_reader: Annotated[bool, typer.Option("--skip-reader")] = False, - skip_score: Annotated[bool, typer.Option("--skip-score")] = False, -) -> None: - """Run the whole LongMemEval-V2 smoke workload into one fail-closed run directory.""" - - resolved_price_policy = _parse_price_policy_option(price_policy, label="--price-policy") - resolved_judge_price_policy = _parse_price_policy_option(judge_price_policy, label="--judge-price-policy") - try: - result = run_smoke( - data_root=data_root, - dataset_lock=dataset_lock, - smoke_manifest=smoke_manifest, - harness_root=harness_root, - harness_python=harness_python, - processor_revision=processor_revision, - output_dir=output_dir, - powercontext_revision=powercontext_revision, - integration_revision=integration_revision, - run_id=run_id, - processor_model=processor_model, - memory_context_max_tokens=memory_context_max_tokens, - powercontext_base_url=powercontext_base_url, - powercontext_token_env=powercontext_token_env, - experiment_arm=experiment_arm, - search_limit=search_limit, - timeout_seconds=timeout_seconds, - reader_provider=reader_provider, - reader_model=reader_model, - reader_base_url=reader_base_url, - reader_base_url_env=reader_base_url_env, - reader_token_env=reader_token_env, - reader_max_tokens=reader_max_tokens, - reader_temperature=reader_temperature, - reader_timeout_seconds=reader_timeout_seconds, - judge_provider=judge_provider, - judge_model=judge_model, - judge_token_env=judge_token_env, - judge_base_url=judge_base_url, - judge_max_tokens=judge_max_tokens, - judge_temperature=judge_temperature, - judge_timeout_seconds=judge_timeout_seconds, - reader_price_policy=resolved_price_policy, - judge_price_policy=resolved_judge_price_policy, - skip_reader=skip_reader, - skip_score=skip_score, - ) - except (RunSmokeError, LongMemEvalV2CatalogError, ExperimentArmError) as error: - typer.echo(f"LongMemEval-V2 smoke run failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset", - "status": result.status, - "manifest": str(result.manifest_path), - "summary": str(result.summary_path), - "completed_phases": list(result.completed_phases), - "skipped_phases": list(result.skipped_phases), - "failed_phase": result.failed_phase, - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - if result.status != "completed": - raise typer.Exit(code=1) - - -@longmemeval_v2_app.command("report") -def longmemeval_v2_report( - run_dir: Annotated[Path, typer.Option("--run-dir")], - output_dir: Annotated[Path | None, typer.Option("--output-dir")] = None, -) -> None: - """Summarize one saved smoke run into a single unified report without a model or server.""" - - try: - result = build_report(run_dir=run_dir, output_dir=output_dir) - except ReportError as error: - typer.echo(f"LongMemEval-V2 report failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "smoke-subset", - "report": str(result.report_path), - "markdown": str(result.markdown_path), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@work_continuity_app.command("validate") -def work_continuity_validate( - task_lock: Annotated[Path, typer.Option("--task-lock")], - attempts: Annotated[Path | None, typer.Option("--attempts")] = None, -) -> None: - """Validate the pinned task lock and any recorded attempts without writing anything.""" - - try: - catalog = TaskCatalog.load(task_lock) - recorded = load_attempts(attempts, catalog=catalog) if attempts is not None else None - if recorded is not None: - # The documented preflight has to reject what `run` would reject: - # a recording bound to another task lock or another context is not - # scoreable, and finding that out here is the point of the command. - require_recording_bindings(catalog=catalog, attempts=recorded) - except (WorkContinuityInputError, AttemptInputError) as error: - raise typer.BadParameter(str(error)) from None - except (WorkContinuityCatalogError, WorkContinuityRunError) as error: - typer.echo(f"Work-continuity validation failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": "work-continuity-input-validation", - "task_set_id": catalog.task_set_id, - "task_count": len(catalog.tasks), - "task_ids": list(catalog.task_ids), - "task_lock_sha256": catalog.content_sha256, - "supported_arms": list(supported_continuation_arm_ids()), - "attempt_count": None if recorded is None else len(recorded.attempts), - "hosts": None if recorded is None else list(recorded.hosts), - "recording_protocol": ( - None - if recorded is None - else { - "task_set_id": recorded.protocol.task_set_id, - "task_lock_sha256": recorded.protocol.task_lock_sha256, - "assembly_max_bytes": recorded.protocol.assembly_max_bytes, - } - ), - "execution_configuration": None if recorded is None else execution_configuration(recorded), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -def _run_manifest(path: Path) -> dict[str, object]: - """Read one run manifest from a run directory or from a manifest file. - - A published run is a directory, so accepting the directory is what makes the - command usable on the artifacts a run actually writes. - """ - - candidate = path / "run-manifest.json" if path.is_dir() else path - if not candidate.is_file(): - raise WorkContinuityRunError(f"no run manifest at {candidate}") - try: - payload = json.loads(candidate.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise WorkContinuityRunError(f"cannot read run manifest {candidate}: {error}") from None - if not isinstance(payload, dict): - raise WorkContinuityRunError(f"run manifest {candidate} is not a JSON object") - return cast("dict[str, object]", payload) - - -@work_continuity_app.command("compare") -def work_continuity_compare( - baseline: Annotated[Path, typer.Option("--baseline")], - treatment: Annotated[Path, typer.Option("--treatment")], -) -> None: - """Decide whether two recorded runs may be compared, and refuse when they may not.""" - - try: - first = _run_manifest(baseline) - second = _run_manifest(treatment) - # The gate is the whole point of the command: it is the only supported way - # to have the documented comparison rules applied to two published runs. - ensure_comparable_work_continuity_runs(first, second) - except ContinuationArmError as error: - typer.echo(f"Work-continuity comparison refused: {error}", err=True) - raise typer.Exit(code=1) from None - except WorkContinuityRunError as error: - typer.echo(f"Work-continuity comparison failed: {error}", err=True) - raise typer.Exit(code=1) from None - first_arms = declared_run_arm_ids(first) or () - second_arms = declared_run_arm_ids(second) or () - typer.echo( - json.dumps( - { - "classification": "work-continuity-comparability", - "comparable": True, - "baseline_run_id": first.get("run_id"), - "treatment_run_id": second.get("run_id"), - "task_set_id": first.get("task_set_id"), - "baseline_arms": list(first_arms), - "treatment_arms": list(second_arms), - "shared_arms": sorted(set(first_arms) & set(second_arms)), - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -def _method_rows(summary: dict[str, object]) -> list[dict[str, object]]: - """Return one per-method echo row, rejecting a summary that lost its shape.""" - - arms = summary.get("arms") - if not isinstance(arms, list): - raise WorkContinuityRunError("work-continuity run summary is missing its per-method rows") - rows: list[dict[str, object]] = [] - for arm in arms: - if not isinstance(arm, dict): - raise WorkContinuityRunError("work-continuity run summary holds a malformed method row") - method = cast("dict[str, object]", arm) - injected = method.get("injected_bytes") - outcome = method.get("outcome") - if not isinstance(injected, dict) or not isinstance(outcome, dict): - raise WorkContinuityRunError("work-continuity run summary holds a malformed method row") - rows.append( - { - "arm_id": method["arm_id"], - "injected_bytes_total": cast("dict[str, object]", injected)["total"], - "task_success": cast("dict[str, object]", outcome)["task_success"], - } - ) - return rows - - -@work_continuity_app.command("run") -def work_continuity_run( - task_lock: Annotated[Path, typer.Option("--task-lock")], - output_dir: Annotated[Path, typer.Option("--output-dir")], - run_id: Annotated[str, typer.Option("--run-id")], - max_bytes: Annotated[int, typer.Option("--max-bytes", min=1)] = DEFAULT_ASSEMBLY_MAX_BYTES, - arm: Annotated[list[str] | None, typer.Option("--arm")] = None, - task_id: Annotated[list[str] | None, typer.Option("--task-id")] = None, - attempts: Annotated[Path | None, typer.Option("--attempts")] = None, - powercontext_revision: Annotated[str | None, typer.Option("--powercontext-revision")] = None, - integration_revision: Annotated[str | None, typer.Option("--integration-revision")] = None, -) -> None: - """Assemble every selected method, score any recorded attempts, and write one run directory.""" - - try: - result = run_work_continuity( - task_lock=task_lock, - run_id=run_id, - max_bytes=max_bytes, - arm_ids=tuple(arm) if arm else None, - task_ids=tuple(task_id) if task_id else None, - attempts_path=attempts, - powercontext_revision=powercontext_revision, - integration_revision=integration_revision, - ) - artifacts = write_run_artifacts(result, output_dir) - report = build_work_continuity_report(result, output_dir=artifacts.output_dir) - methods = _method_rows(run_summary(result)) - except ContinuationArmError as error: - raise typer.BadParameter(str(error)) from None - except (WorkContinuityInputError, AttemptInputError) as error: - raise typer.BadParameter(str(error)) from None - except (WorkContinuityRunError, WorkContinuityReportError, WorkContinuityCatalogError) as error: - typer.echo(f"Work-continuity run failed: {error}", err=True) - raise typer.Exit(code=1) from None - typer.echo( - json.dumps( - { - "classification": result.classification, - "run_id": result.run_id, - "manifest": str(artifacts.manifest_path), - "assembly": str(artifacts.assembly_path), - "scores": str(artifacts.scores_path), - "summary": str(artifacts.summary_path), - "report": str(report.report_path), - "markdown": str(report.markdown_path), - "methods": methods, - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@swebench_pro_app.command("run") -def swebench_pro_run( - root_path: str = typer.Option(..., "--root"), - powercontext_source: str | None = typer.Option(None), - powercontext_ref: str = typer.Option("latest"), - harness_root: str | None = typer.Option(None), - harness_python: str | None = typer.Option(None), - dataset_path: str | None = typer.Option(None), - instance_id: str = typer.Option(...), - codex_bin: str | None = typer.Option(None), - tokensflow_enabled: bool = typer.Option(False, "--tokensflow/--no-tokensflow"), - tokensflow_bin: str | None = typer.Option(None), - tokensflow_user_home: str | None = typer.Option(None), - tokensflow_egress_network: str | None = typer.Option(None), - uv_bin: str | None = typer.Option(None), - registry_bin: str | None = typer.Option(None), - auth_json: str | None = typer.Option(None), - proxy_url: str | None = typer.Option(None), - docker_network_pool: str = typer.Option(DEFAULT_DOCKER_NETWORK_POOL), - extra_no_proxy_hosts: str = typer.Option(""), - model: str = typer.Option(DEFAULT_CODEX_MODEL, "--model"), - reasoning_effort: str = typer.Option(DEFAULT_REASONING_EFFORT, "--reasoning-effort"), - run_id: str | None = typer.Option(None), -) -> None: - """Run Gold, PowerContext OFF/ON, official grading, and report generation.""" - - from powercontext_eval.runner import RunConfig - - root = Path(root_path) - harness = Path(harness_root) if harness_root is not None else root / "cache" / "swebench-pro.git" - dataset = Path(dataset_path) if dataset_path is not None else harness / "helper_code" / "sweap_eval_full_v2.jsonl" - binaries = root / "bin" - - catalog = SweBenchProCatalog.load(dataset) - result = run_swebench_pro_instance( - RunConfig( - root=root, - powercontext_source=( - Path(powercontext_source) if powercontext_source is not None else root / "source" / "powercontext.git" - ), - powercontext_ref=powercontext_ref, - harness_root=harness, - harness_python=( - Path(harness_python) - if harness_python is not None - else root / "venvs" / "swebench-pro" / "bin" / "python" - ), - codex_binary=Path(codex_bin) if codex_bin is not None else binaries / "codex", - tokensflow_enabled=tokensflow_enabled, - tokensflow_binary=( - Path(tokensflow_bin) - if tokensflow_bin is not None - else (binaries / "tokensflow" if tokensflow_enabled else None) - ), - tokensflow_user_home=( - Path(tokensflow_user_home) - if tokensflow_user_home is not None - else (root / "tokensflow-home" if tokensflow_enabled else None) - ), - tokensflow_egress_network=tokensflow_egress_network, - uv_binary=Path(uv_bin) if uv_bin is not None else binaries / "uv", - registry_binary=Path(registry_bin) if registry_bin is not None else binaries / "regctl", - auth_json=Path(auth_json) if auth_json is not None else root / "codex-home" / "auth.json", - proxy_url=proxy_url, - docker_network_pool=docker_network_pool, - extra_no_proxy_hosts=tuple(host for host in extra_no_proxy_hosts.split(",") if host), - run_id=run_id or datetime.now(UTC).strftime("run-%Y%m%d-%H%M%S"), - model=model, - reasoning_effort=reasoning_effort, - ), - instance=catalog.require(instance_id), - ) - typer.echo( - json.dumps( - { - "run_id": result.run_id, - "report": str(result.report_path), - "off_resolved": result.off_resolved, - "on_resolved": result.on_resolved, - }, - sort_keys=True, - ) - ) - - -@swebench_pro_app.command("create-batch") -def swebench_pro_create_batch( - idempotency_key: str = typer.Option(..., "--idempotency-key"), - console_url: str = typer.Option("http://127.0.0.1:8787", "--console-url"), - powercontext_ref: str = typer.Option("latest", "--powercontext-ref"), - task_set: str = typer.Option(PUBLIC_V2_TASK_SET, "--task-set"), - model: str = typer.Option(DEFAULT_CODEX_MODEL, "--model"), - treatment_mode: Annotated[TreatmentMode, typer.Option("--treatment-mode")] = TreatmentMode.OFF_ON, - usage_pause_percent: int = typer.Option(80, "--usage-pause-percent", min=1, max=100), - start_paused: bool = typer.Option(False, "--start-paused/--start-running"), -) -> None: - """Create one full batch through the console API, optionally atomically paused.""" - - from powercontext_eval.web.batches import BatchCreate - - endpoint = _batch_api_endpoint(console_url) - try: - batch = BatchCreate( - powercontext_ref=powercontext_ref, - benchmark="swebench-pro", - task_set=cast(TaskSet, task_set), - model=model, - reasoning_effort=DEFAULT_REASONING_EFFORT, - treatment_mode=treatment_mode, - idempotency_key=idempotency_key, - usage_pause_percent=usage_pause_percent, - initial_control_intent="pause" if start_paused else "run", - ) - except ValidationError: - raise typer.BadParameter("Invalid batch configuration.") from None - request = Request( - endpoint, - data=batch.model_dump_json().encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urlopen(request, timeout=30) as response: - payload = json.loads(response.read()) - except HTTPError as error: - if error.code == 422: - raise typer.BadParameter( - "Codex model is not enabled for new evaluation work.", - param_hint="--model", - ) from None - raise typer.Exit(code=1) from None - except (URLError, OSError, ValueError, UnicodeDecodeError): - raise typer.Exit(code=1) from None - if not isinstance(payload, dict) or not isinstance(payload.get("batch_id"), str): - raise typer.Exit(code=1) - typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) - - -def _batch_api_endpoint(console_url: str) -> str: - parsed = urlsplit(console_url) - if ( - parsed.scheme not in {"http", "https"} - or not parsed.hostname - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - or parsed.path not in {"", "/"} - ): - raise typer.BadParameter("Invalid console URL.", param_hint="--console-url") - return console_url.rstrip("/") + "/api/batches" - - -def main() -> None: - """Run the evaluation command-line application.""" - - app() - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index f4e62cc892..bd88a52358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ builtin = [ "pymysql>=1.2,<1.2.1", "sqlalchemy[asyncio]>=2,<3", "sqlite-vec>=0.1.9,<0.2", + "tzdata>=2026.3", ] seekdb = [ "powercontext[builtin]", @@ -190,7 +191,9 @@ exclude = [ "integrations/opendal", "tests/langgraph_adapter", "tests/e2e/test_langgraph_chain.py", - "evaluation", + "evaluation/coding", + "evaluation/memory/longmemeval_v2", + "evaluation/skills", ] [tool.ty.rules] diff --git a/tests/e2e/test_locomo_scope_resume.py b/tests/e2e/test_locomo_scope_resume.py new file mode 100644 index 0000000000..dd6ca236e4 --- /dev/null +++ b/tests/e2e/test_locomo_scope_resume.py @@ -0,0 +1,199 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LoCoMo run namespaces survive runtime restarts through registered Scopes.""" + +import asyncio +import json +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest +from pydantic_ai.embeddings import TestEmbeddingModel +from pydantic_ai.messages import ModelResponse, TextPart +from pydantic_ai.models import Model +from pydantic_ai.models.function import FunctionModel + +from evaluation.memory.locomo import runner +from evaluation.memory.locomo.dataset import load_locomo +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import BuiltinConfig, InferenceConfig, open_builtin_runtime +from powercontext.builtin.scope import ScopeNotFoundError +from powercontext.server.settings import ServerSettings + + +def test_locomo_ingestion_registers_and_resumes_its_scope_after_restart( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dataset = load_locomo(Path(__file__).parents[2] / "evaluation" / "memory" / "locomo" / "dataset" / "locomo10.json") + conversation = replace( + dataset.conversations[0], + sessions=dataset.conversations[0].sessions[:1], + questions=dataset.conversations[0].questions[:1], + ) + dataset = replace(dataset, conversations=(conversation,)) + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'state.sqlite3'}"), + inference=InferenceConfig( + generation_model="openai:locomo-test", + embedding_model="test", + embedding_profile_id="locomo-test", + embedding_dimension=8, + ), + ) + + async def respond(messages, info): + prompt = next(part.content for part in messages[-1].parts if part.part_kind == "user-prompt") + value = json.loads(prompt) + if "evidence" in value: + output = { + "candidates": [ + { + "intent": "add", + "kind": "fact", + "text": "Caroline went to the LGBTQ support group on 7 May 2023.", + "evidence_ids": [value["evidence"][0]["evidence_id"]], + } + ] + } + else: + output = {"label": "CORRECT"} if "gold_answer" in value else {"answer": "7 May 2023"} + return ModelResponse(parts=[TextPart(json.dumps(output))]) + + def infer_model(model, **kwargs): + return model if isinstance(model, Model) else FunctionModel(respond) + + monkeypatch.setattr("pydantic_ai.models.infer_model", infer_model) + monkeypatch.setattr(runner, "infer_model", infer_model) + + monkeypatch.setattr( + "pydantic_ai.embeddings.infer_embedding_model", + lambda model, **kwargs: TestEmbeddingModel() if isinstance(model, str) else model, + ) + + async def ingest(run_id: str): + return await runner.ingest_dataset( + dataset, + settings=settings, + run_id=run_id, + output_directory=tmp_path / run_id, + concurrency=1, + ) + + first = asyncio.run(ingest("scope-resume")) + first_conversation = first["conversations"][conversation.sample_id] + registered_scope = first_conversation["scope_id"] + assert first["newly_processed_session_count"] == 1 + assert first["memory_entry_count"] == 1 + assert first_conversation["namespace"] == runner.scope_id("scope-resume", conversation.sample_id) + + async def read_persisted_memory(): + async with open_builtin_runtime( + BuiltinConfig(database=settings.database, inference=settings.inference) + ) as runtime: + assert runtime.scopes is not None + descriptor = await runtime.scopes.get(registered_scope) + page = await runtime.memory.for_scope(descriptor.scope_id).list() + assert len(page.entries) == 1 + assert page.entries[0].entry.sources[0].source_id == "D1" + + asyncio.run(read_persisted_memory()) + resumed = asyncio.run(ingest("scope-resume")) + assert resumed["conversations"][conversation.sample_id]["scope_id"] == registered_scope + assert resumed["resumed_session_count"] == 1 + assert resumed["newly_processed_session_count"] == 0 + assert resumed["memory_entry_count"] == 1 + + missing_database_settings = settings.model_copy( + update={"database": SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'missing-state.sqlite3'}")} + ) + ingestion_path = tmp_path / "scope-resume" / "ingestion.json" + frozen_ingestion = ingestion_path.read_bytes() + with pytest.raises(ScopeNotFoundError): + asyncio.run( + runner.evaluate_dataset( + dataset, + settings=missing_database_settings, + run_id="scope-resume", + output_directory=tmp_path / "scope-resume", + top_k=1, + ) + ) + with pytest.raises(ScopeNotFoundError): + asyncio.run( + runner.ingest_dataset( + dataset, + settings=missing_database_settings, + run_id="scope-resume", + output_directory=tmp_path / "scope-resume", + ) + ) + assert ingestion_path.read_bytes() == frozen_ingestion + assert not (tmp_path / "scope-resume" / "observations.jsonl").exists() + + summary = asyncio.run( + runner.evaluate_dataset( + dataset, + settings=settings, + run_id="scope-resume", + output_directory=tmp_path / "scope-resume", + top_k=1, + concurrency=1, + ) + ) + assert summary["metrics"]["overall"]["error_count"] == 0 + assert summary["metrics"]["overall"]["evidence_hit"] == 1 + assert summary["metrics"]["overall"]["llm_judge"] == 1 + + separate = asyncio.run(ingest("another-run")) + assert separate["conversations"][conversation.sample_id]["scope_id"] != registered_scope + assert separate["newly_processed_session_count"] == 1 + + +@pytest.mark.parametrize("legacy_ingestion", [False, True]) +def test_locomo_requires_registered_ingestion_before_evaluation(tmp_path: Path, legacy_ingestion: bool) -> None: + dataset = load_locomo(Path(__file__).parents[2] / "evaluation" / "memory" / "locomo" / "dataset" / "locomo10.json") + database_path = tmp_path / "unopened.sqlite3" + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}"), + inference=InferenceConfig(generation_model="openai:unused-test-model"), + ) + parameters: dict[str, Any] = { + "dataset": dataset, + "settings": settings, + "run_id": "missing-ingestion", + "output_directory": tmp_path, + } + message = "run ingestion first" + if legacy_ingestion: + message = "legacy logical scopes require a new run" + ingestion_path = tmp_path / "ingestion.json" + ingestion_path.write_text( + json.dumps({ + "run_id": "missing-ingestion", + "conversations": { + conversation.sample_id: {"scope_id": runner.scope_id("missing-ingestion", conversation.sample_id)} + for conversation in dataset.conversations + }, + }) + ) + frozen_ingestion = ingestion_path.read_bytes() + with pytest.raises(ValueError, match=message): + asyncio.run(runner.ingest_dataset(**parameters)) + assert ingestion_path.read_bytes() == frozen_ingestion + with pytest.raises(ValueError, match=message): + asyncio.run(runner.evaluate_dataset(**parameters, top_k=1)) + assert not database_path.exists() + assert not (tmp_path / "observations.jsonl").exists() diff --git a/tests/evaluation/test_locomo_benchmark.py b/tests/evaluation/test_locomo_benchmark.py index 364262167a..df69a34a94 100644 --- a/tests/evaluation/test_locomo_benchmark.py +++ b/tests/evaluation/test_locomo_benchmark.py @@ -19,8 +19,8 @@ import pytest -from benchmark.locomo.dataset import load_locomo, render_session -from benchmark.locomo.metrics import ( +from evaluation.memory.locomo.dataset import load_locomo, render_session +from evaluation.memory.locomo.metrics import ( bleu1, diagnose_observations, exact_match, @@ -29,7 +29,7 @@ summarize_observations, token_f1, ) -from benchmark.locomo.runner import ( +from evaluation.memory.locomo.runner import ( normalize_run_id, prepare_rejudge, prepare_run, @@ -39,7 +39,7 @@ from powercontext.builtin.runtime import InferenceConfig, MemoryExtractionProfile, RuntimeConfig from powercontext.server.settings import ServerSettings -DATASET = Path(__file__).parents[2] / "benchmark" / "locomo" / "dataset" / "locomo10.json" +DATASET = Path(__file__).parents[2] / "evaluation" / "memory" / "locomo" / "dataset" / "locomo10.json" def test_canonical_locomo_dataset_has_expected_shape_and_scored_selection() -> None: diff --git a/tests/evaluation/test_locomo_prompts.py b/tests/evaluation/test_locomo_prompts.py index 57e94a8bd2..060a873e60 100644 --- a/tests/evaluation/test_locomo_prompts.py +++ b/tests/evaluation/test_locomo_prompts.py @@ -16,7 +16,7 @@ import pytest -from benchmark.locomo.prompts import ( +from evaluation.memory.locomo.prompts import ( ANSWER_SOURCE_INFERENCE_INSTRUCTIONS_VERSION, ANSWER_SOURCE_UNKNOWN_FALLBACK_INSTRUCTIONS_VERSION, JUDGE_INSTRUCTIONS_VERSION, diff --git a/tests/test_locomo_plus_cli.py b/tests/test_locomo_plus_cli.py index 44b14c91b1..f66d400e5f 100644 --- a/tests/test_locomo_plus_cli.py +++ b/tests/test_locomo_plus_cli.py @@ -17,14 +17,17 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any import pytest +from pydantic_ai.messages import ModelResponse, TextPart +from pydantic_ai.models.function import FunctionModel -from benchmark.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn -from benchmark.locomo_plus import cli -from benchmark.locomo_plus.dataset import DEFAULT_SMOKE_PATH, SMOKE_CASE_IDS, LoCoMoPlusCase, LoCoMoPlusDataset +from evaluation.memory.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn +from evaluation.memory.locomo_plus import cli, runner +from evaluation.memory.locomo_plus.dataset import DEFAULT_SMOKE_PATH, SMOKE_CASE_IDS, LoCoMoPlusCase, LoCoMoPlusDataset @pytest.fixture @@ -226,3 +229,96 @@ def unexpected_service(*_: Any, **__: Any) -> Any: def test_bundled_input_rejects_requests_outside_its_fixed_scope(options: list[str]) -> None: with pytest.raises(SystemExit, match="2"): cli.main(["run", "--dataset-file", str(DEFAULT_SMOKE_PATH), "--dry-run", *options]) + + +def test_run_loads_oceanbase_identity_from_dotenv_without_recording_credentials( + dataset: LoCoMoPlusDataset, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The query-only arm verifies dotenv and resume identity without connecting to a database.""" + for name in tuple(os.environ): + if name.startswith(("POWERCONTEXT_SERVER_", "OPENAI_")): + monkeypatch.delenv(name) + values = { + "POWERCONTEXT_SERVER_DATABASE_KIND": "oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL": ( + "mysql+aoceanbase://tenant:initial-password@database.invalid:2881/evaluation?charset=utf8mb4" + ), + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL": "test-answer", + } + # load_dotenv mutates the environment; register each variable so teardown restores it. + for name, value in values.items(): + monkeypatch.setenv(name, value) + env_file = tmp_path / "evaluation.env" + + def save_env() -> None: + env_file.write_text("\n".join(f"{name}={value}" for name, value in values.items()) + "\n") + + save_env() + monkeypatch.setattr(cli, "_load_dataset", lambda _: dataset) + + async def open_model(name, settings, resources): + async def respond(messages, info): + output = ( + "A quiet place would help." if name == "test-answer" else '{"label":"correct","reason":"Uses the cue."}' + ) + return ModelResponse(parts=[TextPart(output)]) + + return FunctionModel(respond, model_name=name) + + def unexpected_database(*args, **kwargs): + pytest.fail("the query-only arm must not open a database") + + monkeypatch.setattr(runner, "open_model", open_model) + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_database) + output_directory = tmp_path / "results" + arguments = [ + "run", + "--env-file", + str(env_file), + "--run-id", + "dotenv-oceanbase", + "--output-directory", + str(output_directory), + "--judge-model", + "test-judge", + "--arm", + "query-only", + "--limit", + "1", + ] + assert cli.main(arguments) == 0 + manifest_path = output_directory / "run.json" + manifest = json.loads(manifest_path.read_text()) + assert manifest["configuration"]["database_kind"] == "oceanbase" + assert manifest["configuration"]["database_fingerprint"] + assert manifest["configuration"]["persistence"] == "configured database; isolated run scopes" + assert not (output_directory / "state.sqlite3").exists() + + async def unexpected_model(*args, **kwargs): + pytest.fail("a completed run must resume without additional model calls") + + monkeypatch.setattr(runner, "open_model", unexpected_model) + values["POWERCONTEXT_SERVER_DATABASE_URL"] = ( + "mysql+aoceanbase://tenant:rotated-password@database.invalid:2881/evaluation?charset=utf8mb4" + ) + save_env() + assert cli.main(arguments) == 0 + assert json.loads(manifest_path.read_text()) == manifest + + saved_output = capsys.readouterr().out + "\n".join( + path.read_text() for path in output_directory.rglob("*") if path.is_file() + ) + for private_value in ("initial-password", "rotated-password", "database.invalid", "mysql+aoceanbase://"): + assert private_value not in saved_output + + values["POWERCONTEXT_SERVER_DATABASE_URL"] = ( + "mysql+aoceanbase://tenant:rotated-password@database.invalid:2881/other_evaluation?charset=utf8mb4" + ) + save_env() + with pytest.raises(SystemExit, match="2"): + cli.main(arguments) + assert "run identity changed" in capsys.readouterr().err + assert json.loads(manifest_path.read_text()) == manifest diff --git a/tests/test_locomo_plus_dataset.py b/tests/test_locomo_plus_dataset.py index 2f6dbfaf45..1c123e2e54 100644 --- a/tests/test_locomo_plus_dataset.py +++ b/tests/test_locomo_plus_dataset.py @@ -23,7 +23,7 @@ import pytest -from benchmark.locomo_plus.dataset import ( +from evaluation.memory.locomo_plus.dataset import ( DEFAULT_SMOKE_PATH, SMOKE_CASE_IDS, ensure_dataset, diff --git a/tests/test_locomo_plus_metrics.py b/tests/test_locomo_plus_metrics.py index 35dffcb92f..23789f8357 100644 --- a/tests/test_locomo_plus_metrics.py +++ b/tests/test_locomo_plus_metrics.py @@ -18,8 +18,8 @@ import pytest -from benchmark.locomo_plus.metrics import render_summary, summarize_observations -from benchmark.locomo_plus.prompts import ( +from evaluation.memory.locomo_plus.metrics import render_summary, summarize_observations +from evaluation.memory.locomo_plus.prompts import ( ANSWER_INSTRUCTIONS, build_answer_input, build_judge_input, diff --git a/tests/test_locomo_plus_runner.py b/tests/test_locomo_plus_runner.py index d5b04db293..86f238da5f 100644 --- a/tests/test_locomo_plus_runner.py +++ b/tests/test_locomo_plus_runner.py @@ -27,9 +27,9 @@ from pydantic_ai.messages import ModelResponse, RetryPromptPart, TextPart from pydantic_ai.models.function import FunctionModel -from benchmark.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn -from benchmark.locomo_plus import runner -from benchmark.locomo_plus.dataset import SMOKE_CASE_IDS, LoCoMoPlusCase, LoCoMoPlusDataset +from evaluation.memory.locomo.dataset import LoCoMoConversation, LoCoMoSession, LoCoMoTurn +from evaluation.memory.locomo_plus import runner +from evaluation.memory.locomo_plus.dataset import SMOKE_CASE_IDS, LoCoMoPlusCase, LoCoMoPlusDataset from powercontext.builtin.artifacts.memory import ( EmbeddingProfile, LLMMemoryCandidatePipeline, @@ -40,6 +40,7 @@ ) from powercontext.builtin.inference import EmbeddingResult, InvalidInferenceOutputError from powercontext.builtin.inference.pydantic_ai import InferenceLimits, PydanticAIStructuredGenerator +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinConfig, InferenceConfig, open_builtin_runtime from powercontext.builtin.sources import ContentSource @@ -82,7 +83,6 @@ def _dataset() -> LoCoMoPlusDataset: def _settings() -> ServerSettings: return ServerSettings( - database=SQLiteConfig(), inference=InferenceConfig( generation_model="test-answer", embedding_model="test-embedding", @@ -188,6 +188,430 @@ async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) +@pytest.fixture +def offline_benchmark(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """Keep database operations real while controlling the answer and judge.""" + state: dict[str, Any] = {"judge_valid": True} + + @asynccontextmanager + async def runtime_factory(config: BuiltinConfig): + async with open_builtin_runtime( + config.model_copy(update={"inference": InferenceConfig()}), + candidate_pipeline=_CandidatePipeline(), + embedding_model=_EmbeddingModel(), + ) as runtime: + yield runtime + + async def open_model(name, settings, resources): + async def respond(messages, info): + if name == "test-answer": + output = "A walk could help; it helped your mood before." + else: + output = ( + '{"label":"correct","reason":"Mentions the prior walk."}' + if state["judge_valid"] + else "unparseable verdict" + ) + return ModelResponse(parts=[TextPart(output)]) + + return FunctionModel(respond, model_name=name) + + monkeypatch.setattr(runner, "open_builtin_runtime", runtime_factory) + monkeypatch.setattr(runner, "open_model", open_model) + return state + + +@pytest.mark.parametrize("configured", [False, True]) +def test_memory_uses_configured_database_or_result_directory_fallback( + tmp_path: Path, offline_benchmark: dict[str, Any], configured: bool +) -> None: + output_directory = tmp_path / "results" + database_path = tmp_path / "configured.sqlite3" if configured else output_directory / "state.sqlite3" + settings = _settings() + if configured: + settings.database = SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}") + summary = asyncio.run( + runner.run_benchmark( + _dataset(), + settings=settings, + output_directory=output_directory, + run_id="database-choice", + judge_model="test-judge", + arm="memory-source", + limit=1, + ) + ) + assert summary["overall"]["completed_count"] == 1, _rows(output_directory) + assert database_path.exists() + assert summary["configuration"]["database_kind"] == "sqlite" + assert summary["configuration"]["database_fingerprint"] + assert summary["configuration"]["persistence"] == ( + "configured database; isolated run scopes" if configured else "isolated output-directory/state.sqlite3" + ) + if configured: + assert not (output_directory / "state.sqlite3").exists() + + async def read_persisted_scope(): + async with open_builtin_runtime( + BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}")) + ) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.get(_rows(output_directory)[-1]["scope_id"]) + memories = await runtime.memory.for_scope(scope.scope_id).list() + assert len(memories.entries) == 3 + + asyncio.run(read_persisted_scope()) + + +def test_explicit_in_memory_database_is_rejected_instead_of_silently_using_a_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = _settings() + settings.database = SQLiteConfig() + + def unexpected_service(*args, **kwargs): + pytest.fail("a nonresumable database must be rejected before opening services") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + with pytest.raises(ValueError, match="persistent database"): + asyncio.run( + runner.run_benchmark( + _dataset(), + settings=settings, + output_directory=tmp_path, + run_id="in-memory", + judge_model="test-judge", + arm="memory-source", + limit=1, + ) + ) + assert not (tmp_path / "state.sqlite3").exists() + + +def test_separate_results_isolate_equal_run_ids_in_a_shared_database( + tmp_path: Path, offline_benchmark: dict[str, Any] +) -> None: + settings = _settings() + settings.database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'shared.sqlite3'}") + output_directories = (tmp_path / "first", tmp_path / "second") + scope_ids = [] + for directory in output_directories: + summary = asyncio.run( + runner.run_benchmark( + _dataset(), + settings=settings, + output_directory=directory, + run_id="same-run-id", + judge_model="test-judge", + arm="memory-source", + limit=1, + ) + ) + assert summary["overall"]["completed_count"] == 1, _rows(directory) + scope_ids.append(_rows(directory)[-1]["scope_id"]) + assert scope_ids[0] != scope_ids[1] + + async def read_both_scopes(): + async with open_builtin_runtime(BuiltinConfig(database=settings.database)) as runtime: + assert runtime.scopes is not None + for scope_id in scope_ids: + descriptor = await runtime.scopes.get(scope_id) + page = await runtime.memory.for_scope(descriptor.scope_id).list() + assert len(page.entries) == 3 + + asyncio.run(read_both_scopes()) + + +def test_resume_keeps_original_scope_and_rejects_a_different_database_before_opening_services( + tmp_path: Path, offline_benchmark: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + settings = _settings() + settings.database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'original.sqlite3'}") + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": settings, + "output_directory": tmp_path / "results", + "run_id": "resume-database", + "judge_model": "test-judge", + "arm": "memory-source", + "limit": 1, + } + offline_benchmark["judge_valid"] = False + first = asyncio.run(runner.run_benchmark(**parameters)) + assert first["overall"]["failures_by_stage"]["judge"] == 1 + original_scope = _rows(parameters["output_directory"])[-1]["scope_id"] + ingestion_path = parameters["output_directory"] / "ingestion.json" + original_ingestion = ingestion_path.read_bytes() + + def unexpected_service(*args, **kwargs): + pytest.fail("changing a resumed database must be rejected before opening models or databases") + + new_database = tmp_path / "different.sqlite3" + with monkeypatch.context() as guarded: + guarded.setattr(runner, "open_builtin_runtime", unexpected_service) + guarded.setattr(runner, "open_model", unexpected_service) + changed_settings = settings.model_copy( + update={"database": SQLiteConfig(url=f"sqlite+aiosqlite:///{new_database}")} + ) + changed_parameters: dict[str, Any] = {**parameters, "settings": changed_settings} + with pytest.raises(ValueError, match="run identity changed"): + asyncio.run(runner.run_benchmark(**changed_parameters)) + assert not new_database.exists() + assert ingestion_path.read_bytes() == original_ingestion + + offline_benchmark["judge_valid"] = True + resumed = asyncio.run(runner.run_benchmark(**parameters)) + assert resumed["overall"]["completed_count"] == 1 + assert _rows(parameters["output_directory"])[-1]["scope_id"] == original_scope + assert ingestion_path.read_bytes() == original_ingestion + + +def test_resume_rejects_recreated_database_without_replacing_saved_scope( + tmp_path: Path, offline_benchmark: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + database_path = tmp_path / "recreated.sqlite3" + settings = _settings() + settings.database = SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}") + output_directory = tmp_path / "results" + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": settings, + "output_directory": output_directory, + "run_id": "lost-database", + "judge_model": "test-judge", + "arm": "memory-source", + "limit": 1, + } + offline_benchmark["judge_valid"] = False + first = asyncio.run(runner.run_benchmark(**parameters)) + assert first["overall"]["failures_by_stage"]["judge"] == 1 + ingestion_path = output_directory / "ingestion.json" + original_ingestion = ingestion_path.read_bytes() + database_path.unlink() + + async def read_recreated_database(): + async with open_builtin_runtime(BuiltinConfig(database=settings.database)) as runtime: + assert runtime.scopes is not None + return tuple(scope.scope_id for scope in await runtime.scopes.list()) + + empty_database_scopes = asyncio.run(read_recreated_database()) + assert _rows(output_directory)[-1]["scope_id"] not in empty_database_scopes + + async def unexpected_model(*args, **kwargs): + pytest.fail("a lost persisted scope must be rejected before spending on models") + + monkeypatch.setattr(runner, "open_model", unexpected_model) + resumed = asyncio.run(runner.run_benchmark(**parameters)) + assert resumed["overall"]["completed_count"] == 0 + assert resumed["overall"]["failures_by_stage"]["infrastructure"] == 1 + assert _rows(output_directory)[-1]["error"]["type"] == "ScopeNotFoundError" + assert ingestion_path.read_bytes() == original_ingestion + + assert asyncio.run(read_recreated_database()) == empty_database_scopes + + +def test_legacy_manifest_remains_replayable_but_cannot_resume( + tmp_path: Path, offline_benchmark: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": _settings(), + "output_directory": tmp_path, + "run_id": "legacy", + "judge_model": "test-judge", + "arm": "query-only", + "limit": 1, + } + completed = asyncio.run(runner.run_benchmark(**parameters)) + assert completed["overall"]["completed_count"] == 1 + manifest_path = tmp_path / "run.json" + manifest = json.loads(manifest_path.read_text()) + manifest.pop("scope_namespace") + manifest["configuration"].pop("database_fingerprint") + manifest_path.write_text(json.dumps(manifest)) + + def unexpected_service(*args, **kwargs): + pytest.fail("legacy artifact replay and rejected resume must not open services") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + assert runner.replay_results(tmp_path)["overall"]["completed_count"] == 1 + with pytest.raises(ValueError, match="new output directory"): + asyncio.run(runner.run_benchmark(**parameters)) + + +@pytest.mark.parametrize( + ("original_url", "resumed_url"), + [ + ( + "mysql+aoceanbase://tenant:authority-secret@db.example:2881/benchmark?charset=utf8mb4&password=old-secret", + "mysql+aoceanbase://tenant:authority-secret@db.example:2881/benchmark?charset=utf8mb4&password=new-secret", + ), + ( + "mysql+aoceanbase://tenant:old-secret@db.example:2881/benchmark?charset=utf8mb4", + "mysql+aoceanbase://tenant:new-secret@db.example:2881/benchmark?charset=utf8mb4", + ), + ( + "mysql+aoceanbase://tenant:old-secret@db.example:2881/benchmark?charset=utf8mb4", + "mysql+aoceanbase://ignored:new-secret@other.example:2882/ignored" + "?charset=utf8mb4&user=tenant&host=db.example&port=2881&db=benchmark", + ), + ( + "mysql+aoceanbase://tenant:password@db.example:2881/benchmark" + "?charset=utf8mb4&ssl_key=old-secret&auth_plugin=old-auth&server_public_key=old-key", + "mysql+aoceanbase://tenant:password@db.example:2881/benchmark" + "?charset=utf8mb4&ssl_key=new-secret&auth_plugin=new-auth&server_public_key=new-key", + ), + ( + "mysql+aoceanbase://tenant:password@db.example:2881/benchmark" + "?charset=utf8mb4&unix_socket=/evaluation/database.sock", + "mysql+aoceanbase://tenant:password@ignored.example:2882/benchmark" + "?charset=utf8mb4&unix_socket=/evaluation/database.sock&init_command=USE+ignored", + ), + ], + ids=["query-password", "authority-password", "effective-target-overrides", "authentication-options", "unix-socket"], +) +def test_oceanbase_resume_preserves_effective_target_when_credentials_or_spelling_change( + tmp_path: Path, + offline_benchmark: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + original_url: str, + resumed_url: str, +) -> None: + settings = _settings() + settings.database = OceanBaseConfig.model_validate({"url": original_url}) + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": settings, + "output_directory": tmp_path, + "run_id": "credential-rotation", + "judge_model": "test-judge", + "arm": "query-only", + "limit": 1, + } + completed = asyncio.run(runner.run_benchmark(**parameters)) + assert completed["overall"]["completed_count"] == 1 + manifest_path = tmp_path / "run.json" + original_manifest = manifest_path.read_bytes() + + def unexpected_service(*args, **kwargs): + pytest.fail("resuming a completed run must not open a database or model") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + settings.database = OceanBaseConfig.model_validate({"url": resumed_url}) + + assert asyncio.run(runner.run_benchmark(**parameters)) == completed + assert manifest_path.read_bytes() == original_manifest + assert completed["configuration"]["database_fingerprint_version"] == "oceanbase-target-v2" + assert all(secret not in original_manifest for secret in (b"old-secret", b"new-secret", b"authority-secret")) + + +@pytest.mark.parametrize( + "target_override", + ["host=other.example", "port=2882", "user=other-tenant", "db=other-database", "unix_socket=/other/database.sock"], +) +def test_oceanbase_resume_rejects_effective_target_changes_before_opening_services( + tmp_path: Path, offline_benchmark: dict[str, Any], monkeypatch: pytest.MonkeyPatch, target_override: str +) -> None: + url = "mysql+aoceanbase://tenant:password@db.example:2881/benchmark?charset=utf8mb4" + settings = _settings() + settings.database = OceanBaseConfig.model_validate({"url": url}) + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": settings, + "output_directory": tmp_path, + "run_id": "database-routing", + "judge_model": "test-judge", + "arm": "query-only", + "limit": 1, + } + assert asyncio.run(runner.run_benchmark(**parameters))["overall"]["completed_count"] == 1 + original_manifest = (tmp_path / "run.json").read_bytes() + + def unexpected_service(*args, **kwargs): + pytest.fail("a changed target must be rejected before opening a database or model") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + settings.database = OceanBaseConfig.model_validate({"url": f"{url}&{target_override}"}) + with pytest.raises(ValueError, match="run identity changed"): + asyncio.run(runner.run_benchmark(**parameters)) + assert (tmp_path / "run.json").read_bytes() == original_manifest + + +@pytest.mark.parametrize( + "options", + [ + "db=first&db=private-secret", + "password=first&password=private-secret", + "read_default_file=private-secret", + "read_default_group=private-secret", + "sql_mode=private-secret", + ], +) +def test_oceanbase_rejects_ambiguous_routing_before_opening_services( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, options: str +) -> None: + settings = _settings() + settings.database = OceanBaseConfig.model_validate({ + "url": f"mysql+aoceanbase://tenant:password@db.example:2881/benchmark?charset=utf8mb4&{options}" + }) + + def unexpected_service(*args, **kwargs): + pytest.fail("ambiguous routing must be rejected before opening a database or model") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + with pytest.raises(ValueError, match="OceanBase") as error: + asyncio.run( + runner.run_benchmark( + _dataset(), + settings=settings, + output_directory=tmp_path, + run_id="ambiguous-routing", + judge_model="test-judge", + arm="query-only", + limit=1, + ) + ) + assert "private-secret" not in str(error.value) + assert not (tmp_path / "run.json").exists() + + +def test_unversioned_oceanbase_manifest_is_replayable_but_cannot_resume( + tmp_path: Path, offline_benchmark: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + settings = _settings() + settings.database = OceanBaseConfig.model_validate({ + "url": "mysql+aoceanbase://tenant:password@db.example:2881/benchmark?charset=utf8mb4" + }) + parameters: dict[str, Any] = { + "dataset": _dataset(), + "settings": settings, + "output_directory": tmp_path, + "run_id": "old-oceanbase-identity", + "judge_model": "test-judge", + "arm": "query-only", + "limit": 1, + } + assert asyncio.run(runner.run_benchmark(**parameters))["overall"]["completed_count"] == 1 + manifest_path = tmp_path / "run.json" + manifest = json.loads(manifest_path.read_text()) + manifest["configuration"].pop("database_fingerprint_version") + manifest_path.write_text(json.dumps(manifest)) + + def unexpected_service(*args, **kwargs): + pytest.fail("legacy replay or rejected resume must not open a database or model") + + monkeypatch.setattr(runner, "open_builtin_runtime", unexpected_service) + monkeypatch.setattr(runner, "open_model", unexpected_service) + assert runner.replay_results(tmp_path)["overall"]["completed_count"] == 1 + with pytest.raises(ValueError, match="new output directory"): + asyncio.run(runner.run_benchmark(**parameters)) + + def test_memory_source_arm_captures_flushes_searches_and_expands_real_sqlite_sources( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/uv.lock b/uv.lock index e331629544..d3b0473bc2 100644 --- a/uv.lock +++ b/uv.lock @@ -2703,6 +2703,7 @@ builtin = [ { name = "pyyaml" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlite-vec" }, + { name = "tzdata" }, ] cli = [ { name = "httpx", extra = ["socks"] }, @@ -2742,6 +2743,7 @@ seekdb = [ { name = "pyyaml" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlite-vec" }, + { name = "tzdata" }, ] server = [ { name = "aiosqlite" }, @@ -2766,6 +2768,7 @@ server = [ { name = "scalar-fastapi" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlite-vec" }, + { name = "tzdata" }, { name = "uvicorn" }, ] tracing-otlp = [ @@ -2875,6 +2878,9 @@ requires-dist = [ { name = "tree-sitter-typescript", marker = "extra == 'code'", specifier = "==0.23.2" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, + { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, ] provides-extras = ["builtin", "cli", "client", "code", "seekdb", "server", "tracing-otlp"]