From 8d414dd3a3225b3d9a5ab1a8fd6803c59acf6470 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Wed, 18 Nov 2026 23:28:39 +0800 Subject: [PATCH 1/3] refactor(apps): group application workflows in a workflows package Signed-off-by: zengchuang Agent-assisted-by: Codex (author) --- docs/apps-module-migrations.json | 127 ++++++++++++++ docs/architecture.md | 34 ++-- docs/contracts/function-owners.md | 2 +- docs/harness-improvement.md | 2 +- docs/knowledge-retrieval-trace.md | 2 +- docs/local-skill-evaluation.md | 2 +- docs/module-layout.md | 45 +++-- docs/operations.md | 2 +- docs/optimization-workflow-unification.md | 38 ++++- docs/skill-evaluation.md | 2 +- src/inferrsi/apps/benchmark_operations.py | 20 +-- src/inferrsi/apps/campaign.py | 157 +----------------- src/inferrsi/apps/cli/__init__.py | 4 +- src/inferrsi/apps/improvement.py | 30 +--- src/inferrsi/apps/skill_eval_engineering.py | 26 +-- src/inferrsi/apps/skill_evaluation.py | 62 +------ src/inferrsi/apps/views/capture_delivery.py | 2 +- src/inferrsi/apps/views/optimization_story.py | 2 +- src/inferrsi/apps/workflows/__init__.py | 1 + .../apps/workflows/benchmark_operations.py | 19 +++ src/inferrsi/apps/workflows/campaign.py | 156 +++++++++++++++++ src/inferrsi/apps/workflows/improvement.py | 29 ++++ .../apps/workflows/skill_eval_engineering.py | 29 ++++ .../apps/workflows/skill_evaluation.py | 61 +++++++ src/inferrsi/harness/optimization_session.py | 2 +- tests/_gate_policy_support.py | 5 +- tests/check_installed.py | 33 +++- tests/fixtures/dashboard_profile_review.py | 2 +- tests/fixtures/knowledge_retrieval.py | 2 +- tests/test_anthropic_agent.py | 2 +- tests/test_apps_module_migrations.py | 37 +++-- tests/test_benchmark_operations.py | 2 +- tests/test_campaign_record_injection.py | 2 +- tests/test_campaign_task_selection.py | 2 +- tests/test_harness_improvement.py | 4 +- tests/test_knowledge_navigation_browser.py | 2 +- tests/test_optimization_session.py | 122 +++++++++++++- tests/test_optimization_story.py | 2 +- tests/test_package_root_inventory.py | 1 + tests/test_project_optimization.py | 2 +- tests/test_search_archive.py | 2 +- tests/test_search_attribution.py | 2 +- tests/test_skill_eval_compare.py | 4 +- tests/test_skill_eval_engineering.py | 23 ++- tests/test_skill_eval_generation.py | 4 +- tests/test_skill_eval_local.py | 2 +- tests/test_skill_eval_local_cli.py | 2 +- tests/test_skill_eval_pipeline.py | 2 +- tests/test_skill_evaluation.py | 2 +- tests/test_target_module_import_gate.py | 4 +- tools/skill_eval_cloud_replay.py | 2 +- tools/skill_eval_compare.py | 8 +- tools/skill_eval_improve.py | 10 +- tools/skill_eval_local.py | 2 +- tools/skill_eval_pilot.py | 4 +- tools/skill_eval_qualify.py | 2 +- 56 files changed, 779 insertions(+), 372 deletions(-) create mode 100644 src/inferrsi/apps/workflows/__init__.py create mode 100644 src/inferrsi/apps/workflows/benchmark_operations.py create mode 100644 src/inferrsi/apps/workflows/campaign.py create mode 100644 src/inferrsi/apps/workflows/improvement.py create mode 100644 src/inferrsi/apps/workflows/skill_eval_engineering.py create mode 100644 src/inferrsi/apps/workflows/skill_evaluation.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 4040cca2a..28429c515 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -21,6 +21,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/archive.py", "src/inferrsi/apps/delivery/__init__.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", @@ -50,6 +51,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/artifact_reader.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/views/attempt.py", "tests/check_installed.py", @@ -80,6 +82,7 @@ "src/inferrsi/apps/delivery/adapters.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", + "src/inferrsi/apps/result.py", "src/inferrsi/apps/storage/archive.py", "src/inferrsi/apps/storage/artifact_reader.py", "src/inferrsi/apps/views/attempt.py", @@ -100,6 +103,7 @@ "docs/contracts/current-design.md", "docs/module-layout.md", "docs/operations.md", + "src/inferrsi/apps/evidence_layout.py", "src/inferrsi/contracts/evidence.py", "tests/_gate_policy_support.py", "tests/check_installed.py", @@ -125,6 +129,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/views/attempt.py", "tests/check_installed.py", @@ -146,6 +151,7 @@ "docs/architecture.md", "docs/module-layout.md", "docs/operations.md", + "src/inferrsi/apps/context_provider.py", "tests/check_installed.py", "tests/test_apps_context_provider.py", "tests/test_apps_module_migrations.py", @@ -164,8 +170,129 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/candidate_provider.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py" ] + }, + { + "old": "inferrsi.apps.campaign", + "canonical": "inferrsi.apps.workflows.campaign", + "exports": [ + "create_campaign", + "capture_context", + "save_retrieval", + "verify_capture" + ], + "consumers": [ + "docs/architecture.md", + "docs/knowledge-retrieval-trace.md", + "docs/module-layout.md", + "docs/operations.md", + "docs/optimization-workflow-unification.md", + "src/inferrsi/apps/cli/__init__.py", + "src/inferrsi/apps/views/capture_delivery.py", + "src/inferrsi/apps/views/optimization_story.py", + "src/inferrsi/harness/optimization_session.py", + "tests/check_installed.py", + "tests/fixtures/dashboard_profile_review.py", + "tests/fixtures/knowledge_retrieval.py", + "tests/test_apps_module_migrations.py", + "tests/test_campaign_record_injection.py", + "tests/test_campaign_task_selection.py", + "tests/test_knowledge_navigation_browser.py", + "tests/test_optimization_session.py", + "tests/test_optimization_story.py", + "tests/test_search_archive.py", + "tests/test_search_attribution.py" + ] + }, + { + "old": "inferrsi.apps.benchmark_operations", + "canonical": "inferrsi.apps.workflows.benchmark_operations", + "exports": [ + "run_task" + ], + "consumers": [ + "docs/architecture.md", + "docs/module-layout.md", + "src/inferrsi/apps/cli/__init__.py", + "tests/_gate_policy_support.py", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_operations.py", + "tests/test_optimization_session.py", + "tests/test_package_root_inventory.py", + "tests/test_project_optimization.py", + "tests/test_target_module_import_gate.py" + ] + }, + { + "old": "inferrsi.apps.improvement", + "canonical": "inferrsi.apps.workflows.improvement", + "exports": [ + "assessment", + "choose", + "holdout_accepts", + "rerun_assessment" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/harness-improvement.md", + "docs/module-layout.md", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_harness_improvement.py", + "tools/skill_eval_improve.py" + ] + }, + { + "old": "inferrsi.apps.skill_evaluation", + "canonical": "inferrsi.apps.workflows.skill_evaluation", + "exports": [ + "export_bundle", + "summarize", + "run_trial" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/module-layout.md", + "docs/skill-evaluation.md", + "tests/check_installed.py", + "tests/test_anthropic_agent.py", + "tests/test_apps_module_migrations.py", + "tests/test_skill_eval_compare.py", + "tests/test_skill_eval_generation.py", + "tests/test_skill_eval_pipeline.py", + "tests/test_skill_evaluation.py", + "tools/skill_eval_compare.py", + "tools/skill_eval_pilot.py" + ] + }, + { + "old": "inferrsi.apps.skill_eval_engineering", + "canonical": "inferrsi.apps.workflows.skill_eval_engineering", + "exports": [ + "verify", + "main" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/local-skill-evaluation.md", + "docs/module-layout.md", + "src/inferrsi/apps/workflows/skill_evaluation.py", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_skill_eval_compare.py", + "tests/test_skill_eval_engineering.py", + "tests/test_skill_eval_generation.py", + "tests/test_skill_eval_local.py", + "tests/test_skill_eval_local_cli.py", + "tools/skill_eval_cloud_replay.py", + "tools/skill_eval_compare.py", + "tools/skill_eval_local.py", + "tools/skill_eval_qualify.py" + ], + "script_entry": "main" } ] diff --git a/docs/architecture.md b/docs/architecture.md index d92b8eed6..fb22f5805 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,7 +7,7 @@ packages. `inferrsi benchmarks run ...` launches the package's `benchmark.py`. Task-specific orchestration, sampling and assessment live inside each package. The task library owns its `index.json` registration. The existing `benchmark/registry.py` discovers the resources. The CLI package `apps/cli/` -and `apps/benchmark_operations.py` provide application assembly and launch. +and `apps/workflows/benchmark_operations.py` provide application assembly and launch. Each task retains its own protocol and task-specific code. The [current-Agent optimization session](optimization-benchmarks.md) assembles @@ -16,7 +16,7 @@ budget, attempts, feedback records, selection and independent final execution. `harness/task_protocol.py` freezes a supplied resource directory and loads its task-owned assessment protocol. `harness/optimization_session.py` assembles execution and assessment; the CLI resolves task resources and injects knowledge -access and the capture callback from `apps/campaign.py`. +access and the capture callback from `apps/workflows/campaign.py`. The external Agent chooses and edits candidates between round submissions. The same controller serves ordinary projects through `campaign start/round/final` and packaged tasks through `benchmarks session`. The @@ -162,7 +162,6 @@ the comparator; the engine AB/BA path retains its independent acceptance contrac ```text src/inferrsi/ - apps/cli/ CLI entry `main` and its argument parser benchmark/ registry.py Generic task resource discovery tasks/ Task-specific definitions, workloads and assessment @@ -177,19 +176,6 @@ src/inferrsi/ skill_eval_diagnostic.py Synthetic KV admission diagnosis task and grader skill_eval_engineering.py Serving-parser task, public vectors and vector judgment improvement.py Improvement-step thresholds and acceptance rules - apps/ - api.py Python entry points for assembling and publishing Apps operations - benchmark_operations.py Resolve a task entry and launch it through Harness - archive.py Publish and read the standardized Apps output in a workspace - result.py Assemble and check the AppsResultV1 output envelope - skill_evaluation.py Composed bundle export, trial run and summary - skill_eval_engineering.py Composed artifact check: task rules around sandbox replay - improvement.py Composed assessment, selection, rerun and holdout acceptance - inspection.py Environment inspection bound to the host probe and Git identity - legacy/ - run.py Old run composed from Harness, Benchmark and the report - evaluation.py Old evaluate entry: RunSpec to the Benchmark gates - reporting.py HTML evidence report contracts/ runspec.py Immutable experiment contract; the initializer re-exports only RunSpec json_v1.py Canonical inferrsi-json-v1 encoder and decoder @@ -199,6 +185,13 @@ src/inferrsi/ __init__.py Import-free initializer repository_packs.py Neutral REPO_TYPES and pack.json reader; imports no inferrsi module apps/ + api.py Python entry points for assembling and publishing Apps operations + cli/ CLI entry `main` and its argument parser + inspection.py Environment inspection bound to the host probe and Git identity + legacy/ + run.py Old run composed from Harness, Benchmark and the report + evaluation.py Old evaluate entry: RunSpec to the Benchmark gates + reporting.py HTML evidence report handoff/ bridge.py Bind preparations to specs and fixed records to evidence context_provider.py Deliver Knowledge bytes and pins for Harness sealing @@ -208,7 +201,12 @@ src/inferrsi/ artifact_reader.py Read verified artifacts through authorized mounts result.py AppsResultV1 construction and validation evidence_layout.py Create plan/results/patch/validator evidence directories - campaign.py Knowledge values connected to Harness campaign records + workflows/ + campaign.py Knowledge values connected to Harness campaign records + benchmark_operations.py Resolve a task entry and launch it through Harness + skill_evaluation.py Composed bundle export, trial run and summary + skill_eval_engineering.py Composed artifact check: task rules around sandbox replay + improvement.py Composed assessment, selection, rerun and holdout acceptance mindie/ project.py Opt-in per-checkout grant, the worker launch and the recorded read context.py Export supplied MindIE reads into a caller-reviewed campaign trace @@ -295,7 +293,7 @@ The package root holds exactly two tracked files: `__init__.py` and `__main__.py `mcp.py` and the standalone `read_worker.py`) and deleted root `mindie_project.py`, `mindie_context.py`, `mindie_mcp.py` and `_mindie_read_worker.py` with no forwarding shell or alias. #408 A2 moved the chart renderer to -`apps/views/summary_chart.py` and the task launcher to `apps/benchmark_operations.py` +`apps/views/summary_chart.py` and the task launcher to `apps/workflows/benchmark_operations.py` and deleted root `summary_chart.py` and `benchmark_operations.py` the same way. #408 A1 moved the CLI and its parser to `apps/cli/` and deleted root `cli.py` and `arguments.py`; `__main__.py` and the console script call `inferrsi.apps.cli:main`. #412 R1 deleted eleven root modules with no diff --git a/docs/contracts/function-owners.md b/docs/contracts/function-owners.md index 6f147b30a..5d9f173ea 100644 --- a/docs/contracts/function-owners.md +++ b/docs/contracts/function-owners.md @@ -31,7 +31,7 @@ Any later split of a mixed function is accepted by real pytest/host/wheel/record ## Implemented rows -The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py` and `apps/improvement.py`. Two points follow the tree as it stands after #338, not the row text: +The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/workflows/skill_evaluation.py`, `apps/workflows/skill_eval_engineering.py` and `apps/workflows/improvement.py`. Two points follow the tree as it stands after #338, not the row text: - `usage_totals` stays in `harness/usage.py`, because `harness/optimization.py` consumes it for budget accounting. B `assessment` receives the baseline and candidate totals from Apps and does not import H. - `SelectionPolicy` holds only the two thresholds, so it lives whole in B. H `choose` receives the assessments, the validity block and the composition flag; it holds no threshold and no policy class. diff --git a/docs/harness-improvement.md b/docs/harness-improvement.md index df136e83c..d7d0abdbc 100644 --- a/docs/harness-improvement.md +++ b/docs/harness-improvement.md @@ -46,7 +46,7 @@ flowchart TB `src/inferrsi/benchmark/improvement.py` owns the thresholds and acceptance rules, `src/inferrsi/harness/improvement.py` the search progression and -`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/improvement.py` +`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/workflows/improvement.py` composes them. `tools/skill_eval_improve.py` owns freezing, declarative snapshot materialization and orchestration through `tools/skill_eval_compare.py`. There is no second diff --git a/docs/knowledge-retrieval-trace.md b/docs/knowledge-retrieval-trace.md index a59583476..816b44205 100644 --- a/docs/knowledge-retrieval-trace.md +++ b/docs/knowledge-retrieval-trace.md @@ -107,7 +107,7 @@ same immutable capture envelope used by `capture-context`: ```python from pathlib import Path from inferrsi.knowledge.catalog import search_knowledge -from inferrsi.apps.campaign import save_retrieval +from inferrsi.apps.workflows.campaign import save_retrieval result = search_knowledge("decode launch", explain=True) path, sha256 = save_retrieval( diff --git a/docs/local-skill-evaluation.md b/docs/local-skill-evaluation.md index 81e3cf44e..fae594063 100644 --- a/docs/local-skill-evaluation.md +++ b/docs/local-skill-evaluation.md @@ -23,7 +23,7 @@ snapshots. `tools/skill_eval_local.py` adds manual stage/capture, without launch an agent. `src/inferrsi/benchmark/skill_eval_engineering.py` owns the one task, its 19 public behavior probes and their judgment. `src/inferrsi/harness/artifact_sandbox.py` replays the retained artifact under bubblewrap and separates the single writable -artifact from frozen inputs; `src/inferrsi/apps/skill_eval_engineering.py` composes the two. +artifact from frozen inputs; `src/inferrsi/apps/workflows/skill_eval_engineering.py` composes the two. The existing cloud read/submit route is unchanged; it cannot perform this artifact-editing task and is rejected for this adapter. diff --git a/docs/module-layout.md b/docs/module-layout.md index 9c794f251..f3d4b400a 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -6,14 +6,14 @@ moved Knowledge bodies. R1 (#412) deleted the seven static root compatibility shells that K01 left, so those names import only from `inferrsi.knowledge`. The engineering task contribution now implements `benchmark/registry.py` and `benchmark/tasks/`. The CLI package `apps/cli/` and -`apps/benchmark_operations.py` assemble this slice. The former `observability/` +`apps/workflows/benchmark_operations.py` assemble this slice. The former `observability/` package is dissolved. Its campaign records, traces, logs and profile captures are in `harness/records/`, its dashboard views and templates are in `apps/views/`, and the collected-context projection is in `knowledge/context_projection.py`. #371 owns the Apps composition. Its accepted directory follow-up -[#490](https://github.com/InferMatrixLab/InferRSI/issues/490) starts with the -`apps/storage/` slice described below. +[#490](https://github.com/InferMatrixLab/InferRSI/issues/490) groups the implemented +storage, handoff and workflows slices described below. The old `run` chain is split by owner. Its gates are in `benchmark/legacy_run_v0.py`, its execution facts stay in `harness/`, and @@ -41,8 +41,8 @@ through these exports; new pickles use the canonical definition. The retired root `inferrsi.evidence_paths` has no alias and remains unavailable. Shim removal requires a separate reviewed PR recording consumer migration, -wheel and plugin validation, and the first removal version. Workflows, -integrations and integrated acceptance remain subsequent #490 slices. +wheel and plugin validation, and the first removal version. Integrations and +integrated acceptance remain subsequent #490 slices. ### Apps handoff migration @@ -61,6 +61,28 @@ path performs no Knowledge reads, device probes, process launches or service registration. Harness owns the sealed context, original attempt and request digest; these ownership boundaries and DTOs remain unchanged. +### Apps workflows migration + +`apps/workflows/` owns `campaign.py`, `benchmark_operations.py`, `improvement.py`, +`skill_evaluation.py` and `skill_eval_engineering.py`. It joins Knowledge reads, +Harness execution and Benchmark judgments. CLI commands, views and evaluation +tools import these canonical implementations. Its initializer imports nothing; +imports of either path perform no data reads, probes, process launches or service +registration. Signatures, return values and retained-record schemas are unchanged. + +The five former Apps paths keep explicit exports from the same +[migration inventory](apps-module-migrations.json). The engineering verifier also +keeps its former file and `python -m` entry points through a static absolute import +and a `__main__` guard, recorded as `script_entry` in the inventory. Other shims +contain only their docstring, explicit import and literal `__all__`. New pickles +use the canonical module; patches to helpers or imported dependencies target it. + +New optimization plans and evaluation controller snapshots fingerprint the +canonical source files, not the compatibility shims. A frozen pre-move runtime +map remains incompatible with a new runtime, including for `status` and `recover`. +Do not edit a frozen plan to bypass this check. Use the original pinned runtime +to resume, following [release-boundary recovery](optimization-workflow-unification.md#release-boundary-recovery). + ### Core and technical packages The four target modules are peers: `knowledge`, `harness`, `benchmark`, and @@ -89,7 +111,8 @@ The two are `__init__.py` and `__main__.py`. M1 moved root `mindie_project.py`, the four root files with no forwarding shell or alias. A2 moved root `summary_chart.py` to `apps/views/summary_chart.py` and root `benchmark_operations.py` to -`apps/benchmark_operations.py` and deleted both root files the same way. A1 moved root `cli.py` and +`apps/benchmark_operations.py` and deleted both root files the same way. The workflows +slice later moved that Apps implementation into `apps/workflows/`. A1 moved root `cli.py` and `arguments.py` to `apps/cli/__init__.py` and `apps/cli/arguments.py` and deleted both root files. A new root file now fails the guard, so a later module belongs in a subpackage. The build-generated `_version.py` is ignored wheel @@ -148,17 +171,17 @@ Several flat modules remain shared seams until their slice starts. | Current path | Tentative reading | This slice | | --- | --- | --- | | `apps/cli/` (`__init__.py`, `arguments.py`), root `__main__.py` | Apps CLI entry and its parser | A1 (#408) moved root `cli.py` and `arguments.py` here; the entry is `inferrsi.apps.cli:main`, and `__main__.py` only delegates to it | -| `apps/benchmark_operations.py` | Single-task application launcher | A2 (#408) moved root `benchmark_operations.py` here; resolves a task entry and invokes the Harness runner, and ordinary projects do not use it | +| `apps/workflows/benchmark_operations.py` | Single-task application launcher | A2 (#408) moved the root launcher into Apps; #490 groups its implementation under workflows. Resolves a task entry and invokes the Harness runner; ordinary projects do not use it | | `benchmark/registry.py`, `benchmark/tasks/` | Task discovery, definitions and specialized rules | Task-specific quality and measurement code stays with each task | | `harness/optimization_session.py`, `harness/project.py`, `harness/task_protocol.py` | Shared preparation, execution and retained-result assessment | CLI supplies task resources and knowledge access; ordinary project lifecycle does not import Benchmark | | `harness/optimization.py`, `harness/candidates.py`, `harness/usage.py` | Shared project and packaged-task lifecycle | Injected execution/assessment; declared file/directory candidates, selection direction and exact/up-to rounds | | `harness/agent_sessions.py` | Native host handoff records | Freezes requests and retains native IDs and exported observations; the initiating Agent calls host tools | | `harness/skill_evaluation.py`, `harness/artifact_sandbox.py`, `harness/improvement.py`, `harness/atomic_json.py` | Skill-evaluation host adapter, trial execution and retention, sandbox replay, search progression | Verdict rules and task definitions arrive as injected values; no Model, Infra, Knowledge, Benchmark or Apps import | | `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py`, `benchmark/improvement.py` | Skill-evaluation criteria, task definitions and acceptance rules | Pure judgments over retained facts; no Harness import and no process start | -| `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py`, `apps/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | +| `apps/workflows/skill_evaluation.py`, `apps/workflows/skill_eval_engineering.py`, `apps/workflows/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | | `knowledge/knowledge_operations.py`, `knowledge/knowledge_store.py`, `knowledge/catalog.py`, `knowledge/research.py`, `knowledge/skill_catalog.py`, `knowledge/_skill_selection.py`, `knowledge/_source_snapshot.py` | The seven Knowledge implementations, moved by K01 | Interfaces unchanged; the same-named root shells were deleted by R1 | -| `harness/records/` | Campaign manifests, traces, logs, profile captures and delivered-context captures | Imports no Knowledge or Apps module; `apps/campaign.py` supplies the backend names, the selection default and the context projection | -| `apps/views/`, `apps/campaign.py` | Dashboard views, templates and the Knowledge-to-records connection | Apps composition under #371; campaign organization follows in #490 | +| `harness/records/` | Campaign manifests, traces, logs, profile captures and delivered-context captures | Imports no Knowledge or Apps module; `apps/workflows/campaign.py` supplies the backend names, the selection default and the context projection | +| `apps/views/`, `apps/workflows/campaign.py` | Dashboard views, templates and the Knowledge-to-records connection | Apps composition under #371; campaign composition is grouped under workflows by #490 | | `apps/mindie/` (`project.py`, `context.py`, `mcp.py`, `read_worker.py`) | The local MindIE adapter: project grant, stdio MCP, context export and the read worker | M1 (#452) moved the four root MindIE files here; no Harness, Knowledge or Infra module names MindIE, and the initializer imports nothing | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `harness/runner.py`, `harness/scenarios/external.py`, `harness/atomic_json.py` | Old run execution facts and Harness-owned JSON writes | Ports injected; no Model edge, and no Infra edge except the local process executor | @@ -561,7 +584,7 @@ A2 (#408) deleted two more root modules the same way: | Retired root module | Canonical module | | --- | --- | | `inferrsi.summary_chart` | `inferrsi.apps.views.summary_chart` | -| `inferrsi.benchmark_operations` | `inferrsi.apps.benchmark_operations` | +| `inferrsi.benchmark_operations` | `inferrsi.apps.workflows.benchmark_operations` | `tools/campaign_summary_chart.py` keeps its command line and calls `inferrsi.apps.views.summary_chart:main`, so its arguments, exit codes, output and diff --git a/docs/operations.md b/docs/operations.md index 9c1cd3081..6bbe09e99 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -27,7 +27,7 @@ Effect classes used below: | Intent | Entry | Effect class | Boundary | | --- | --- | --- | --- | -| Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution | +| Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution using its [pinned runtime](optimization-workflow-unification.md#release-boundary-recovery) | | Optimize an ordinary project in the current Agent | CLI `campaign start/knowledge/round/status/final/recover` | Experiment-executing / local records | Harness freezes explicit caller-provided checks, manages multi-file candidates, executes and assesses retained results. This path does not depend on Benchmark or infer acceptance from the goal. See [shared workflow](optimization-workflow-unification.md) | | Reuse one project service across multi-scenario checks | CLI `campaign start --evaluation-mode service_reuse`, then `baseline/round/status/recover/final` | Experiment-executing / local records | Caller supplies the protocol, service and profiler scripts; `resume`, `refresh-references` and explicit retry retain recovery decisions. See [service reuse](service-reuse-campaigns.md) | | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | diff --git a/docs/optimization-workflow-unification.md b/docs/optimization-workflow-unification.md index 09d127a77..b3fef36b7 100644 --- a/docs/optimization-workflow-unification.md +++ b/docs/optimization-workflow-unification.md @@ -32,7 +32,7 @@ CLI `final` 返回后自动生成 `delivery//report.md`、离线 `rep ## 时序图 -以下是普通项目默认 AB/BA 模式的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 +以下是普通项目默认 AB/BA 模式的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/workflows/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 先准备任务条件,再调用 start。start 内复制任务、脚本和基线并写入计划/状态,不执行测量。Agent 逐次调用 round,最后调用 final 独立重测已保存的最佳版本。 @@ -151,7 +151,7 @@ Harness 没有常驻 Agent 驱动。Agent 逐次调用这些操作,代码在 两个优化入口都调用 `harness.optimization_session.session_command`,使用同一个 `harness.optimization` 控制器。AB/BA 与题包使用 `evaluate_round`;显式 service_reuse 使用该控制器内的 `service_command` 和模式分支,共用锁及单一控制记录。执行与评测适配也在 Harness 内;普通项目整个生命周期不导入 Benchmark。 -题库 `start` 由 CLI 经 `benchmark.registry` 定位资源目录,再交给 `harness.task_protocol.prepare` 固化;后续从冻结目录加载题包自己的 `optimization.py`,无需再次查询注册表。专用协议检查仍属于题包。两类 `assess` 都只读留存结果,不重新测量。KnowledgeReader 与 `apps/campaign.py` 的记录回调由 CLI 注入,Harness 不直接导入 Benchmark 或目标 Knowledge 模块,也不经记录模块间接引入知识实现。 +题库 `start` 由 CLI 经 `benchmark.registry` 定位资源目录,再交给 `harness.task_protocol.prepare` 固化;后续从冻结目录加载题包自己的 `optimization.py`,无需再次查询注册表。专用协议检查仍属于题包。两类 `assess` 都只读留存结果,不重新测量。KnowledgeReader 与 `apps/workflows/campaign.py` 的记录回调由 CLI 注入,Harness 不直接导入 Benchmark 或目标 Knowledge 模块,也不经记录模块间接引入知识实现。 本次交付可以分成三块:**Benchmark 的优化题目、Harness 的优化管理与验收、Harness 与宿主协作的新会话派发和回收**。第三块不改变第二块的生命周期;新会话仍自行 start 到 final。 @@ -219,6 +219,40 @@ inferrsi dashboard --input --output 执行器抛出异常时,若已有完成记录则收集该记录,不重复启动;没有记录则标记 blocked,清理状态保持未知,不自动重试。进程被中断后的 `recover` 只接收已有结果,不重新启动命令;没有完成记录时,需确认原进程及资源清理。运行中的旧 session 应继续使用其冻结 runtime,不能直接换新代码恢复。 +### Release-boundary recovery + +Apps workflows 目录迁移会改变运行时摘要的路径和控制器源码。迁移前的 +`plan.json` 即使只是执行 `status` 或 `recover`,在新版本下也会被拒绝。 +保留原始计划、冻结输入和所有已留存结果,不要替换摘要、重新格式化计划或 +把旧运行当作新运行初始化。评测工具的冻结 controller 快照也遵循相同原则。 + +恢复时,从运行准备记录取得创建该计划时的精确软件版本和 Python 环境。 +`runtime_identity` 是校验表,不是可安装的软件包;单靠摘要不能重建原始环境。 +优先使用先前留存的虚拟环境或 wheel。若只有精确源码版本,应在独立目录恢复 +该版本及原依赖,构建 wheel 并安装到独立环境;不要覆盖新版本环境。 + +```text + -I -m inferrsi benchmarks session status --session + -I -m inferrsi benchmarks session recover --session +``` + +普通项目对应 `campaign status/recover --directory `。 +原环境的所有文件摘要与计划匹配后,Harness 才会接收已有 execution 记录; +恢复不会再次执行已完成的候选。无法取得匹配的原始环境时保持拒绝,保留证据。 +需要在新版本继续实验,应另开新运行,明确新的版本和冻结输入。 + +跨版本回归使用两个独立安装环境:迁移前 runtime 创建计划并在执行完成后 +中断;新 runtime 的 status/recover 必须拒绝且不改动计划和结果;原 runtime +恢复同一运行并验证候选只执行一次。设置 `INFERRSI_PRE_MOVE_PYTHON` 指向 +固定旧 wheel 的解释器,`INFERRSI_POST_MOVE_PYTHON` 指向新 wheel 的解释器后运行: + +```text +python -m pytest tests/test_optimization_session.py::test_pinned_pre_move_wheel_recovers_its_original_plan +``` + +未提供旧 runtime 时该安装集成用例明确跳过;默认回归仍检查旧路径的摘要表 +被拒绝、计划字节不变,以及新计划摘要读取 `apps/workflows/campaign.py` 的实际源码。 + 判定器异常按基础设施失败结算,`assessment.reason` 保留异常类型和消息,避免普通编程错误被隐藏成无类型的失败描述。执行器异常的原始类型和消息保存在该次尝试的 `coordinator-error.json` 中。此处捕获异常用于保留失败状态,不代表候选通过;进程中断仍沿用上述恢复流程。 每轮 token 只接受实际增量导出,未知不补零。整会话用量另存,不重复叠加到曲线。终止后的总耗时固定,之后打开 dashboard 不增加。HTML 只投影已清洗字段;原始本地记录可能含命令、路径等私有信息,公开前需要检查。 diff --git a/docs/skill-evaluation.md b/docs/skill-evaluation.md index 739d1ea96..1f42f440d 100644 --- a/docs/skill-evaluation.md +++ b/docs/skill-evaluation.md @@ -22,7 +22,7 @@ platform source is copied. Existing knowledge retains its source/license limits. Ownership follows the module split. `inferrsi.harness.skill_evaluation` adapts the client, executes each trial and retains its facts. `inferrsi.benchmark.skill_evaluation` holds check validity, the task verdict and condition statistics. -`inferrsi.apps.skill_evaluation` composes both with the bundle export behind the +`inferrsi.apps.workflows.skill_evaluation` composes both with the bundle export behind the `run_trial`, `summarize` and `export_bundle` signatures; retained trial records are unchanged. diff --git a/src/inferrsi/apps/benchmark_operations.py b/src/inferrsi/apps/benchmark_operations.py index 7770c3f53..c425b6182 100644 --- a/src/inferrsi/apps/benchmark_operations.py +++ b/src/inferrsi/apps/benchmark_operations.py @@ -1,19 +1,5 @@ -"""Resolve packaged tasks at the application boundary and launch their entries.""" +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.benchmark_operations`.""" -import subprocess -import sys -from pathlib import Path +from .workflows.benchmark_operations import run_task -from inferrsi.benchmark.registry import task_resources -from inferrsi.harness import optimization_session - - -def run_task(name: str, arguments: list[str]) -> int: - entry = task_resources(name).joinpath("benchmark.py") - if not entry.is_file(): - raise ValueError(f"Missing benchmark.py entry for task: {name}") - return subprocess.call( - [sys.executable, str(Path(optimization_session.__file__).resolve()), - str(entry), *arguments], - env=optimization_session.runtime_environment(), - ) +__all__ = ['run_task'] diff --git a/src/inferrsi/apps/campaign.py b/src/inferrsi/apps/campaign.py index b30c2aefb..4ac664831 100644 --- a/src/inferrsi/apps/campaign.py +++ b/src/inferrsi/apps/campaign.py @@ -1,156 +1,5 @@ -"""Connect Knowledge values to the campaign records that Harness keeps. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.campaign`.""" -Harness records never import Knowledge. This module supplies the backend -vocabulary, the selection default, the context projection and the capture -orchestration they need: it calls the reader, keeps the retrieval envelope -the reader returned and hands Harness only the bytes to store. -""" +from .workflows.campaign import capture_context, create_campaign, save_retrieval, verify_capture -from __future__ import annotations - -from pathlib import Path - -from inferrsi import __version__ -from inferrsi.harness.records import campaign_records, knowledge_capture -from inferrsi.knowledge._skill_selection import TASK_RELEVANCE -from inferrsi.knowledge.catalog import BACKENDS -from inferrsi.knowledge.context_projection import ( - PROVENANCE_FIELDS, - assemble_context_text, - context_item, -) - - -def create_campaign(root: Path, campaign_id: str, *, target: float | None = None, - noise: float | None = None, synthetic: bool = False, - backend: str | None = None, profile_first: bool = False) -> Path: - """Create a campaign manifest that accepts the Knowledge backend names.""" - return campaign_records.create_campaign( - root, campaign_id, target=target, noise=noise, synthetic=synthetic, backend=backend, - profile_first=profile_first, backends=BACKENDS, - ) - - -def capture_context(root: Path, *, context_id: str, reader, query: str = "", - relevance: str | None = None, - relevance_method: str | None = None, - ids: list[str] | None = None, kind: str | None = None, - repo: str | None = None, tier: str | None = None, - backend: str | None = None, scenario: str | None = None, - limit: int = 5, context_bytes: int = 65536, - sources: tuple[str, ...] = ("package",)) -> tuple[Path, str]: - """Run ``collect`` with a context budget and keep its complete envelope. - - The capture is produced by the same ``collect`` operation the agent uses, - not by a caller narrative. Returns the new campaign-relative file and its - SHA-256. - """ - campaign = knowledge_capture.capture_campaign(root, context_id) - if relevance_method is None: - relevance_method = TASK_RELEVANCE if relevance and ids is None else "weighted-lexical-v1" - if relevance_method == TASK_RELEVANCE and ids is None and backend is None: - backend = campaign.get("backend") - result = reader.collect( - query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, - scenario=scenario, limit=limit, relevance=relevance, - relevance_method=relevance_method, context_bytes=context_bytes, - ) - request = { - "query": query, "relevance": relevance, - "relevance_method": relevance_method if relevance else None, - "ids": ids, "kind": kind, "repo": repo, "tier": tier, - "backend": backend, "scenario": scenario, "limit": limit, - "context_bytes": context_bytes, "sources": list(sources), - } - if _supplies_no_observation(result): - # Such a reader gets the record this entry wrote before observations - # existed: the same fields, order and bytes, with no empty provenance. - contexts = _contexts(result) - return knowledge_capture.store_capture( - root, campaign_id=campaign["campaign_id"], context_id=context_id, - # Only the source categories are kept; store and checkout paths can be private. - request={**request, "ids": ids or []}, - selection=result.get("selection"), - items=[context_item(view) for view in result.get("items", [])], - omissions=result.get("omissions", []), - context_omissions=result.get("context_omissions", []), - context_budget=result.get("context_budget"), - context_bytes_used=result.get("context_bytes"), - contexts=contexts, - assembled_text=assemble_context_text(contexts), - ) - return save_retrieval(root, context_id=context_id, result=result, request=request, - wrapper="campaign capture-context") - - -def _supplies_no_observation(result: dict) -> bool: - """Whether a result has neither a retrieval observation nor any item provenance.""" - return not isinstance(result.get("retrieval"), dict) and not any( - key in view for view in result.get("items", []) for key in PROVENANCE_FIELDS) - - -def _contexts(result: dict) -> list[dict]: - return [{ - "ref": context["ref"], "revision": context["revision"], - "content_bytes": context["content_bytes"], - "resources": [{key: resource[key] for key in ("path", "role", "sha256", "content")} - for resource in context["resources"]], - "omissions": context["omissions"], - "sources": context.get("sources", []), - } for context in result.get("contexts", [])] - - -def save_retrieval(root: Path, *, context_id: str, result: dict, - request: dict | None = None, wrapper: str | None = None) -> tuple[Path, str]: - """Explicitly retain a search/collect result without executing another search. - - The caller owns this local capture. Review/redact the bundle before sharing; - hashes attest bytes, not publication permission or actual model use. - """ - campaign = knowledge_capture.capture_campaign(root, context_id) - retrieval = result.get("retrieval") - if isinstance(retrieval, dict): - retrieval = {**retrieval} - if wrapper: - retrieval.update(entrypoint=wrapper, calls=[wrapper, *retrieval["calls"]]) - request = request if request is not None else (retrieval or {}).get("request", {}) - observations = {item["ref"]: item for item in (retrieval or {}).get("items", [])} - items = [] - for view in result.get("items", []): - ref = view.get("ref") or "package:" + view["id"] - items.append({**context_item({**view, "ref": ref}, provenance=True), - "revision": view.get("revision") - or observations.get(ref, {}).get("revision")}) - contexts = _contexts(result) - return knowledge_capture.store_capture( - root, campaign_id=campaign["campaign_id"], context_id=context_id, - # Only the source categories are kept; store and checkout paths can be private. - request=request, - selection=result.get("selection"), - items=items, - omissions=result.get("omissions", []), - context_omissions=result.get("context_omissions", []), - context_budget=result.get("context_budget"), - context_bytes_used=result.get("context_bytes"), - contexts=contexts, - assembled_text=assemble_context_text(contexts), - retrieval=retrieval, - # A result without an observation has no recorded entry point. It is - # labelled unknown; a plausible current default is not evidence. - captured_by=f"inferrsi {(retrieval or {}).get('entrypoint', 'unknown')} {__version__}", - ) - - -def verify_capture(raw: bytes) -> tuple[dict | None, str]: - """Return the capture and ``verified``, or ``None`` and a fixed reason code. - - Harness checks the stored bytes. This adds that the assembled text is the - Knowledge projection of the listed resources, so a reader sees exactly the - text those resources produce. - """ - document, state = knowledge_capture.verify_capture(raw) - if document is None: - return None, state - if assemble_context_text(document["contexts"]) != document["assembled_text"]: - return None, "assembly_mismatch" - return document, "verified" +__all__ = ['create_campaign', 'capture_context', 'save_retrieval', 'verify_capture'] diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a3..a5889c8d3 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -11,11 +11,11 @@ from ...knowledge.catalog import get_knowledge, search_knowledge from ...knowledge.knowledge_operations import KnowledgeReader from ...knowledge.research import research_knowledge -from ..campaign import capture_context, create_campaign from ..inspection import inspect_environment from ..knowledge import knowledge_reader from ..knowledge import read as read_knowledge from ..legacy.reporting import write_report +from ..workflows.campaign import capture_context, create_campaign from .arguments import device_ids, make_spec, parser @@ -368,7 +368,7 @@ def main(argv: list[str] | None = None) -> int: _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 if args.benchmark_action == "run": - from ..benchmark_operations import run_task + from ..workflows.benchmark_operations import run_task return run_task(args.task, args.arguments) from ...benchmark.registry import builtin_tasks, describe_task diff --git a/src/inferrsi/apps/improvement.py b/src/inferrsi/apps/improvement.py index 187329e74..d3a4d6ef3 100644 --- a/src/inferrsi/apps/improvement.py +++ b/src/inferrsi/apps/improvement.py @@ -1,29 +1,5 @@ -# Composed improvement-step entries: Harness usage totals and search progression -# joined with Benchmark acceptance rules. Signatures and results match the -# functions that harness.improvement held before the ownership split. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.improvement`.""" -from inferrsi.benchmark import improvement as rules -from inferrsi.benchmark.improvement import SelectionPolicy -from inferrsi.harness import improvement as progression -from inferrsi.harness.usage import usage_totals +from .workflows.improvement import assessment, choose, holdout_accepts, rerun_assessment - -def assessment(report: dict, policy: SelectionPolicy) -> dict: - pairs = report["pairs"] - usage = {role: usage_totals([p[role] for p in pairs]) for role in ("baseline", "candidate")} - return rules.assessment(report, policy, usage) - - -def choose(probes: dict[str, dict], policy: SelectionPolicy, *, composable: bool) -> dict: - scores = {name: assessment(report, policy) for name, report in probes.items()} - return progression.choose( - scores, blocked=rules.search_block(scores, probes), composable=composable) - - -def holdout_accepts(report: dict, policy: SelectionPolicy) -> bool: - return rules.holdout_accepts(assessment(report, policy)) - - -def rerun_assessment(report: dict, probes: dict[str, dict], policy: SelectionPolicy) -> dict: - scores = {name: assessment(probe, policy) for name, probe in probes.items()} - return rules.rerun_assessment(assessment(report, policy), report, probes, scores) +__all__ = ['assessment', 'choose', 'holdout_accepts', 'rerun_assessment'] diff --git a/src/inferrsi/apps/skill_eval_engineering.py b/src/inferrsi/apps/skill_eval_engineering.py index b907e9a53..bbe7de38c 100644 --- a/src/inferrsi/apps/skill_eval_engineering.py +++ b/src/inferrsi/apps/skill_eval_engineering.py @@ -1,28 +1,8 @@ -# Composed serving-parser artifact check: Benchmark task rules and vector judgment -# around the Harness sandbox replay. No expected result enters the sandbox. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.skill_eval_engineering`.""" -import json -import os -import sys -from pathlib import Path +from inferrsi.apps.workflows.skill_eval_engineering import main, verify -from inferrsi.benchmark import skill_eval_engineering as task_rules -from inferrsi.harness import artifact_sandbox - - -def verify(task, output): - source, rejected = task_rules.submission(task, output) - if rejected is not None: - return rejected - return task_rules.judge(artifact_sandbox.replay(source, task_rules.vector_inputs())) - - -def main(): - task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) - output = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) - result = verify(task, output) - Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") - raise SystemExit(0 if result["passed"] else 1) +__all__ = ['verify', 'main'] if __name__ == "__main__": diff --git a/src/inferrsi/apps/skill_evaluation.py b/src/inferrsi/apps/skill_evaluation.py index 064825104..0fd84045b 100644 --- a/src/inferrsi/apps/skill_evaluation.py +++ b/src/inferrsi/apps/skill_evaluation.py @@ -1,61 +1,5 @@ -# Composed skill-evaluation entries: Knowledge bundle export, and Harness trial -# execution joined with Benchmark verdict rules. Signatures and retained records -# match the functions that harness.skill_evaluation held before the ownership split. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.skill_evaluation`.""" -from pathlib import Path +from .workflows.skill_evaluation import export_bundle, run_trial, summarize -from inferrsi.benchmark import skill_eval_engineering as engineering_task -from inferrsi.benchmark import skill_evaluation as rules -from inferrsi.harness import skill_evaluation as host -from inferrsi.harness.usage import normalize_usage -from inferrsi.knowledge.knowledge_operations import KnowledgeReader - -from . import skill_eval_engineering - - -def export_bundle(reader: KnowledgeReader, reference: str, destination: Path) -> dict: - # Knowledge resolves the bundle; Harness receives and retains bytes only. - bundle = reader.read(reference) - host.write_files(destination, bundle.content) - return bundle.view() - - -def summarize(trials: list[dict]) -> dict: - normalized = [ - {**trial, "observations": {**trial["observations"], - "usage": normalize_usage(trial["observations"]["usage"])}} - for trial in trials - ] - return rules.summarize(normalized) - - -def run_trial( - client, - task: dict, - condition: str, - repeat: int, - workspace: Path, - trial_dir: Path, - schema: Path, - verifier: Path, - prompt: str, - deadline: float, - *, - timeout_seconds: float | None = None, - engineering_artifact: bool = False, - generation_only: bool = False, -) -> dict: - artifact = None - if engineering_artifact: - artifact = host.ArtifactPort( - kind=engineering_task.ADAPTER, - generation_mode=engineering_task.GENERATION_MODE, - starter=engineering_task.STARTER.encode("utf-8"), - max_source_bytes=engineering_task.MAX_SOURCE_BYTES, - verify=skill_eval_engineering.verify, - ) - return host.run_trial( - client, task, condition, repeat, workspace, trial_dir, schema, verifier, prompt, - deadline, valid_checks=rules.valid_checks, task_verdict=rules.task_verdict, - artifact=artifact, timeout_seconds=timeout_seconds, generation_only=generation_only, - ) +__all__ = ['export_bundle', 'summarize', 'run_trial'] diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 5bae55f46..0a1d0121c 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -3,7 +3,7 @@ import hashlib import json -from inferrsi.apps.campaign import verify_capture +from inferrsi.apps.workflows.campaign import verify_capture from inferrsi.harness.records.campaign_records import _identifier from inferrsi.harness.records.evidence import ( inside as _inside, diff --git a/src/inferrsi/apps/views/optimization_story.py b/src/inferrsi/apps/views/optimization_story.py index f3bb95480..b87616209 100644 --- a/src/inferrsi/apps/views/optimization_story.py +++ b/src/inferrsi/apps/views/optimization_story.py @@ -14,7 +14,7 @@ from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from inferrsi.harness.records.optimization_log import read_latest_log -from ..campaign import verify_capture +from ..workflows.campaign import verify_capture _IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") _GOOD = {"improved"} diff --git a/src/inferrsi/apps/workflows/__init__.py b/src/inferrsi/apps/workflows/__init__.py new file mode 100644 index 000000000..79aac8519 --- /dev/null +++ b/src/inferrsi/apps/workflows/__init__.py @@ -0,0 +1 @@ +"""Application workflows joining Knowledge, Harness and Benchmark ports.""" diff --git a/src/inferrsi/apps/workflows/benchmark_operations.py b/src/inferrsi/apps/workflows/benchmark_operations.py new file mode 100644 index 000000000..7770c3f53 --- /dev/null +++ b/src/inferrsi/apps/workflows/benchmark_operations.py @@ -0,0 +1,19 @@ +"""Resolve packaged tasks at the application boundary and launch their entries.""" + +import subprocess +import sys +from pathlib import Path + +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import optimization_session + + +def run_task(name: str, arguments: list[str]) -> int: + entry = task_resources(name).joinpath("benchmark.py") + if not entry.is_file(): + raise ValueError(f"Missing benchmark.py entry for task: {name}") + return subprocess.call( + [sys.executable, str(Path(optimization_session.__file__).resolve()), + str(entry), *arguments], + env=optimization_session.runtime_environment(), + ) diff --git a/src/inferrsi/apps/workflows/campaign.py b/src/inferrsi/apps/workflows/campaign.py new file mode 100644 index 000000000..b30c2aefb --- /dev/null +++ b/src/inferrsi/apps/workflows/campaign.py @@ -0,0 +1,156 @@ +"""Connect Knowledge values to the campaign records that Harness keeps. + +Harness records never import Knowledge. This module supplies the backend +vocabulary, the selection default, the context projection and the capture +orchestration they need: it calls the reader, keeps the retrieval envelope +the reader returned and hands Harness only the bytes to store. +""" + +from __future__ import annotations + +from pathlib import Path + +from inferrsi import __version__ +from inferrsi.harness.records import campaign_records, knowledge_capture +from inferrsi.knowledge._skill_selection import TASK_RELEVANCE +from inferrsi.knowledge.catalog import BACKENDS +from inferrsi.knowledge.context_projection import ( + PROVENANCE_FIELDS, + assemble_context_text, + context_item, +) + + +def create_campaign(root: Path, campaign_id: str, *, target: float | None = None, + noise: float | None = None, synthetic: bool = False, + backend: str | None = None, profile_first: bool = False) -> Path: + """Create a campaign manifest that accepts the Knowledge backend names.""" + return campaign_records.create_campaign( + root, campaign_id, target=target, noise=noise, synthetic=synthetic, backend=backend, + profile_first=profile_first, backends=BACKENDS, + ) + + +def capture_context(root: Path, *, context_id: str, reader, query: str = "", + relevance: str | None = None, + relevance_method: str | None = None, + ids: list[str] | None = None, kind: str | None = None, + repo: str | None = None, tier: str | None = None, + backend: str | None = None, scenario: str | None = None, + limit: int = 5, context_bytes: int = 65536, + sources: tuple[str, ...] = ("package",)) -> tuple[Path, str]: + """Run ``collect`` with a context budget and keep its complete envelope. + + The capture is produced by the same ``collect`` operation the agent uses, + not by a caller narrative. Returns the new campaign-relative file and its + SHA-256. + """ + campaign = knowledge_capture.capture_campaign(root, context_id) + if relevance_method is None: + relevance_method = TASK_RELEVANCE if relevance and ids is None else "weighted-lexical-v1" + if relevance_method == TASK_RELEVANCE and ids is None and backend is None: + backend = campaign.get("backend") + result = reader.collect( + query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, + scenario=scenario, limit=limit, relevance=relevance, + relevance_method=relevance_method, context_bytes=context_bytes, + ) + request = { + "query": query, "relevance": relevance, + "relevance_method": relevance_method if relevance else None, + "ids": ids, "kind": kind, "repo": repo, "tier": tier, + "backend": backend, "scenario": scenario, "limit": limit, + "context_bytes": context_bytes, "sources": list(sources), + } + if _supplies_no_observation(result): + # Such a reader gets the record this entry wrote before observations + # existed: the same fields, order and bytes, with no empty provenance. + contexts = _contexts(result) + return knowledge_capture.store_capture( + root, campaign_id=campaign["campaign_id"], context_id=context_id, + # Only the source categories are kept; store and checkout paths can be private. + request={**request, "ids": ids or []}, + selection=result.get("selection"), + items=[context_item(view) for view in result.get("items", [])], + omissions=result.get("omissions", []), + context_omissions=result.get("context_omissions", []), + context_budget=result.get("context_budget"), + context_bytes_used=result.get("context_bytes"), + contexts=contexts, + assembled_text=assemble_context_text(contexts), + ) + return save_retrieval(root, context_id=context_id, result=result, request=request, + wrapper="campaign capture-context") + + +def _supplies_no_observation(result: dict) -> bool: + """Whether a result has neither a retrieval observation nor any item provenance.""" + return not isinstance(result.get("retrieval"), dict) and not any( + key in view for view in result.get("items", []) for key in PROVENANCE_FIELDS) + + +def _contexts(result: dict) -> list[dict]: + return [{ + "ref": context["ref"], "revision": context["revision"], + "content_bytes": context["content_bytes"], + "resources": [{key: resource[key] for key in ("path", "role", "sha256", "content")} + for resource in context["resources"]], + "omissions": context["omissions"], + "sources": context.get("sources", []), + } for context in result.get("contexts", [])] + + +def save_retrieval(root: Path, *, context_id: str, result: dict, + request: dict | None = None, wrapper: str | None = None) -> tuple[Path, str]: + """Explicitly retain a search/collect result without executing another search. + + The caller owns this local capture. Review/redact the bundle before sharing; + hashes attest bytes, not publication permission or actual model use. + """ + campaign = knowledge_capture.capture_campaign(root, context_id) + retrieval = result.get("retrieval") + if isinstance(retrieval, dict): + retrieval = {**retrieval} + if wrapper: + retrieval.update(entrypoint=wrapper, calls=[wrapper, *retrieval["calls"]]) + request = request if request is not None else (retrieval or {}).get("request", {}) + observations = {item["ref"]: item for item in (retrieval or {}).get("items", [])} + items = [] + for view in result.get("items", []): + ref = view.get("ref") or "package:" + view["id"] + items.append({**context_item({**view, "ref": ref}, provenance=True), + "revision": view.get("revision") + or observations.get(ref, {}).get("revision")}) + contexts = _contexts(result) + return knowledge_capture.store_capture( + root, campaign_id=campaign["campaign_id"], context_id=context_id, + # Only the source categories are kept; store and checkout paths can be private. + request=request, + selection=result.get("selection"), + items=items, + omissions=result.get("omissions", []), + context_omissions=result.get("context_omissions", []), + context_budget=result.get("context_budget"), + context_bytes_used=result.get("context_bytes"), + contexts=contexts, + assembled_text=assemble_context_text(contexts), + retrieval=retrieval, + # A result without an observation has no recorded entry point. It is + # labelled unknown; a plausible current default is not evidence. + captured_by=f"inferrsi {(retrieval or {}).get('entrypoint', 'unknown')} {__version__}", + ) + + +def verify_capture(raw: bytes) -> tuple[dict | None, str]: + """Return the capture and ``verified``, or ``None`` and a fixed reason code. + + Harness checks the stored bytes. This adds that the assembled text is the + Knowledge projection of the listed resources, so a reader sees exactly the + text those resources produce. + """ + document, state = knowledge_capture.verify_capture(raw) + if document is None: + return None, state + if assemble_context_text(document["contexts"]) != document["assembled_text"]: + return None, "assembly_mismatch" + return document, "verified" diff --git a/src/inferrsi/apps/workflows/improvement.py b/src/inferrsi/apps/workflows/improvement.py new file mode 100644 index 000000000..187329e74 --- /dev/null +++ b/src/inferrsi/apps/workflows/improvement.py @@ -0,0 +1,29 @@ +# Composed improvement-step entries: Harness usage totals and search progression +# joined with Benchmark acceptance rules. Signatures and results match the +# functions that harness.improvement held before the ownership split. + +from inferrsi.benchmark import improvement as rules +from inferrsi.benchmark.improvement import SelectionPolicy +from inferrsi.harness import improvement as progression +from inferrsi.harness.usage import usage_totals + + +def assessment(report: dict, policy: SelectionPolicy) -> dict: + pairs = report["pairs"] + usage = {role: usage_totals([p[role] for p in pairs]) for role in ("baseline", "candidate")} + return rules.assessment(report, policy, usage) + + +def choose(probes: dict[str, dict], policy: SelectionPolicy, *, composable: bool) -> dict: + scores = {name: assessment(report, policy) for name, report in probes.items()} + return progression.choose( + scores, blocked=rules.search_block(scores, probes), composable=composable) + + +def holdout_accepts(report: dict, policy: SelectionPolicy) -> bool: + return rules.holdout_accepts(assessment(report, policy)) + + +def rerun_assessment(report: dict, probes: dict[str, dict], policy: SelectionPolicy) -> dict: + scores = {name: assessment(probe, policy) for name, probe in probes.items()} + return rules.rerun_assessment(assessment(report, policy), report, probes, scores) diff --git a/src/inferrsi/apps/workflows/skill_eval_engineering.py b/src/inferrsi/apps/workflows/skill_eval_engineering.py new file mode 100644 index 000000000..b907e9a53 --- /dev/null +++ b/src/inferrsi/apps/workflows/skill_eval_engineering.py @@ -0,0 +1,29 @@ +# Composed serving-parser artifact check: Benchmark task rules and vector judgment +# around the Harness sandbox replay. No expected result enters the sandbox. + +import json +import os +import sys +from pathlib import Path + +from inferrsi.benchmark import skill_eval_engineering as task_rules +from inferrsi.harness import artifact_sandbox + + +def verify(task, output): + source, rejected = task_rules.submission(task, output) + if rejected is not None: + return rejected + return task_rules.judge(artifact_sandbox.replay(source, task_rules.vector_inputs())) + + +def main(): + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + output = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, output) + Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") + raise SystemExit(0 if result["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/src/inferrsi/apps/workflows/skill_evaluation.py b/src/inferrsi/apps/workflows/skill_evaluation.py new file mode 100644 index 000000000..064825104 --- /dev/null +++ b/src/inferrsi/apps/workflows/skill_evaluation.py @@ -0,0 +1,61 @@ +# Composed skill-evaluation entries: Knowledge bundle export, and Harness trial +# execution joined with Benchmark verdict rules. Signatures and retained records +# match the functions that harness.skill_evaluation held before the ownership split. + +from pathlib import Path + +from inferrsi.benchmark import skill_eval_engineering as engineering_task +from inferrsi.benchmark import skill_evaluation as rules +from inferrsi.harness import skill_evaluation as host +from inferrsi.harness.usage import normalize_usage +from inferrsi.knowledge.knowledge_operations import KnowledgeReader + +from . import skill_eval_engineering + + +def export_bundle(reader: KnowledgeReader, reference: str, destination: Path) -> dict: + # Knowledge resolves the bundle; Harness receives and retains bytes only. + bundle = reader.read(reference) + host.write_files(destination, bundle.content) + return bundle.view() + + +def summarize(trials: list[dict]) -> dict: + normalized = [ + {**trial, "observations": {**trial["observations"], + "usage": normalize_usage(trial["observations"]["usage"])}} + for trial in trials + ] + return rules.summarize(normalized) + + +def run_trial( + client, + task: dict, + condition: str, + repeat: int, + workspace: Path, + trial_dir: Path, + schema: Path, + verifier: Path, + prompt: str, + deadline: float, + *, + timeout_seconds: float | None = None, + engineering_artifact: bool = False, + generation_only: bool = False, +) -> dict: + artifact = None + if engineering_artifact: + artifact = host.ArtifactPort( + kind=engineering_task.ADAPTER, + generation_mode=engineering_task.GENERATION_MODE, + starter=engineering_task.STARTER.encode("utf-8"), + max_source_bytes=engineering_task.MAX_SOURCE_BYTES, + verify=skill_eval_engineering.verify, + ) + return host.run_trial( + client, task, condition, repeat, workspace, trial_dir, schema, verifier, prompt, + deadline, valid_checks=rules.valid_checks, task_verdict=rules.task_verdict, + artifact=artifact, timeout_seconds=timeout_seconds, generation_only=generation_only, + ) diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index 9030624cd..ab216a110 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -121,7 +121,7 @@ def _runtime_identity(): "harness/usage.py", "harness/agent_sessions.py", "harness/task_protocol.py", "harness/protocol_child.py", "harness/candidates.py", "harness/project.py", - "harness/records/knowledge_capture.py", "apps/campaign.py", + "harness/records/knowledge_capture.py", "apps/workflows/campaign.py", "knowledge/context_projection.py") return {name: sha256(root / name) for name in names} diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index fe959a664..71ccfcc15 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -18,7 +18,8 @@ NEW_CORES = ("knowledge", "benchmark", "apps") CORE_NAMES = ("knowledge", "harness", "benchmark", "apps") # No root file is application assembly. A1 (#408) moved the CLI and its parser into -# ``apps/cli/``, and A2 (#408) moved the task launcher to ``apps/benchmark_operations.py``. +# ``apps/cli/``, and A2 (#408) moved the task launcher into Apps. The #490 workflows +# slice groups it at ``apps/workflows/benchmark_operations.py``. # With that last move the ``APPLICATION_ENTRIES`` listing is gone: Apps is the # ``inferrsi.apps`` package by its first path segment and nothing else, so a root # ``cli.py``, ``arguments.py`` or ``benchmark_operations.py`` that returns is an unlisted @@ -76,7 +77,7 @@ RETIRED_BY_A1 = frozenset({"inferrsi.cli", "inferrsi.arguments"}) # #408 A2 deleted the root chart renderer and task launcher the same way. The # implementations are ``inferrsi.apps.views.summary_chart`` and -# ``inferrsi.apps.benchmark_operations``. +# ``inferrsi.apps.workflows.benchmark_operations``. RETIRED_BY_A2 = frozenset({"inferrsi.summary_chart", "inferrsi.benchmark_operations"}) # #452 M1 deleted the four root MindIE files the same way. The adapter is the # ``inferrsi.apps.mindie`` package: ``project``, ``context``, ``mcp`` and ``read_worker``. diff --git a/tests/check_installed.py b/tests/check_installed.py index 7f4977d53..0054365a9 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -17,6 +17,7 @@ from inferrsi.apps.api import CandidateProvider, EvaluationPort from inferrsi.apps.cli import main from inferrsi.benchmark.registry import builtin_tasks, describe_task, task_resources +from inferrsi.benchmark.skill_eval_engineering import REQUEST from inferrsi.contracts import RunSpec, claim, evidence, json_v1 from inferrsi.knowledge._source_snapshot import BUNDLE_LIBRARY, LIBRARY from inferrsi.knowledge.catalog import knowledge_records, search_knowledge @@ -621,6 +622,9 @@ def _inventory(root, prefix): ("result", "storage"), ("evidence_layout", "storage"), ("bridge", "handoff"), ("context_provider", "handoff"), ("candidate_provider", "handoff"), + ("campaign", "workflows"), ("benchmark_operations", "workflows"), + ("improvement", "workflows"), ("skill_evaluation", "workflows"), + ("skill_eval_engineering", "workflows"), ): old_name = f"inferrsi.apps.{name}" canonical_name = f"inferrsi.apps.{group}.{name}" @@ -637,6 +641,32 @@ def _inventory(root, prefix): ) assert CandidateProvider is import_module("inferrsi.apps.candidate_provider").CandidateProvider assert EvaluationPort.__module__ == "inferrsi.apps.api" +# Both the former engineering verifier script and its canonical entry survive installation. +with tempfile.TemporaryDirectory() as script_directory: + script_root = Path(script_directory) + script_task, script_output, script_result = ( + script_root / name for name in ("task.json", "output.json", "result.json") + ) + script_task.write_text(json.dumps({ + "id": "synthetic-parser", "split": "development", "request": REQUEST, + "provenance": "Synthetic installed-entry control", + }), encoding="utf-8") + script_output.write_text("{}", encoding="utf-8") + for script_module in ("inferrsi.apps.skill_eval_engineering", + "inferrsi.apps.workflows.skill_eval_engineering"): + for script_entry in (["-I", "-m", script_module], + ["-I", import_module(script_module).__file__]): + script_result.unlink(missing_ok=True) + script_run = subprocess.run( + [sys.executable, *script_entry, str(script_task), str(script_output)], + env={**os.environ, "INFERRSI_RESULT_PATH": str(script_result)}, + cwd=script_root, capture_output=True, text=True, timeout=30, + ) + assert script_run.returncode == 1, script_run.stderr + assert script_run.stdout == script_run.stderr == "" + assert script_result.read_bytes() == ( + b'{"passed": false, "assertions": {"artifact_schema": false}}' + ) for retired in ("_repo_pack_data", "evidence_paths"): assert find_spec(f"inferrsi.{retired}") is None, retired assert not installed_root & {"_repo_pack_data.py", "evidence_paths.py"} @@ -686,7 +716,8 @@ def _inventory(root, prefix): else: raise AssertionError(f"retired module inferrsi.{a1_retired} must not import") assert main.__module__ == "inferrsi.apps.cli" -for canonical in ("inferrsi.apps.views.summary_chart", "inferrsi.apps.benchmark_operations", +for canonical in ("inferrsi.apps.views.summary_chart", + "inferrsi.apps.workflows.benchmark_operations", "inferrsi.apps.mindie.project", "inferrsi.apps.mindie.context", "inferrsi.apps.mindie.mcp", "inferrsi.apps.mindie.read_worker"): assert find_spec(canonical) is not None, canonical diff --git a/tests/fixtures/dashboard_profile_review.py b/tests/fixtures/dashboard_profile_review.py index 519c07690..61ca7b307 100644 --- a/tests/fixtures/dashboard_profile_review.py +++ b/tests/fixtures/dashboard_profile_review.py @@ -24,7 +24,7 @@ def build(root: Path): start = datetime(2026, 1, 1, 12, tzinfo=timezone.utc) with ( patch("inferrsi.__version__", "0+synthetic"), - patch("inferrsi.apps.campaign.__version__", "0+synthetic"), + patch("inferrsi.apps.workflows.campaign.__version__", "0+synthetic"), patch("inferrsi.harness.records.knowledge_capture.datetime", wraps=datetime) as clock, patch("inferrsi.harness.records.optimization_log.datetime", wraps=datetime) as log_clock, ): diff --git a/tests/fixtures/knowledge_retrieval.py b/tests/fixtures/knowledge_retrieval.py index 454fc30cb..e53ef2e3c 100644 --- a/tests/fixtures/knowledge_retrieval.py +++ b/tests/fixtures/knowledge_retrieval.py @@ -10,9 +10,9 @@ from pathlib import Path from unittest.mock import patch -from inferrsi.apps.campaign import capture_context, create_campaign, save_retrieval from inferrsi.apps.views.dashboard import write_dashboard from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import capture_context, create_campaign, save_retrieval from inferrsi.harness.records.campaign_records import record_candidate from inferrsi.harness.records.optimization_log import record_log from inferrsi.knowledge._source_snapshot import source_entry diff --git a/tests/test_anthropic_agent.py b/tests/test_anthropic_agent.py index f00c5f97f..be63f3569 100644 --- a/tests/test_anthropic_agent.py +++ b/tests/test_anthropic_agent.py @@ -6,7 +6,7 @@ import pytest -from inferrsi.apps import skill_evaluation as evaluation +from inferrsi.apps.workflows import skill_evaluation as evaluation from inferrsi.harness import skill_evaluation as harness from inferrsi.infra.artifacts import sha256, write_json from inferrsi.model import anthropic_agent as agent diff --git a/tests/test_apps_module_migrations.py b/tests/test_apps_module_migrations.py index 606297e03..4582c3ca5 100644 --- a/tests/test_apps_module_migrations.py +++ b/tests/test_apps_module_migrations.py @@ -22,10 +22,21 @@ def test_shim_is_an_explicit_export_of_the_same_objects(entry): path = REPO / "src" / (entry["old"].replace(".", "/") + ".py") tree = ast.parse(path.read_text(encoding="utf-8")) - docstring, imported, declared = tree.body + if entry.get("script_entry"): + docstring, imported, declared, guard = tree.body + assert ast.dump(guard) == ast.dump(ast.parse( + f'if __name__ == "__main__":\n {entry["script_entry"]}()\n' + ).body[0]) + else: + docstring, imported, declared = tree.body assert isinstance(docstring, ast.Expr) and isinstance(docstring.value, ast.Constant) - assert isinstance(imported, ast.ImportFrom) and imported.level == 1 - assert "inferrsi.apps." + imported.module == entry["canonical"] + assert isinstance(imported, ast.ImportFrom) + if entry.get("script_entry"): + # This former file entry must also work without a package context. + assert imported.level == 0 and imported.module == entry["canonical"] + else: + assert imported.level == 1 + assert "inferrsi.apps." + imported.module == entry["canonical"] assert sorted((name.name, name.asname) for name in imported.names) == sorted( (name, None) for name in entry["exports"] ) @@ -148,7 +159,7 @@ def test_storage_dependency_check_rejects_a_handoff_back_edge(tmp_path, indirect def test_migrated_initializers_import_nothing_and_module_bodies_do_no_io(): - packages = ["inferrsi.apps.storage", "inferrsi.apps.handoff"] + packages = ["inferrsi.apps.storage", "inferrsi.apps.handoff", "inferrsi.apps.workflows"] for package in packages: path = REPO / "src" / package.replace(".", "/") / "__init__.py" tree = ast.parse(path.read_text(encoding="utf-8")) @@ -212,13 +223,15 @@ def test_delivered_context_draft_loads_a_legacy_instance_pickle(): assert type(restored) is DeliveredContextDraft and restored == value -def test_cold_handoff_imports_do_not_read_data_probe_or_launch_services(): +@pytest.mark.parametrize("group", ["handoff", "workflows"]) +def test_cold_handoff_and_workflow_imports_do_not_read_data_probe_or_launch_services(group): script = """ import builtins, importlib, importlib.abc, io, json, socket, sqlite3, subprocess, sys, warnings from unittest.mock import patch -blocked = ('inferrsi.apps.api', 'inferrsi.infra', 'inferrsi.benchmark', 'inferrsi.model', - 'inferrsi.harness.api', 'inferrsi.harness.run_store', 'inferrsi.harness.generation', - 'torch', 'numpy', 'PIL', 'vllm') +blocked = ('inferrsi.apps.api', 'inferrsi.harness.api', 'inferrsi.harness.run_store', + 'inferrsi.harness.generation', 'torch', 'numpy', 'PIL', 'vllm') +if sys.argv[2] == 'handoff': + blocked += ('inferrsi.infra', 'inferrsi.benchmark', 'inferrsi.model') class Block(importlib.abc.MetaPathFinder): def find_spec(self, name, path=None, target=None): if any(name == item or name.startswith(item + '.') for item in blocked): @@ -228,7 +241,7 @@ def deny(*args, **kwargs): raise AssertionError('unexpected data read, device probe, process or service') with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') - importlib.import_module('inferrsi.apps.handoff') + importlib.import_module('inferrsi.apps.' + sys.argv[2]) parent_only = sorted(name for name in sys.modules if name.startswith('inferrsi.apps')) with patch.object(builtins, 'open', deny), patch.object(io, 'open', deny), \\ patch.object(subprocess, 'Popen', deny), patch.object(sqlite3, 'connect', deny), \\ @@ -242,8 +255,8 @@ def deny(*args, **kwargs): print(json.dumps({'parent_only': parent_only, 'warnings': [str(w.message) for w in caught], 'control': control})) """ - modules = [entry[key] for entry in MIGRATIONS if ".handoff." in entry["canonical"] + modules = [entry[key] for entry in MIGRATIONS if f".{group}." in entry["canonical"] for key in ("old", "canonical")] - result = _fresh(script, json.dumps(modules)) - assert result == {"parent_only": ["inferrsi.apps", "inferrsi.apps.handoff"], + result = _fresh(script, json.dumps(modules), group) + assert result == {"parent_only": ["inferrsi.apps", f"inferrsi.apps.{group}"], "warnings": [], "control": True} diff --git a/tests/test_benchmark_operations.py b/tests/test_benchmark_operations.py index 091dd462d..0f23e9172 100644 --- a/tests/test_benchmark_operations.py +++ b/tests/test_benchmark_operations.py @@ -4,8 +4,8 @@ import pytest -from inferrsi.apps import benchmark_operations from inferrsi.apps.cli import main +from inferrsi.apps.workflows import benchmark_operations from inferrsi.benchmark import registry diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index 1c94519ba..6664b07e3 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -10,7 +10,7 @@ import pytest from inferrsi import __version__ -from inferrsi.apps import campaign +from inferrsi.apps.workflows import campaign from inferrsi.harness.records import campaign_records, knowledge_capture from inferrsi.knowledge.context_projection import context_item from tests._gate_policy_support import ( diff --git a/tests/test_campaign_task_selection.py b/tests/test_campaign_task_selection.py index 5d1a2da1a..575d9d707 100644 --- a/tests/test_campaign_task_selection.py +++ b/tests/test_campaign_task_selection.py @@ -1,6 +1,6 @@ import json -from inferrsi.apps.campaign import capture_context, create_campaign, verify_capture +from inferrsi.apps.workflows.campaign import capture_context, create_campaign, verify_capture from inferrsi.knowledge.knowledge_operations import KnowledgeReader BASELINE_CUE = "reproducible diffusion baseline correctness fixed seed end to end benchmark" diff --git a/tests/test_harness_improvement.py b/tests/test_harness_improvement.py index d2890fff4..149d261c1 100644 --- a/tests/test_harness_improvement.py +++ b/tests/test_harness_improvement.py @@ -5,7 +5,7 @@ import pytest -from inferrsi.apps.improvement import assessment, choose, holdout_accepts +from inferrsi.apps.workflows.improvement import assessment, choose, holdout_accepts from inferrsi.benchmark.improvement import SelectionPolicy from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import usage_totals @@ -265,7 +265,7 @@ def factory(phase, candidate): def test_rerun_with_unstable_baseline_is_rejected(tmp_path): - from inferrsi.apps.improvement import rerun_assessment + from inferrsi.apps.workflows.improvement import rerun_assessment _, _, output, _, _ = run(tmp_path) probe = compare.read_report(output / "results/inv") diff --git a/tests/test_knowledge_navigation_browser.py b/tests/test_knowledge_navigation_browser.py index 0742394e3..1005e0bff 100644 --- a/tests/test_knowledge_navigation_browser.py +++ b/tests/test_knowledge_navigation_browser.py @@ -8,9 +8,9 @@ from test_optimization_story import campaign as make_campaign from test_retrieval_projection import FIXTURE -from inferrsi.apps.campaign import capture_context from inferrsi.apps.views.dashboard import dashboard_snapshot, render_dashboard from inferrsi.apps.views.dashboard_live import DashboardServer +from inferrsi.apps.workflows.campaign import capture_context playwright = pytest.importorskip("playwright.sync_api") diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index a37836ff0..c671ec2dd 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -1,6 +1,10 @@ """Run the current-Agent protocol using real child processes and synthetic work.""" import json +import os +import subprocess +import sys +from pathlib import Path import pytest @@ -410,7 +414,7 @@ def test_child_argument_error_keeps_its_cli_message_at_start(task, tmp_path, cap def test_task_runner_does_not_expose_harness_as_top_level_modules(task, tmp_path): - from inferrsi.apps.benchmark_operations import run_task + from inferrsi.apps.workflows.benchmark_operations import run_task output = tmp_path / "imports.json" (task / "benchmark.py").write_text( @@ -460,6 +464,122 @@ def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executo assert "harness/artifact_primitives.py" in identity # #412 R3: the protocol child entry runs every task protocol operation. assert "harness/protocol_child.py" in identity + assert "apps/workflows/campaign.py" in identity + assert "apps/campaign.py" not in identity assert "infra/executors/local.py" not in identity for name in identity: assert (root / name).is_file(), name + from inferrsi.harness.artifact_primitives import sha256 + + assert identity["apps/workflows/campaign.py"] == sha256(root / "apps/workflows/campaign.py") + + +@pytest.mark.parametrize("action", ["status", "recover"]) +def test_pre_move_runtime_identity_is_rejected_without_rewriting_plan(session, action): + path = session / "plan.json" + plan = json.loads(path.read_text(encoding="utf-8")) + plan["runtime_identity"]["apps/campaign.py"] = plan["runtime_identity"].pop( + "apps/workflows/campaign.py" + ) + # A prior release used the old key. Even equal bytes cannot admit that runtime map. + frozen = json.dumps(plan, indent=2).encode("utf-8") + path.write_bytes(frozen) + with pytest.raises(SystemExit): + main(["benchmarks", "session", action, "--session", str(session)]) + assert path.read_bytes() == frozen + assert not (session / "calls.log").exists() + + +@pytest.fixture +def pre_move_python(): + executable = os.environ.get("INFERRSI_PRE_MOVE_PYTHON") + if not executable: + pytest.skip("Set INFERRSI_PRE_MOVE_PYTHON to an isolated, pinned pre-move wheel runtime") + assert Path(executable).is_file() + return executable + + +def test_pinned_pre_move_wheel_recovers_its_original_plan( + tmp_path, task, pre_move_python, +): + """Two independent runtimes; no patched identity map or rewritten frozen plan.""" + driver = """ +import json, sys +from pathlib import Path +if sys.argv[1]: + sys.path.insert(0, sys.argv[1]) +from inferrsi.apps.cli import main +from inferrsi.benchmark import registry +from inferrsi.harness import optimization_session +registry._library_root = lambda: Path(sys.argv[2]) +arguments = sys.argv[3:] +if arguments[0] == 'interrupt': + arguments.pop(0) + execute = optimization_session._session_execute + def interrupt(*args): + execute(*args) + raise KeyboardInterrupt('synthetic interruption after the retained execution') + optimization_session._session_execute = interrupt + try: + main(arguments) + except KeyboardInterrupt: + print(json.dumps({'interrupted': True})) + else: + raise AssertionError('execution was not interrupted') +else: + raise SystemExit(main(arguments)) +""" + post_move_python = os.environ.get("INFERRSI_POST_MOVE_PYTHON", sys.executable) + current_source = "" if "INFERRSI_POST_MOVE_PYTHON" in os.environ else str( + Path(__file__).resolve().parents[1] / "src" + ) + + def run(python, source, arguments, expected=0): + result = subprocess.run( + [python, "-I", "-c", driver, source, str(task.parent), *arguments], + cwd=tmp_path, capture_output=True, text=True, timeout=60, + ) + assert result.returncode == expected, result.stderr + return result + + def start(root): + return ["benchmarks", "session", "start", task.name, "--session", str(root), + "--rounds", "3", "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "synthetic-release-boundary", "--evaluation-args", ""] + + old = tmp_path / "old-session" + run(pre_move_python, "", start(old)) + frozen = (old / "plan.json").read_bytes() + old_identity = json.loads(frozen)["runtime_identity"] + assert "apps/campaign.py" in old_identity + assert "apps/workflows/campaign.py" not in old_identity + candidate = old / "work/candidate.py" + candidate.write_text("FACTOR = 2\n", encoding="utf-8") + interrupted = run(pre_move_python, "", [ + "interrupt", "benchmarks", "session", "round", "--session", str(old), + "--candidate", str(candidate), "--hypothesis", "Synthetic release recovery control", + ]) + assert json.loads(interrupted.stdout)["interrupted"] + retained = {p.relative_to(old): p.read_bytes() for p in old.rglob("*") if p.is_file()} + for action in ("status", "recover"): + rejected = run(post_move_python, current_source, + ["benchmarks", "session", action, "--session", str(old)], expected=2) + assert "Optimization runtime changed" in rejected.stderr + after = {p.relative_to(old): p.read_bytes() for p in old.rglob("*") if p.is_file()} + assert after == retained + recovered = json.loads(run(pre_move_python, "", [ + "benchmarks", "session", "recover", "--session", str(old), + ]).stdout) + assert recovered["completed_rounds"] == 1 and recovered["best"]["score"] == 2 + run(pre_move_python, "", ["benchmarks", "session", "status", "--session", str(old)]) + assert (old / "plan.json").read_bytes() == frozen + assert (old / "calls.log").read_text().splitlines() == ["2"] + + new = tmp_path / "new-session" + run(post_move_python, current_source, start(new)) + new_identity = json.loads((new / "plan.json").read_bytes())["runtime_identity"] + assert "apps/workflows/campaign.py" in new_identity and "apps/campaign.py" not in new_identity + from inferrsi.harness.artifact_primitives import sha256 + + canonical = Path(__file__).resolve().parents[1] / "src/inferrsi/apps/workflows/campaign.py" + assert new_identity["apps/workflows/campaign.py"] == sha256(canonical) diff --git a/tests/test_optimization_story.py b/tests/test_optimization_story.py index c22048a86..725a1ce3a 100644 --- a/tests/test_optimization_story.py +++ b/tests/test_optimization_story.py @@ -9,10 +9,10 @@ import pytest -from inferrsi.apps.campaign import capture_context, verify_capture from inferrsi.apps.cli import main from inferrsi.apps.views.dashboard import dashboard_snapshot, render_dashboard, write_dashboard from inferrsi.apps.views.dashboard_data import load_dashboard +from inferrsi.apps.workflows.campaign import capture_context, verify_capture from inferrsi.harness.records.campaign_records import ( create_campaign, record_candidate, diff --git a/tests/test_package_root_inventory.py b/tests/test_package_root_inventory.py index 8f8002056..a2bc065df 100644 --- a/tests/test_package_root_inventory.py +++ b/tests/test_package_root_inventory.py @@ -120,6 +120,7 @@ def test_inventory_is_the_exact_final_two_file_root() -> None: if isinstance(node, (ast.Import, ast.ImportFrom))] # A2 moved its two files into Apps. assert (REPO / PACKAGE / "apps" / "benchmark_operations.py").is_file() + assert (REPO / PACKAGE / "apps" / "workflows" / "benchmark_operations.py").is_file() assert (REPO / PACKAGE / "apps" / "views" / "summary_chart.py").is_file() # A1 moved its two files; the implementation is the apps/cli package. for name in ("__init__.py", "arguments.py"): diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 9e08239c3..32ae27ce3 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -77,7 +77,7 @@ def test_project_lifecycle_never_imports_benchmark(project): from pathlib import Path class NoBenchmark(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path=None, target=None): - if fullname in ('inferrsi.benchmark', 'inferrsi.apps.benchmark_operations') or ( + if fullname in ('inferrsi.benchmark', 'inferrsi.apps.workflows.benchmark_operations') or ( fullname.startswith('inferrsi.benchmark.') ): raise AssertionError('Ordinary optimization imported ' + fullname) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index e0c29138f..a47a9ba1b 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -9,8 +9,8 @@ import pytest -from inferrsi.apps.campaign import capture_context, create_campaign from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import capture_context, create_campaign from inferrsi.knowledge.context_projection import assemble_context_text REVISION = "f6d200" + "a" * 34 diff --git a/tests/test_search_attribution.py b/tests/test_search_attribution.py index 127d87c4c..6c45c3714 100644 --- a/tests/test_search_attribution.py +++ b/tests/test_search_attribution.py @@ -4,9 +4,9 @@ import pytest -from inferrsi.apps.campaign import create_campaign from inferrsi.apps.cli import main from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import create_campaign def command(corpus, cmd, output, event_id, exit_code=0): diff --git a/tests/test_skill_eval_compare.py b/tests/test_skill_eval_compare.py index 4a64ea080..7f98a7d9f 100644 --- a/tests/test_skill_eval_compare.py +++ b/tests/test_skill_eval_compare.py @@ -1013,9 +1013,9 @@ def test_changed_runtime_controllers_reject_execution_and_report_without_calls(t assert "src/inferrsi/infra/artifacts.py" in module.CONTROLLERS assert "tools/skill_eval_artifact_guidance.py" in module.CONTROLLERS # Each owner of the split trial path is frozen: composed entry, rules and host. - for owner in ("apps", "benchmark", "harness"): + for owner in ("apps/workflows", "benchmark", "harness"): assert f"src/inferrsi/{owner}/skill_evaluation.py" in module.CONTROLLERS - for name in ("apps/skill_eval_engineering.py", "benchmark/skill_eval_engineering.py", + for name in ("apps/workflows/skill_eval_engineering.py", "benchmark/skill_eval_engineering.py", "benchmark/skill_eval_diagnostic.py", "harness/artifact_sandbox.py", "harness/atomic_json.py", "harness/input_binding.py"): assert "src/inferrsi/" + name in module.CONTROLLERS diff --git a/tests/test_skill_eval_engineering.py b/tests/test_skill_eval_engineering.py index c0b0434ff..c5cab4975 100644 --- a/tests/test_skill_eval_engineering.py +++ b/tests/test_skill_eval_engineering.py @@ -1,5 +1,6 @@ import copy import json +import os import signal import subprocess import sys @@ -8,7 +9,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox @@ -17,6 +18,26 @@ ANSWER = {"artifact": "parser.py", "summary": "Repair", "artifact_source": engineering.STARTER} +@pytest.mark.parametrize("module", [ + "inferrsi.apps.skill_eval_engineering", "inferrsi.apps.workflows.skill_eval_engineering", +]) +@pytest.mark.parametrize("by_file", [False, True]) +def test_old_and_canonical_script_entries_write_the_same_result(tmp_path, module, by_file): + task, output, result = (tmp_path / name for name in ("task.json", "output.json", "result.json")) + task.write_text(json.dumps(TASK), encoding="utf-8") + output.write_text("{}", encoding="utf-8") + source = Path(__file__).resolve().parents[1] / "src" + arguments = [str(source / (module.replace(".", "/") + ".py"))] if by_file else ["-m", module] + environment = {**os.environ, "PYTHONPATH": str(source), "INFERRSI_RESULT_PATH": str(result)} + executed = subprocess.run( + [sys.executable, *arguments, str(task), str(output)], cwd=tmp_path, + env=environment, capture_output=True, text=True, timeout=30, + ) + assert executed.returncode == 1, executed.stderr + assert executed.stdout == executed.stderr == "" + assert result.read_bytes() == b'{"passed": false, "assertions": {"artifact_schema": false}}' + + def test_starter_matches_frozen_fixture(): fixture = Path(__file__).parent / "fixtures/skill_eval_engineering/parser.py" assert fixture.read_text() == engineering.STARTER diff --git a/tests/test_skill_eval_generation.py b/tests/test_skill_eval_generation.py index 79e312ae2..872e840c0 100644 --- a/tests/test_skill_eval_generation.py +++ b/tests/test_skill_eval_generation.py @@ -6,8 +6,8 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check -from inferrsi.apps import skill_evaluation as evaluation +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_evaluation as evaluation from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.harness import skill_evaluation as harness diff --git a/tests/test_skill_eval_local.py b/tests/test_skill_eval_local.py index d0a68e56e..2afbf54b6 100644 --- a/tests/test_skill_eval_local.py +++ b/tests/test_skill_eval_local.py @@ -3,7 +3,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.infra.artifacts import sha256, write_json diff --git a/tests/test_skill_eval_local_cli.py b/tests/test_skill_eval_local_cli.py index 3f08e1d54..0eb98df3d 100644 --- a/tests/test_skill_eval_local_cli.py +++ b/tests/test_skill_eval_local_cli.py @@ -11,7 +11,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.infra.artifacts import sha256, write_json diff --git a/tests/test_skill_eval_pipeline.py b/tests/test_skill_eval_pipeline.py index 15aa483e0..3ebb4a188 100644 --- a/tests/test_skill_eval_pipeline.py +++ b/tests/test_skill_eval_pipeline.py @@ -8,7 +8,7 @@ import pytest -from inferrsi.apps.skill_evaluation import run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import run_trial, summarize from inferrsi.infra.artifacts import atomic_write_json, sha256 from tools import skill_eval_pilot diff --git a/tests/test_skill_evaluation.py b/tests/test_skill_evaluation.py index ae398a384..5944038a2 100644 --- a/tests/test_skill_evaluation.py +++ b/tests/test_skill_evaluation.py @@ -6,7 +6,7 @@ import pytest -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.benchmark.skill_evaluation import task_verdict from inferrsi.harness.skill_evaluation import CodexClient, toml_value, trace_metrics from inferrsi.knowledge.knowledge_operations import KnowledgeReader diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index a6b417da8..17ef790f4 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -146,6 +146,7 @@ def test_only_files_directly_in_a_non_package_bundle_are_bundle_programs(tmp_pat "harness/task_protocol.py", "harness/other_protocol.py", "apps/benchmark_operations.py", + "apps/workflows/benchmark_operations.py", "knowledge/data/skills/scripts/loader.py", ]) def test_by_path_loading_is_admitted_nowhere_else(tmp_path, relative): @@ -240,6 +241,7 @@ def test_no_new_cross_core_imports(self) -> None: "_mindie_read_worker.py"): self.assertFalse((ROOT / retired).exists(), retired) for moved in ("cli/__init__.py", "cli/arguments.py", "benchmark_operations.py", + "workflows/benchmark_operations.py", "views/summary_chart.py", "mindie/project.py", "mindie/context.py", "mindie/mcp.py", "mindie/read_worker.py"): self.assertTrue((ROOT / "apps" / moved).is_file(), moved) @@ -366,7 +368,7 @@ def test_real_tree_allows_apps_and_same_core_only(self) -> None: "arguments_extra.py": "from inferrsi.knowledge import catalog\n", # #408 A2: the task launcher is Apps by path too. The root file that held # it is no longer an application entry and fails like the others. - "apps/benchmark_operations.py": "import inferrsi.benchmark\n", + "apps/workflows/benchmark_operations.py": "import inferrsi.benchmark\n", "benchmark_operations.py": "import inferrsi.benchmark\n", "unowned.py": "import inferrsi.benchmark\n", } diff --git a/tools/skill_eval_cloud_replay.py b/tools/skill_eval_cloud_replay.py index 876879acc..230a75533 100644 --- a/tools/skill_eval_cloud_replay.py +++ b/tools/skill_eval_cloud_replay.py @@ -6,7 +6,7 @@ import urllib.request from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox from tools import skill_eval_artifact_guidance as artifact_guidance diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 5615d61cd..43b312f68 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -12,8 +12,8 @@ from dataclasses import replace from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.benchmark import skill_eval_diagnostic as diagnostic from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox @@ -39,8 +39,8 @@ GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "tools/skill_eval_artifact_guidance.py", - "src/inferrsi/apps/skill_evaluation.py", - "src/inferrsi/apps/skill_eval_engineering.py", + "src/inferrsi/apps/workflows/skill_evaluation.py", + "src/inferrsi/apps/workflows/skill_eval_engineering.py", "src/inferrsi/benchmark/skill_evaluation.py", "src/inferrsi/benchmark/skill_eval_diagnostic.py", "src/inferrsi/benchmark/skill_eval_engineering.py", diff --git a/tools/skill_eval_improve.py b/tools/skill_eval_improve.py index 46bc6d705..b1a6098e3 100644 --- a/tools/skill_eval_improve.py +++ b/tools/skill_eval_improve.py @@ -12,7 +12,12 @@ import time from pathlib import Path -from inferrsi.apps.improvement import assessment, choose, holdout_accepts, rerun_assessment +from inferrsi.apps.workflows.improvement import ( + assessment, + choose, + holdout_accepts, + rerun_assessment, +) from inferrsi.benchmark.improvement import SelectionPolicy from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import usage_totals @@ -21,7 +26,8 @@ from tools import skill_eval_compare as compare CONTROLLERS = (*compare.CONTROLLERS, "tools/skill_eval_improve.py", - "src/inferrsi/apps/improvement.py", "src/inferrsi/benchmark/improvement.py", + "src/inferrsi/apps/workflows/improvement.py", + "src/inferrsi/benchmark/improvement.py", "src/inferrsi/harness/improvement.py") diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py index 95f7253d7..46967643d 100644 --- a/tools/skill_eval_local.py +++ b/tools/skill_eval_local.py @@ -12,7 +12,7 @@ import stat from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox from inferrsi.harness.usage import normalize_usage diff --git a/tools/skill_eval_pilot.py b/tools/skill_eval_pilot.py index 7d17f8537..579b121c1 100644 --- a/tools/skill_eval_pilot.py +++ b/tools/skill_eval_pilot.py @@ -7,7 +7,7 @@ import time from pathlib import Path -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import normalize_usage from inferrsi.infra.artifacts import sha256, write_json @@ -205,7 +205,7 @@ def prepare( bundles = {name: reader.read("checkout:" + name).view() for name in SKILLS} schedule = build_schedule(tasks, final_only) source_files = [ - ROOT / "src/inferrsi/apps/skill_evaluation.py", + ROOT / "src/inferrsi/apps/workflows/skill_evaluation.py", ROOT / "src/inferrsi/benchmark/skill_evaluation.py", ROOT / "src/inferrsi/harness/skill_evaluation.py", Path(__file__), diff --git a/tools/skill_eval_qualify.py b/tools/skill_eval_qualify.py index 2ff267149..fc775de1b 100644 --- a/tools/skill_eval_qualify.py +++ b/tools/skill_eval_qualify.py @@ -12,7 +12,7 @@ import subprocess from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox From b250620b2f39297b2b463ffd6b21127b451edef6 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Thu, 19 Nov 2026 15:36:06 +0800 Subject: [PATCH 2/3] Group the paired benchmark workflow under Apps workflows Signed-off-by: zengchuang Agent-assisted-by: Codex (author) --- docs/apps-module-migrations.json | 26 +- docs/architecture.md | 1 + docs/module-layout.md | 12 +- src/inferrsi/apps/benchmark_compare.py | 607 +----------------- src/inferrsi/apps/cli/__init__.py | 2 +- .../apps/workflows/benchmark_compare.py | 580 +++++++++++++++++ tests/check_installed.py | 2 +- tests/test_benchmark_compare.py | 2 +- 8 files changed, 645 insertions(+), 587 deletions(-) create mode 100644 src/inferrsi/apps/workflows/benchmark_compare.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index d152c58d2..2c1229b2e 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -22,7 +22,7 @@ "docs/operations.md", "src/inferrsi/apps/api.py", "src/inferrsi/apps/archive.py", - "src/inferrsi/apps/benchmark_compare.py", + "src/inferrsi/apps/workflows/benchmark_compare.py", "src/inferrsi/apps/delivery/__init__.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", @@ -295,5 +295,29 @@ "tools/skill_eval_qualify.py" ], "script_entry": "main" + }, + { + "old": "inferrsi.apps.benchmark_compare", + "canonical": "inferrsi.apps.workflows.benchmark_compare", + "exports": [ + "DEFAULT_CONFIG", + "CONFIG_NAME", + "OBSERVED_FIELDS", + "resolve_config", + "prepare", + "status", + "dispatch", + "collect", + "report", + "compare_command" + ], + "consumers": [ + "docs/architecture.md", + "docs/module-layout.md", + "src/inferrsi/apps/cli/__init__.py", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_compare.py" + ] } ] diff --git a/docs/architecture.md b/docs/architecture.md index fb22f5805..c75da6f2f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -204,6 +204,7 @@ src/inferrsi/ workflows/ campaign.py Knowledge values connected to Harness campaign records benchmark_operations.py Resolve a task entry and launch it through Harness + benchmark_compare.py Paired benchmark plans, native handoff, collection and reports skill_evaluation.py Composed bundle export, trial run and summary skill_eval_engineering.py Composed artifact check: task rules around sandbox replay improvement.py Composed assessment, selection, rerun and holdout acceptance diff --git a/docs/module-layout.md b/docs/module-layout.md index f3d4b400a..25f673190 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -63,20 +63,26 @@ digest; these ownership boundaries and DTOs remain unchanged. ### Apps workflows migration -`apps/workflows/` owns `campaign.py`, `benchmark_operations.py`, `improvement.py`, -`skill_evaluation.py` and `skill_eval_engineering.py`. It joins Knowledge reads, +`apps/workflows/` owns `campaign.py`, `benchmark_operations.py`, `benchmark_compare.py`, +`improvement.py`, `skill_evaluation.py` and `skill_eval_engineering.py`. It joins Knowledge reads, Harness execution and Benchmark judgments. CLI commands, views and evaluation tools import these canonical implementations. Its initializer imports nothing; imports of either path perform no data reads, probes, process launches or service registration. Signatures, return values and retained-record schemas are unchanged. -The five former Apps paths keep explicit exports from the same +The six former Apps paths keep explicit exports from the same [migration inventory](apps-module-migrations.json). The engineering verifier also keeps its former file and `python -m` entry points through a static absolute import and a `__main__` guard, recorded as `script_entry` in the inventory. Other shims contain only their docstring, explicit import and literal `__all__`. New pickles use the canonical module; patches to helpers or imported dependencies target it. +`benchmark_compare.py`, introduced by #501 while this migration was in progress, +composes paired benchmark plans, native host handoffs, collection and reports. +It returns host actions; the Agent performs the native calls. Its previous Apps +path retains the same public functions and configuration objects. The comparison +rules remain in Benchmark and the report view remains in `apps/views/`. + New optimization plans and evaluation controller snapshots fingerprint the canonical source files, not the compatibility shims. A frozen pre-move runtime map remains incompatible with a new runtime, including for `status` and `recover`. diff --git a/src/inferrsi/apps/benchmark_compare.py b/src/inferrsi/apps/benchmark_compare.py index 318e3d87d..c10791522 100644 --- a/src/inferrsi/apps/benchmark_compare.py +++ b/src/inferrsi/apps/benchmark_compare.py @@ -1,580 +1,27 @@ -"""Compose paired optimization experiments with native host handoffs. - -The Agent calls the host tools returned by this application. Python never -creates a conversation, invents host observations, or edits a candidate. -""" - -import json -import math -import os -import shlex -import shutil -import uuid -from copy import deepcopy -from datetime import datetime, timezone -from importlib.resources import files -from pathlib import Path -from tempfile import TemporaryDirectory - -from inferrsi.benchmark.comparison import DEFAULT_POLICY, DEFAULT_TASKS, summarize, validate_policy -from inferrsi.benchmark.registry import task_resources -from inferrsi.harness import agent_sessions, optimization -from inferrsi.harness.artifact_primitives import digest, file_lock, sha256 -from inferrsi.harness.atomic_json import atomic_write_json -from inferrsi.harness.optimization_session import _runtime_identity -from inferrsi.harness.task_protocol import run as protocol_operation - -from .delivery import project -from .delivery.schema import decode -from .storage.archive import _publish_directory -from .views.comparison_report import write_comparison_report - -DEFAULT_CONFIG = { - "schema_version": 1, "tasks": list(DEFAULT_TASKS), "repeats": 1, "rounds": 8, - "max_minutes": 60, "evaluation_timeout_seconds": 1200, "final_reserve_seconds": 1200, - "job_max_minutes": 90, "model": None, "effort": None, - "environment_id": None, "evaluation_prefix": "", "execution_instructions": "", - "python": "python", "policy": DEFAULT_POLICY, - "task_arguments": { - "qwen21-denoise-npu": "--pairs 2 --samples 3 --warmups 1", - "moe-forward-npu": "--pairs 2 --timeout-seconds 900", - }, -} -CONFIG_NAME = Path(".inferrsi/benchmarks.json") -OBSERVED_FIELDS = ("model", "effort", "environment_id", "environment_fingerprint") - - -def _io_path(path): - """Use Windows extended paths at I/O boundaries without persisting that spelling.""" - path = Path(path).absolute() - value = str(path) - if os.name == "nt" and not value.startswith("\\\\?\\"): - value = "\\\\?\\UNC\\" + value[2:] if value.startswith("\\\\") else "\\\\?\\" + value - return Path(value) - - -def _object(path): - path = _io_path(path) - if path.stat().st_size > 2 * 1024 * 1024: - raise ValueError("Comparison JSON document exceeds 2 MiB") - value = decode(path.read_bytes()) - if not isinstance(value, dict): - raise ValueError("Comparison JSON must contain an object") - return value - - -def resolve_config(args): - """CLI > project JSON > built-ins; retain field-level provenance.""" - config = deepcopy(DEFAULT_CONFIG) - sources = {key: "default" for key in config} - path = args.config or (CONFIG_NAME if CONFIG_NAME.is_file() else None) - if path is not None: - supplied = _object(path) - if supplied.keys() - config.keys(): - raise ValueError("Unknown comparison configuration fields") - for key, value in supplied.items(): - if key in ("policy", "task_arguments"): - if not isinstance(value, dict): - raise ValueError(f"{key} must be an object") - if key == "policy" and isinstance(value.get("tolerances"), dict): - value = {**value, "tolerances": { - **config[key]["tolerances"], **value["tolerances"]}} - config[key].update(value) - else: - config[key] = value - sources[key] = "config" - for key in config: - value = getattr(args, "task" if key == "tasks" else key, None) - if value is not None: - config[key] = value - sources[key] = "cli" - for override in args.evaluation_args or []: - task, sep, arguments = override.partition("=") - if not sep or task not in config["tasks"]: - raise ValueError("--evaluation-args requires selected TASK=ARGUMENTS") - config["task_arguments"][task] = arguments - sources["task_arguments"] = "cli" - _validate_config(config) - return config, sources - - -def _validate_config(config): - if type(config["schema_version"]) is not int or config["schema_version"] != 1: - raise ValueError("Unsupported comparison configuration version") - for key in ("rounds", "repeats"): - if type(config[key]) is not int or config[key] < 1: - raise ValueError(f"{key} must be a positive integer") - for key in ("max_minutes", "evaluation_timeout_seconds", "final_reserve_seconds", - "job_max_minutes"): - if (type(config[key]) not in (int, float) or not math.isfinite(config[key]) - or config[key] <= 0): - raise ValueError(f"{key} must be finite and positive") - if not config["evaluation_timeout_seconds"] <= config["final_reserve_seconds"] < ( - config["max_minutes"] * 60 - ): - raise ValueError("Reserve at least one evaluation timeout within the campaign budget") - if config["job_max_minutes"] <= config["max_minutes"]: - raise ValueError("Agent job budget must exceed campaign budget for handoff and collection") - tasks = config["tasks"] - if (not isinstance(tasks, list) or not tasks or any(not isinstance(t, str) for t in tasks) - or len(set(tasks)) != len(tasks)): - raise ValueError("tasks must be a nonempty list of unique registered task names") - for task in tasks: - root = Path(str(task_resources(task))) - if not (root / "optimization.py").is_file(): - raise ValueError(f"Task has no iterative protocol: {task}") - if not isinstance(config["task_arguments"].get(task), str): - raise ValueError(f"Missing task arguments: {task}") - for key in ("model", "effort", "environment_id"): - if config[key] is not None and (not isinstance(config[key], str) - or not config[key].strip()): - raise ValueError(f"{key} must be nonempty or null") - for key in ("evaluation_prefix", "execution_instructions", "python"): - if not isinstance(config[key], str) or len(config[key]) > 16384: - raise ValueError(f"{key} must be a bounded string") - if not config["python"].strip(): - raise ValueError("python must name the prepared target interpreter") - # Only the task-owned device count can be interpolated, never a shell command. - shlex.split(config["evaluation_prefix"].replace("{devices}", "1")) - if "{" in config["evaluation_prefix"].replace("{devices}", ""): - raise ValueError("Only {devices} is supported in evaluation_prefix") - config["policy"] = validate_policy(config["policy"]) - - -def _manifest(root): - root = _io_path(root) - paths = sorted(root.rglob("*")) - if root.is_symlink() or any(path.is_symlink() for path in paths): - raise ValueError("Comparison resources cannot contain symbolic links") - return {path.relative_to(root).as_posix(): sha256(path) - for path in paths if path.is_file() - and "__pycache__" not in path.parts and path.suffix != ".pyc"} - - -def _copy_runtime(source, destination, condition): - source, destination = _io_path(source), _io_path(destination) - def ignored(directory, names): - result = [name for name in names if name == "__pycache__" or name.endswith(".pyc")] - if Path(directory) == source / "knowledge" and condition == "off": - result.append("data") - return result - shutil.copytree(source, destination, ignore=ignored) - - -def _prefix(config, resources): - manifest = _object(resources / "task.json") - devices = manifest.get("devices", manifest.get("world_size", 2)) - if type(devices) is not int or devices < 1: - raise ValueError("Task must declare its positive device count") - return config["evaluation_prefix"].replace("{devices}", str(devices)) - - -def prepare(args): - config, sources = resolve_config(args) - if not config["environment_id"]: - return {"state": "configuration_required", "resolved_config": config, - "parameter_sources": sources, - "next_action": { - "operation": "select_prepared_execution_environment", - "instructions": "Select an existing authorized environment satisfying the " - "task manifests. Supply --environment-id, --evaluation-prefix and optional " - "--execution-instructions or project .inferrsi/benchmarks.json. Inspect task " - "requirements with benchmarks show. Credentials stay outside configuration.", - }} - root = (args.output or Path("outputs") / ( - "knowledge-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - + "-" + uuid.uuid4().hex[:8] - )).resolve() - if root.exists(): - raise ValueError("Comparison output must be new; use --action status to resume") - runtime = Path(str(files("inferrsi"))).resolve() - if root.is_relative_to(runtime) or any(p.is_symlink() for p in runtime.rglob("*")): - raise ValueError("Runtime must contain regular files and output must be outside it") - task_settings = {} - for name in config["tasks"]: - resource = Path(str(task_resources(name))) - settings = protocol_operation(resource, sha256(resource / "optimization.py"), - "validate_arguments", - arguments=shlex.split(config["task_arguments"][name])) - task_settings[name] = {"settings": settings, "files": _manifest(resource), - "evaluation_prefix": _prefix(config, resource)} - root.parent.mkdir(parents=True, exist_ok=True) - with TemporaryDirectory(prefix=".compare-", dir=_io_path(root.parent)) as temporary: - staging = Path(temporary) / "experiment" - staging.mkdir() - _prepare_at(config, sources, runtime, task_settings, staging) - _publish_directory(staging, _io_path(root)) - return status(root) - - -def _prepare_at(config, sources, runtime, task_settings, root): - runtime_files = _manifest(runtime) - knowledge_files = {key: value for key, value in runtime_files.items() - if key.startswith("knowledge/data/")} - trials = [] - for task in config["tasks"]: - for repeat in range(1, config["repeats"] + 1): - for condition in (("on", "off") if repeat % 2 else ("off", "on")): - trial_id = f"{task}-r{repeat:03d}-{condition}" - workspace = root / "trials" / f"{len(trials) + 1:04d}" - workspace.mkdir(parents=True) - _copy_runtime(runtime, workspace / "runtime/inferrsi", condition) - trials.append({"id": trial_id, "task": task, "repeat": repeat, - "condition": condition, - "workspace": workspace.relative_to(root).as_posix()}) - plan = {"schema_version": 1, "focus": "knowledge", "config": config, - "parameter_sources": sources, "task_settings": task_settings, - "runtime_files": {key: value for key, value in runtime_files.items() - if key not in knowledge_files}, - "session_runtime_identity": _runtime_identity(), - "knowledge_files": knowledge_files, - "trials": trials, - "knowledge_boundary": "isolated runtime corpus and session gate; native host " - "filesystem/network access follows identical task instructions, not an OS sandbox"} - atomic_write_json(root / "experiment.json", plan) - atomic_write_json(root / "state.json", {"schema_version": 1, "plan_digest": digest(plan), - "collections": {}}) - - -def _load(root): - plan, state = _object(root / "experiment.json"), _object(root / "state.json") - if plan.get("schema_version") != 1 or state.get("plan_digest") != digest(plan): - raise ValueError("Frozen comparison plan changed") - for trial in plan["trials"]: - runtime = root / trial["workspace"] / "runtime/inferrsi" - expected = dict(plan["runtime_files"]) - if trial["condition"] == "on": - expected.update(plan["knowledge_files"]) - if _manifest(runtime) != expected: - raise ValueError(f"Frozen comparison runtime changed: {trial['id']}") - return plan, state - - -def _job(root, trial): - return root / "jobs" / trial["id"] - - -def _session_key(native): - # Optional routing metadata cannot establish that a reused ID is independent. - return native["host"], native["session_id"] - - -def _select(plan, trial_id): - found = [trial for trial in plan["trials"] if trial["id"] == trial_id] - if len(found) != 1: - raise ValueError("Select a declared --trial ID from comparison status") - return found[0] - - -def _prompt(root, plan, trial): - config = plan["config"] - workspace = root / trial["workspace"] - runtime = _io_path(workspace / "runtime") - campaign = workspace / "campaign" - start = [config["python"], "-B", "-m", "inferrsi", "benchmarks", "session", "start", - trial["task"], "--session", str(campaign), "--rounds", str(config["rounds"]), - "--max-minutes", str(config["max_minutes"]), - "--evaluation-timeout-seconds", str(config["evaluation_timeout_seconds"]), - "--final-reserve-seconds", str(config["final_reserve_seconds"]), - "--condition", trial["id"], "--knowledge", trial["condition"], - "--evaluation-prefix", plan["task_settings"][trial["task"]]["evaluation_prefix"], - "--evaluation-args", config["task_arguments"][trial["task"]], - "--python", config["python"]] - return ( - f"Run complete RSI optimization: {trial['task']}, condition {trial['condition']}.\n" - f"Work only in {workspace}. Set PYTHONPATH to {runtime}; disable bytecode. " - "Do not import another installed InferRSI or parent checkout. Use this fresh conversation; " - "do not inspect other trials, prior solutions, the initiating conversation or reports. " - "Do not delegate or create another conversation.\n" - f"Prepared execution environment: {config['environment_id']}.\n" - f"{config['execution_instructions']}\n" - "Preflight before search; retain Python implementation/version, device model, framework " - "and runtime versions, and allocator profile in environment-observation.json. " - "Preserve shared dependencies. " - "Record actual model/effort from the host observation when available; " - "requested settings alone do not establish actual settings.\n" - "The permitted optimization knowledge source is this runtime's session knowledge operation " - "when enabled. Other skill libraries, stores, external knowledge tools, web research " - "and prior solutions are excluded in both. Public task/command operating instructions " - "and target library APIs remain available to both. The off runtime omits the corpus. " - "Report any observed policy violation; do not claim an OS sandbox.\n" - f"Start argv (a JSON array, execute without a shell): {json.dumps(start)}\n" - "For remote execution, copy this exact runtime to the prepared target. Use a private " - "campaign directory there; return its entire unchanged tree to workspace/campaign above. " - "Preserve bytes and relative paths. Exclude other trials and runtimes.\n" - "Read frozen TASK.md. Analyze feedback and choose a justified optimization each round. " - "Submit candidate, --strategy and --hypothesis; cite delivered --knowledge-ref paths when " - "used and retain failures. When enabled, retrieve/read relevant material with session " - "knowledge. Complete frozen rounds and independently retest the selected candidate " - "using session final. Never replace final with a search score.\n" - "Return campaign, selected code, final delivery and environment observation. Report " - "blocked/failed execution honestly. The initiator collects host receipts and actual usage." - ) - - -def status(root): - return _status(root, *_load(root)) - - -def _status(root, plan, state): - rows, active = [], None - for trial in plan["trials"]: - job = _job(root, trial) - view = agent_sessions.status(job) if job.exists() else None - collection = state["collections"].get(trial["id"]) - row = {**trial, "job": str(job), "campaign": str(root / trial["workspace"] / "campaign"), - "state": view["state"] if view else "pending", "collection": collection, - "host": view} - rows.append(row) - if active is None and collection is None: - active = row - action = {"operation": "report", "instructions": "All declared trials were collected; " - "generate the comparison report, including failures."} - published = state.get("report") - if (active is None and published - and published["collection_digest"] == digest(state["collections"])): - action = {"operation": "completed", "reports": published["paths"], - "instructions": "Read the generated comparison and report its scoped conclusion. " - "Use --action report to reverify evidence or regenerate a snapshot."} - if active is not None: - if active["host"] is None: - action = {"operation": "dispatch", "trial": active["id"], - "instructions": "Reserve one handoff; call the host's fresh-session tool " - "with its exact prompt and workspace. Bind the returned native ID."} - elif active["state"] in agent_sessions.TERMINAL: - action = {"operation": "collect", "trial": active["id"], - "instructions": "Retain actual host metadata and campaign tree; collect. " - "Host completion is separate from independent final acceptance."} - else: - action = {"operation": active["host"]["next_action"], "trial": active["id"], - "native_session": active["host"]["native_session"], - "instructions": "Observe or resolve this original session before dispatching " - "another. Export observations with agents record or compare --action record.", - "observation_fields": { - "host": "bound host", "host_id": "bound host ID or null", - "session_id": "bound session ID", "state": "actual native state", - "native_receipt": "original host tool response object", - "benchmark_observation": { - "model": "actually observed model", "effort": "observed effort", - "environment_id": plan["config"]["environment_id"], - "environment_fingerprint": "digest of observed Python implementation " - "and version, runtime versions, device model and allocator profile", - "knowledge_policy_violation": "false only when no violation observed", - }, - "usage": "optional actual cumulative input/output/cached_input tokens", - "usage_scope": "session_total when usage is present", - }} - return {"output": str(root), "focus": plan["focus"], "config": plan["config"], - "parameter_sources": plan["parameter_sources"], "trials": rows, - "state": "collected" if active is None else "running", "next_action": action} - - -def dispatch(root, trial_id, *, parallel=False): - plan, state = _load(root) - trial = _select(plan, trial_id) - job = _job(root, trial) - if not job.exists(): - if not parallel: - current = _status(root, plan, state)["next_action"] - if current.get("trial") != trial_id or current["operation"] != "dispatch": - raise ValueError("Dispatch the next declared trial or pass --parallel") - job.parent.mkdir(exist_ok=True) - agent_sessions.prepare( - job, prompt=_prompt(root, plan, trial), workspace=root / trial["workspace"], - title=trial_id, max_seconds=plan["config"]["job_max_minutes"] * 60, - model=plan["config"]["model"], effort=plan["config"]["effort"], - task_reference={"comparison": digest(plan), "trial": trial_id}, - frozen_inputs=(root / "experiment.json",), - ) - return agent_sessions.dispatch(job) - - -def _trial_result(root, plan, trial, host): - result = {"id": trial["id"], "task": trial["task"], "repeat": trial["repeat"], - "condition": trial["condition"], "status": "blocked", "eligible": False, - "gaps": [], "metrics": {"speedup": None, "wall_seconds": None, "tokens": None}, - "trace": [], "final": None, "usage": {}, "knowledge_deliveries": [], - "observed_settings": None} - campaign = root / trial["workspace"] / "campaign" - if host is None or host["state"] not in agent_sessions.TERMINAL: - result["status"] = "pending" if host is None else "running" - result["gaps"].append("Native execution has not terminated") - return result - result["status"] = "failed" if host["state"] != "completed" else "blocked" - receipt = host["last_observation"] - if receipt: - payload = _object(_job(root, trial) / receipt["path"]) - observed = payload.get("benchmark_observation") - if isinstance(observed, dict): - result["observed_settings"] = observed - observed = result["observed_settings"] or {} - if any(not isinstance(observed.get(key), str) or not observed[key].strip() - for key in OBSERVED_FIELDS): - result["gaps"].append("Actual host model, effort or environment observation is missing") - if observed.get("environment_id") != plan["config"]["environment_id"]: - result["gaps"].append("Observed execution environment differs from the frozen profile") - for key in ("model", "effort"): - if plan["config"][key] and observed.get(key) != plan["config"][key]: - result["gaps"].append(f"Observed {key} differs from the requested setting") - if observed.get("knowledge_policy_violation") is not False: - result["gaps"].append("Knowledge-policy observation is missing or reports a violation") - if host["usage"] is not None and host["state"] == "completed": - result["usage"]["host_session_tokens"] = {"value": host["usage"], - "coverage": "complete", "source": "caller_exported_terminal_native_receipt"} - result["metrics"]["tokens"] = host["usage"]["input_tokens"] + host["usage"]["output_tokens"] - if not campaign.is_dir(): - result["gaps"].append("Campaign evidence was not returned") - return result - try: - snapshot, delivery, _ = project(campaign, export_id="comparison-read") - campaign_plan, control, _ = optimization.read_records(campaign) - settings = plan["task_settings"][trial["task"]] - wanted = {"task": trial["task"], "condition": trial["id"], - "knowledge_allowed": trial["condition"] == "on", - "round_limit": plan["config"]["rounds"], "round_mode": "exact", - "max_seconds": plan["config"]["max_minutes"] * 60, - "evaluation_timeout_seconds": plan["config"]["evaluation_timeout_seconds"], - "final_reserve_seconds": plan["config"]["final_reserve_seconds"], - "evaluation_settings": settings["settings"], - "evaluation_prefix": shlex.split(settings["evaluation_prefix"]), - "frozen_files": settings["files"]} - if any(campaign_plan.get(key) != value for key, value in wanted.items()): - result["gaps"].append("Campaign task, condition, protocol or budget differs from plan") - if campaign_plan.get("runtime_identity") != plan["session_runtime_identity"]: - result["gaps"].append("Campaign runtime identity differs from the frozen runtime") - result["knowledge_deliveries"] = control["contexts"] - if trial["condition"] == "off" and control["contexts"]: - result["gaps"].append("Knowledge was delivered in the off condition") - result["usage"].update(delivery["usage"]) - if delivery["acceptance"]["delivery_accepted"] and control["final"]: - final_assessment = control["final"]["assessment"] - result["metrics"]["speedup"] = ( - 1.0 if control["best"]["kind"] == "baseline" - else (final_assessment.get("speedup") - if final_assessment.get("speedup") is not None - else final_assessment.get("score")) - ) - result["metrics"]["wall_seconds"] = delivery["usage"]["wall_seconds"]["value"] - result["final"] = {"assessment": control["final"]["assessment"] - if control["final"] else None, - "speedup": result["metrics"]["speedup"], - "binding": (delivery["final"] or {}).get("binding"), - "selected_candidate": control["best"], - "delivery_accepted": delivery["acceptance"]["delivery_accepted"]} - for index, record in enumerate(control["rounds"], 1): - assessment = record["assessment"] - result["trace"].append({ - "round": index, "strategy": record.get("strategy") or record["hypothesis"], - "hypothesis": record["hypothesis"], - "knowledge_refs": record.get("knowledge_refs"), - "available_contexts": record["contexts"], - "knowledge_attribution": "agent_declared; delivery is not proof of use", - "speedup": assessment.get("speedup", assessment.get("score")), - "best_speedup": record["best_after"]["score"], "quality": assessment["verdict"], - "adopted": record["best_after"].get("candidate_id") == record["candidate_id"], - }) - if (host["state"] != "completed" or not delivery["acceptance"]["delivery_accepted"] - or delivery["completion"]["status"] != "completed"): - result["gaps"].append("A completed, verified independent final is required") - if not snapshot.stable(): - result["gaps"].append("Campaign changed while collecting") - result["status"] = "completed" if control["state"] == "completed" else "blocked" - except (OSError, ValueError, TypeError, KeyError) as error: - result["gaps"].append(f"Campaign evidence could not be verified: {type(error).__name__}") - result["eligible"] = not result["gaps"] - return result - - -def collect(root, trial_id): - plan, state = _load(root) - trial = _select(plan, trial_id) - job = _job(root, trial) - host = agent_sessions.status(job) if job.exists() else None - if host is None or host["state"] not in agent_sessions.TERMINAL: - raise ValueError("Collect only after recording a terminal native host observation") - result = _trial_result(root, plan, trial, host) - directory = root / "collections" - directory.mkdir(exist_ok=True) - path = directory / (trial_id + "-" + uuid.uuid4().hex + ".json") - atomic_write_json(path, result) - state["collections"][trial_id] = {"path": path.relative_to(root).as_posix(), - "sha256": sha256(path)} - atomic_write_json(root / "state.json", state) - return {"trial": result, "next": _status(root, plan, state)["next_action"]} - - -def report(root): - plan, state = _load(root) - trials = [] - sessions = {} - for trial in plan["trials"]: - reference = state["collections"].get(trial["id"]) - if reference and sha256(root / reference["path"]) != reference["sha256"]: - raise ValueError("Retained comparison collection changed") - job = _job(root, trial) - host = agent_sessions.status(job) if job.exists() else None - # Reverify current evidence; a stale collection cannot hide later evidence loss. - result = _trial_result(root, plan, trial, host) - trials.append(result) - if host and host["native_session"]: - sessions.setdefault(_session_key(host["native_session"]), []).append(result) - for members in sessions.values(): - if len(members) > 1: - for trial in members: - trial["eligible"] = False - trial["gaps"].append("Native session is bound to multiple comparison trials") - for task in plan["config"]["tasks"]: - for repeat in range(1, plan["config"]["repeats"] + 1): - pair = [t for t in trials if t["task"] == task and t["repeat"] == repeat] - if (all(t["eligible"] for t in pair) and len(pair) == 2 - and any(pair[0]["observed_settings"].get(key) - != pair[1]["observed_settings"].get(key) for key in OBSERVED_FIELDS)): - for trial in pair: - trial["eligible"] = False - trial["gaps"].append("Paired actual Agent/environment settings differ") - result = {"schema_version": 1, "focus": "knowledge", "tasks": plan["config"]["tasks"], - "repeats": plan["config"]["repeats"], "policy": plan["config"]["policy"], - "knowledge_boundary": plan["knowledge_boundary"], "plan_digest": digest(plan), - "trials": trials, "assessment": summarize(trials, plan["config"]["tasks"], - plan["config"]["repeats"], plan["config"]["policy"])} - destination = root / "reports" / uuid.uuid4().hex - paths = write_comparison_report(result, destination) - state["report"] = {"collection_digest": digest(state["collections"]), "paths": paths} - atomic_write_json(root / "state.json", state) - return {"assessment": result["assessment"], "reports": paths, - "next_action": _status(root, plan, state)["next_action"]} - - -def compare_command(args): - if args.action == "prepare": - return prepare(args) - if args.output is None: - raise ValueError("Existing comparison operations require --output") - root = args.output.resolve() - with file_lock(root / ".compare.lock"): - if args.action == "status": - return status(root) - if args.action == "report": - return report(root) - if args.action == "dispatch": - return dispatch(root, args.trial, parallel=args.parallel) - if args.action == "collect": - return collect(root, args.trial) - plan, _ = _load(root) - trial = _select(plan, args.trial) - job = _job(root, trial) - if args.action == "bind": - if not args.host or not args.session_id or args.receipt is None: - raise ValueError("bind requires --host, --session-id and --receipt") - for other in plan["trials"]: - other_job = _job(root, other) - if other_job == job or not other_job.exists(): - continue - native = agent_sessions.status(other_job)["native_session"] - if native and _session_key(native) == (args.host, args.session_id): - raise ValueError(f"Native session is already bound to trial {other['id']}") - return agent_sessions.bind(job, host=args.host, session_id=args.session_id, - host_id=args.host_id, receipt=args.receipt) - if args.observation is None: - raise ValueError("record requires --observation") - return agent_sessions.record(job, observation=args.observation) +"""Compatibility exports for :mod:`inferrsi.apps.workflows.benchmark_compare`.""" + +from .workflows.benchmark_compare import ( + CONFIG_NAME, + DEFAULT_CONFIG, + OBSERVED_FIELDS, + collect, + compare_command, + dispatch, + prepare, + report, + resolve_config, + status, +) + +__all__ = [ + "DEFAULT_CONFIG", + "CONFIG_NAME", + "OBSERVED_FIELDS", + "resolve_config", + "prepare", + "status", + "dispatch", + "collect", + "report", + "compare_command", +] diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 8d1cec095..7c5cd9e10 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -351,7 +351,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "benchmarks": if args.benchmark_action == "compare": - from ..benchmark_compare import compare_command + from ..workflows.benchmark_compare import compare_command _print(compare_command(args)) return 0 diff --git a/src/inferrsi/apps/workflows/benchmark_compare.py b/src/inferrsi/apps/workflows/benchmark_compare.py new file mode 100644 index 000000000..3542d46a7 --- /dev/null +++ b/src/inferrsi/apps/workflows/benchmark_compare.py @@ -0,0 +1,580 @@ +"""Compose paired optimization experiments with native host handoffs. + +The Agent calls the host tools returned by this application. Python never +creates a conversation, invents host observations, or edits a candidate. +""" + +import json +import math +import os +import shlex +import shutil +import uuid +from copy import deepcopy +from datetime import datetime, timezone +from importlib.resources import files +from pathlib import Path +from tempfile import TemporaryDirectory + +from inferrsi.benchmark.comparison import DEFAULT_POLICY, DEFAULT_TASKS, summarize, validate_policy +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import agent_sessions, optimization +from inferrsi.harness.artifact_primitives import digest, file_lock, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import _runtime_identity +from inferrsi.harness.task_protocol import run as protocol_operation + +from ..delivery import project +from ..delivery.schema import decode +from ..storage.archive import _publish_directory +from ..views.comparison_report import write_comparison_report + +DEFAULT_CONFIG = { + "schema_version": 1, "tasks": list(DEFAULT_TASKS), "repeats": 1, "rounds": 8, + "max_minutes": 60, "evaluation_timeout_seconds": 1200, "final_reserve_seconds": 1200, + "job_max_minutes": 90, "model": None, "effort": None, + "environment_id": None, "evaluation_prefix": "", "execution_instructions": "", + "python": "python", "policy": DEFAULT_POLICY, + "task_arguments": { + "qwen21-denoise-npu": "--pairs 2 --samples 3 --warmups 1", + "moe-forward-npu": "--pairs 2 --timeout-seconds 900", + }, +} +CONFIG_NAME = Path(".inferrsi/benchmarks.json") +OBSERVED_FIELDS = ("model", "effort", "environment_id", "environment_fingerprint") + + +def _io_path(path): + """Use Windows extended paths at I/O boundaries without persisting that spelling.""" + path = Path(path).absolute() + value = str(path) + if os.name == "nt" and not value.startswith("\\\\?\\"): + value = "\\\\?\\UNC\\" + value[2:] if value.startswith("\\\\") else "\\\\?\\" + value + return Path(value) + + +def _object(path): + path = _io_path(path) + if path.stat().st_size > 2 * 1024 * 1024: + raise ValueError("Comparison JSON document exceeds 2 MiB") + value = decode(path.read_bytes()) + if not isinstance(value, dict): + raise ValueError("Comparison JSON must contain an object") + return value + + +def resolve_config(args): + """CLI > project JSON > built-ins; retain field-level provenance.""" + config = deepcopy(DEFAULT_CONFIG) + sources = {key: "default" for key in config} + path = args.config or (CONFIG_NAME if CONFIG_NAME.is_file() else None) + if path is not None: + supplied = _object(path) + if supplied.keys() - config.keys(): + raise ValueError("Unknown comparison configuration fields") + for key, value in supplied.items(): + if key in ("policy", "task_arguments"): + if not isinstance(value, dict): + raise ValueError(f"{key} must be an object") + if key == "policy" and isinstance(value.get("tolerances"), dict): + value = {**value, "tolerances": { + **config[key]["tolerances"], **value["tolerances"]}} + config[key].update(value) + else: + config[key] = value + sources[key] = "config" + for key in config: + value = getattr(args, "task" if key == "tasks" else key, None) + if value is not None: + config[key] = value + sources[key] = "cli" + for override in args.evaluation_args or []: + task, sep, arguments = override.partition("=") + if not sep or task not in config["tasks"]: + raise ValueError("--evaluation-args requires selected TASK=ARGUMENTS") + config["task_arguments"][task] = arguments + sources["task_arguments"] = "cli" + _validate_config(config) + return config, sources + + +def _validate_config(config): + if type(config["schema_version"]) is not int or config["schema_version"] != 1: + raise ValueError("Unsupported comparison configuration version") + for key in ("rounds", "repeats"): + if type(config[key]) is not int or config[key] < 1: + raise ValueError(f"{key} must be a positive integer") + for key in ("max_minutes", "evaluation_timeout_seconds", "final_reserve_seconds", + "job_max_minutes"): + if (type(config[key]) not in (int, float) or not math.isfinite(config[key]) + or config[key] <= 0): + raise ValueError(f"{key} must be finite and positive") + if not config["evaluation_timeout_seconds"] <= config["final_reserve_seconds"] < ( + config["max_minutes"] * 60 + ): + raise ValueError("Reserve at least one evaluation timeout within the campaign budget") + if config["job_max_minutes"] <= config["max_minutes"]: + raise ValueError("Agent job budget must exceed campaign budget for handoff and collection") + tasks = config["tasks"] + if (not isinstance(tasks, list) or not tasks or any(not isinstance(t, str) for t in tasks) + or len(set(tasks)) != len(tasks)): + raise ValueError("tasks must be a nonempty list of unique registered task names") + for task in tasks: + root = Path(str(task_resources(task))) + if not (root / "optimization.py").is_file(): + raise ValueError(f"Task has no iterative protocol: {task}") + if not isinstance(config["task_arguments"].get(task), str): + raise ValueError(f"Missing task arguments: {task}") + for key in ("model", "effort", "environment_id"): + if config[key] is not None and (not isinstance(config[key], str) + or not config[key].strip()): + raise ValueError(f"{key} must be nonempty or null") + for key in ("evaluation_prefix", "execution_instructions", "python"): + if not isinstance(config[key], str) or len(config[key]) > 16384: + raise ValueError(f"{key} must be a bounded string") + if not config["python"].strip(): + raise ValueError("python must name the prepared target interpreter") + # Only the task-owned device count can be interpolated, never a shell command. + shlex.split(config["evaluation_prefix"].replace("{devices}", "1")) + if "{" in config["evaluation_prefix"].replace("{devices}", ""): + raise ValueError("Only {devices} is supported in evaluation_prefix") + config["policy"] = validate_policy(config["policy"]) + + +def _manifest(root): + root = _io_path(root) + paths = sorted(root.rglob("*")) + if root.is_symlink() or any(path.is_symlink() for path in paths): + raise ValueError("Comparison resources cannot contain symbolic links") + return {path.relative_to(root).as_posix(): sha256(path) + for path in paths if path.is_file() + and "__pycache__" not in path.parts and path.suffix != ".pyc"} + + +def _copy_runtime(source, destination, condition): + source, destination = _io_path(source), _io_path(destination) + def ignored(directory, names): + result = [name for name in names if name == "__pycache__" or name.endswith(".pyc")] + if Path(directory) == source / "knowledge" and condition == "off": + result.append("data") + return result + shutil.copytree(source, destination, ignore=ignored) + + +def _prefix(config, resources): + manifest = _object(resources / "task.json") + devices = manifest.get("devices", manifest.get("world_size", 2)) + if type(devices) is not int or devices < 1: + raise ValueError("Task must declare its positive device count") + return config["evaluation_prefix"].replace("{devices}", str(devices)) + + +def prepare(args): + config, sources = resolve_config(args) + if not config["environment_id"]: + return {"state": "configuration_required", "resolved_config": config, + "parameter_sources": sources, + "next_action": { + "operation": "select_prepared_execution_environment", + "instructions": "Select an existing authorized environment satisfying the " + "task manifests. Supply --environment-id, --evaluation-prefix and optional " + "--execution-instructions or project .inferrsi/benchmarks.json. Inspect task " + "requirements with benchmarks show. Credentials stay outside configuration.", + }} + root = (args.output or Path("outputs") / ( + "knowledge-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + uuid.uuid4().hex[:8] + )).resolve() + if root.exists(): + raise ValueError("Comparison output must be new; use --action status to resume") + runtime = Path(str(files("inferrsi"))).resolve() + if root.is_relative_to(runtime) or any(p.is_symlink() for p in runtime.rglob("*")): + raise ValueError("Runtime must contain regular files and output must be outside it") + task_settings = {} + for name in config["tasks"]: + resource = Path(str(task_resources(name))) + settings = protocol_operation(resource, sha256(resource / "optimization.py"), + "validate_arguments", + arguments=shlex.split(config["task_arguments"][name])) + task_settings[name] = {"settings": settings, "files": _manifest(resource), + "evaluation_prefix": _prefix(config, resource)} + root.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=".compare-", dir=_io_path(root.parent)) as temporary: + staging = Path(temporary) / "experiment" + staging.mkdir() + _prepare_at(config, sources, runtime, task_settings, staging) + _publish_directory(staging, _io_path(root)) + return status(root) + + +def _prepare_at(config, sources, runtime, task_settings, root): + runtime_files = _manifest(runtime) + knowledge_files = {key: value for key, value in runtime_files.items() + if key.startswith("knowledge/data/")} + trials = [] + for task in config["tasks"]: + for repeat in range(1, config["repeats"] + 1): + for condition in (("on", "off") if repeat % 2 else ("off", "on")): + trial_id = f"{task}-r{repeat:03d}-{condition}" + workspace = root / "trials" / f"{len(trials) + 1:04d}" + workspace.mkdir(parents=True) + _copy_runtime(runtime, workspace / "runtime/inferrsi", condition) + trials.append({"id": trial_id, "task": task, "repeat": repeat, + "condition": condition, + "workspace": workspace.relative_to(root).as_posix()}) + plan = {"schema_version": 1, "focus": "knowledge", "config": config, + "parameter_sources": sources, "task_settings": task_settings, + "runtime_files": {key: value for key, value in runtime_files.items() + if key not in knowledge_files}, + "session_runtime_identity": _runtime_identity(), + "knowledge_files": knowledge_files, + "trials": trials, + "knowledge_boundary": "isolated runtime corpus and session gate; native host " + "filesystem/network access follows identical task instructions, not an OS sandbox"} + atomic_write_json(root / "experiment.json", plan) + atomic_write_json(root / "state.json", {"schema_version": 1, "plan_digest": digest(plan), + "collections": {}}) + + +def _load(root): + plan, state = _object(root / "experiment.json"), _object(root / "state.json") + if plan.get("schema_version") != 1 or state.get("plan_digest") != digest(plan): + raise ValueError("Frozen comparison plan changed") + for trial in plan["trials"]: + runtime = root / trial["workspace"] / "runtime/inferrsi" + expected = dict(plan["runtime_files"]) + if trial["condition"] == "on": + expected.update(plan["knowledge_files"]) + if _manifest(runtime) != expected: + raise ValueError(f"Frozen comparison runtime changed: {trial['id']}") + return plan, state + + +def _job(root, trial): + return root / "jobs" / trial["id"] + + +def _session_key(native): + # Optional routing metadata cannot establish that a reused ID is independent. + return native["host"], native["session_id"] + + +def _select(plan, trial_id): + found = [trial for trial in plan["trials"] if trial["id"] == trial_id] + if len(found) != 1: + raise ValueError("Select a declared --trial ID from comparison status") + return found[0] + + +def _prompt(root, plan, trial): + config = plan["config"] + workspace = root / trial["workspace"] + runtime = _io_path(workspace / "runtime") + campaign = workspace / "campaign" + start = [config["python"], "-B", "-m", "inferrsi", "benchmarks", "session", "start", + trial["task"], "--session", str(campaign), "--rounds", str(config["rounds"]), + "--max-minutes", str(config["max_minutes"]), + "--evaluation-timeout-seconds", str(config["evaluation_timeout_seconds"]), + "--final-reserve-seconds", str(config["final_reserve_seconds"]), + "--condition", trial["id"], "--knowledge", trial["condition"], + "--evaluation-prefix", plan["task_settings"][trial["task"]]["evaluation_prefix"], + "--evaluation-args", config["task_arguments"][trial["task"]], + "--python", config["python"]] + return ( + f"Run complete RSI optimization: {trial['task']}, condition {trial['condition']}.\n" + f"Work only in {workspace}. Set PYTHONPATH to {runtime}; disable bytecode. " + "Do not import another installed InferRSI or parent checkout. Use this fresh conversation; " + "do not inspect other trials, prior solutions, the initiating conversation or reports. " + "Do not delegate or create another conversation.\n" + f"Prepared execution environment: {config['environment_id']}.\n" + f"{config['execution_instructions']}\n" + "Preflight before search; retain Python implementation/version, device model, framework " + "and runtime versions, and allocator profile in environment-observation.json. " + "Preserve shared dependencies. " + "Record actual model/effort from the host observation when available; " + "requested settings alone do not establish actual settings.\n" + "The permitted optimization knowledge source is this runtime's session knowledge operation " + "when enabled. Other skill libraries, stores, external knowledge tools, web research " + "and prior solutions are excluded in both. Public task/command operating instructions " + "and target library APIs remain available to both. The off runtime omits the corpus. " + "Report any observed policy violation; do not claim an OS sandbox.\n" + f"Start argv (a JSON array, execute without a shell): {json.dumps(start)}\n" + "For remote execution, copy this exact runtime to the prepared target. Use a private " + "campaign directory there; return its entire unchanged tree to workspace/campaign above. " + "Preserve bytes and relative paths. Exclude other trials and runtimes.\n" + "Read frozen TASK.md. Analyze feedback and choose a justified optimization each round. " + "Submit candidate, --strategy and --hypothesis; cite delivered --knowledge-ref paths when " + "used and retain failures. When enabled, retrieve/read relevant material with session " + "knowledge. Complete frozen rounds and independently retest the selected candidate " + "using session final. Never replace final with a search score.\n" + "Return campaign, selected code, final delivery and environment observation. Report " + "blocked/failed execution honestly. The initiator collects host receipts and actual usage." + ) + + +def status(root): + return _status(root, *_load(root)) + + +def _status(root, plan, state): + rows, active = [], None + for trial in plan["trials"]: + job = _job(root, trial) + view = agent_sessions.status(job) if job.exists() else None + collection = state["collections"].get(trial["id"]) + row = {**trial, "job": str(job), "campaign": str(root / trial["workspace"] / "campaign"), + "state": view["state"] if view else "pending", "collection": collection, + "host": view} + rows.append(row) + if active is None and collection is None: + active = row + action = {"operation": "report", "instructions": "All declared trials were collected; " + "generate the comparison report, including failures."} + published = state.get("report") + if (active is None and published + and published["collection_digest"] == digest(state["collections"])): + action = {"operation": "completed", "reports": published["paths"], + "instructions": "Read the generated comparison and report its scoped conclusion. " + "Use --action report to reverify evidence or regenerate a snapshot."} + if active is not None: + if active["host"] is None: + action = {"operation": "dispatch", "trial": active["id"], + "instructions": "Reserve one handoff; call the host's fresh-session tool " + "with its exact prompt and workspace. Bind the returned native ID."} + elif active["state"] in agent_sessions.TERMINAL: + action = {"operation": "collect", "trial": active["id"], + "instructions": "Retain actual host metadata and campaign tree; collect. " + "Host completion is separate from independent final acceptance."} + else: + action = {"operation": active["host"]["next_action"], "trial": active["id"], + "native_session": active["host"]["native_session"], + "instructions": "Observe or resolve this original session before dispatching " + "another. Export observations with agents record or compare --action record.", + "observation_fields": { + "host": "bound host", "host_id": "bound host ID or null", + "session_id": "bound session ID", "state": "actual native state", + "native_receipt": "original host tool response object", + "benchmark_observation": { + "model": "actually observed model", "effort": "observed effort", + "environment_id": plan["config"]["environment_id"], + "environment_fingerprint": "digest of observed Python implementation " + "and version, runtime versions, device model and allocator profile", + "knowledge_policy_violation": "false only when no violation observed", + }, + "usage": "optional actual cumulative input/output/cached_input tokens", + "usage_scope": "session_total when usage is present", + }} + return {"output": str(root), "focus": plan["focus"], "config": plan["config"], + "parameter_sources": plan["parameter_sources"], "trials": rows, + "state": "collected" if active is None else "running", "next_action": action} + + +def dispatch(root, trial_id, *, parallel=False): + plan, state = _load(root) + trial = _select(plan, trial_id) + job = _job(root, trial) + if not job.exists(): + if not parallel: + current = _status(root, plan, state)["next_action"] + if current.get("trial") != trial_id or current["operation"] != "dispatch": + raise ValueError("Dispatch the next declared trial or pass --parallel") + job.parent.mkdir(exist_ok=True) + agent_sessions.prepare( + job, prompt=_prompt(root, plan, trial), workspace=root / trial["workspace"], + title=trial_id, max_seconds=plan["config"]["job_max_minutes"] * 60, + model=plan["config"]["model"], effort=plan["config"]["effort"], + task_reference={"comparison": digest(plan), "trial": trial_id}, + frozen_inputs=(root / "experiment.json",), + ) + return agent_sessions.dispatch(job) + + +def _trial_result(root, plan, trial, host): + result = {"id": trial["id"], "task": trial["task"], "repeat": trial["repeat"], + "condition": trial["condition"], "status": "blocked", "eligible": False, + "gaps": [], "metrics": {"speedup": None, "wall_seconds": None, "tokens": None}, + "trace": [], "final": None, "usage": {}, "knowledge_deliveries": [], + "observed_settings": None} + campaign = root / trial["workspace"] / "campaign" + if host is None or host["state"] not in agent_sessions.TERMINAL: + result["status"] = "pending" if host is None else "running" + result["gaps"].append("Native execution has not terminated") + return result + result["status"] = "failed" if host["state"] != "completed" else "blocked" + receipt = host["last_observation"] + if receipt: + payload = _object(_job(root, trial) / receipt["path"]) + observed = payload.get("benchmark_observation") + if isinstance(observed, dict): + result["observed_settings"] = observed + observed = result["observed_settings"] or {} + if any(not isinstance(observed.get(key), str) or not observed[key].strip() + for key in OBSERVED_FIELDS): + result["gaps"].append("Actual host model, effort or environment observation is missing") + if observed.get("environment_id") != plan["config"]["environment_id"]: + result["gaps"].append("Observed execution environment differs from the frozen profile") + for key in ("model", "effort"): + if plan["config"][key] and observed.get(key) != plan["config"][key]: + result["gaps"].append(f"Observed {key} differs from the requested setting") + if observed.get("knowledge_policy_violation") is not False: + result["gaps"].append("Knowledge-policy observation is missing or reports a violation") + if host["usage"] is not None and host["state"] == "completed": + result["usage"]["host_session_tokens"] = {"value": host["usage"], + "coverage": "complete", "source": "caller_exported_terminal_native_receipt"} + result["metrics"]["tokens"] = host["usage"]["input_tokens"] + host["usage"]["output_tokens"] + if not campaign.is_dir(): + result["gaps"].append("Campaign evidence was not returned") + return result + try: + snapshot, delivery, _ = project(campaign, export_id="comparison-read") + campaign_plan, control, _ = optimization.read_records(campaign) + settings = plan["task_settings"][trial["task"]] + wanted = {"task": trial["task"], "condition": trial["id"], + "knowledge_allowed": trial["condition"] == "on", + "round_limit": plan["config"]["rounds"], "round_mode": "exact", + "max_seconds": plan["config"]["max_minutes"] * 60, + "evaluation_timeout_seconds": plan["config"]["evaluation_timeout_seconds"], + "final_reserve_seconds": plan["config"]["final_reserve_seconds"], + "evaluation_settings": settings["settings"], + "evaluation_prefix": shlex.split(settings["evaluation_prefix"]), + "frozen_files": settings["files"]} + if any(campaign_plan.get(key) != value for key, value in wanted.items()): + result["gaps"].append("Campaign task, condition, protocol or budget differs from plan") + if campaign_plan.get("runtime_identity") != plan["session_runtime_identity"]: + result["gaps"].append("Campaign runtime identity differs from the frozen runtime") + result["knowledge_deliveries"] = control["contexts"] + if trial["condition"] == "off" and control["contexts"]: + result["gaps"].append("Knowledge was delivered in the off condition") + result["usage"].update(delivery["usage"]) + if delivery["acceptance"]["delivery_accepted"] and control["final"]: + final_assessment = control["final"]["assessment"] + result["metrics"]["speedup"] = ( + 1.0 if control["best"]["kind"] == "baseline" + else (final_assessment.get("speedup") + if final_assessment.get("speedup") is not None + else final_assessment.get("score")) + ) + result["metrics"]["wall_seconds"] = delivery["usage"]["wall_seconds"]["value"] + result["final"] = {"assessment": control["final"]["assessment"] + if control["final"] else None, + "speedup": result["metrics"]["speedup"], + "binding": (delivery["final"] or {}).get("binding"), + "selected_candidate": control["best"], + "delivery_accepted": delivery["acceptance"]["delivery_accepted"]} + for index, record in enumerate(control["rounds"], 1): + assessment = record["assessment"] + result["trace"].append({ + "round": index, "strategy": record.get("strategy") or record["hypothesis"], + "hypothesis": record["hypothesis"], + "knowledge_refs": record.get("knowledge_refs"), + "available_contexts": record["contexts"], + "knowledge_attribution": "agent_declared; delivery is not proof of use", + "speedup": assessment.get("speedup", assessment.get("score")), + "best_speedup": record["best_after"]["score"], "quality": assessment["verdict"], + "adopted": record["best_after"].get("candidate_id") == record["candidate_id"], + }) + if (host["state"] != "completed" or not delivery["acceptance"]["delivery_accepted"] + or delivery["completion"]["status"] != "completed"): + result["gaps"].append("A completed, verified independent final is required") + if not snapshot.stable(): + result["gaps"].append("Campaign changed while collecting") + result["status"] = "completed" if control["state"] == "completed" else "blocked" + except (OSError, ValueError, TypeError, KeyError) as error: + result["gaps"].append(f"Campaign evidence could not be verified: {type(error).__name__}") + result["eligible"] = not result["gaps"] + return result + + +def collect(root, trial_id): + plan, state = _load(root) + trial = _select(plan, trial_id) + job = _job(root, trial) + host = agent_sessions.status(job) if job.exists() else None + if host is None or host["state"] not in agent_sessions.TERMINAL: + raise ValueError("Collect only after recording a terminal native host observation") + result = _trial_result(root, plan, trial, host) + directory = root / "collections" + directory.mkdir(exist_ok=True) + path = directory / (trial_id + "-" + uuid.uuid4().hex + ".json") + atomic_write_json(path, result) + state["collections"][trial_id] = {"path": path.relative_to(root).as_posix(), + "sha256": sha256(path)} + atomic_write_json(root / "state.json", state) + return {"trial": result, "next": _status(root, plan, state)["next_action"]} + + +def report(root): + plan, state = _load(root) + trials = [] + sessions = {} + for trial in plan["trials"]: + reference = state["collections"].get(trial["id"]) + if reference and sha256(root / reference["path"]) != reference["sha256"]: + raise ValueError("Retained comparison collection changed") + job = _job(root, trial) + host = agent_sessions.status(job) if job.exists() else None + # Reverify current evidence; a stale collection cannot hide later evidence loss. + result = _trial_result(root, plan, trial, host) + trials.append(result) + if host and host["native_session"]: + sessions.setdefault(_session_key(host["native_session"]), []).append(result) + for members in sessions.values(): + if len(members) > 1: + for trial in members: + trial["eligible"] = False + trial["gaps"].append("Native session is bound to multiple comparison trials") + for task in plan["config"]["tasks"]: + for repeat in range(1, plan["config"]["repeats"] + 1): + pair = [t for t in trials if t["task"] == task and t["repeat"] == repeat] + if (all(t["eligible"] for t in pair) and len(pair) == 2 + and any(pair[0]["observed_settings"].get(key) + != pair[1]["observed_settings"].get(key) for key in OBSERVED_FIELDS)): + for trial in pair: + trial["eligible"] = False + trial["gaps"].append("Paired actual Agent/environment settings differ") + result = {"schema_version": 1, "focus": "knowledge", "tasks": plan["config"]["tasks"], + "repeats": plan["config"]["repeats"], "policy": plan["config"]["policy"], + "knowledge_boundary": plan["knowledge_boundary"], "plan_digest": digest(plan), + "trials": trials, "assessment": summarize(trials, plan["config"]["tasks"], + plan["config"]["repeats"], plan["config"]["policy"])} + destination = root / "reports" / uuid.uuid4().hex + paths = write_comparison_report(result, destination) + state["report"] = {"collection_digest": digest(state["collections"]), "paths": paths} + atomic_write_json(root / "state.json", state) + return {"assessment": result["assessment"], "reports": paths, + "next_action": _status(root, plan, state)["next_action"]} + + +def compare_command(args): + if args.action == "prepare": + return prepare(args) + if args.output is None: + raise ValueError("Existing comparison operations require --output") + root = args.output.resolve() + with file_lock(root / ".compare.lock"): + if args.action == "status": + return status(root) + if args.action == "report": + return report(root) + if args.action == "dispatch": + return dispatch(root, args.trial, parallel=args.parallel) + if args.action == "collect": + return collect(root, args.trial) + plan, _ = _load(root) + trial = _select(plan, args.trial) + job = _job(root, trial) + if args.action == "bind": + if not args.host or not args.session_id or args.receipt is None: + raise ValueError("bind requires --host, --session-id and --receipt") + for other in plan["trials"]: + other_job = _job(root, other) + if other_job == job or not other_job.exists(): + continue + native = agent_sessions.status(other_job)["native_session"] + if native and _session_key(native) == (args.host, args.session_id): + raise ValueError(f"Native session is already bound to trial {other['id']}") + return agent_sessions.bind(job, host=args.host, session_id=args.session_id, + host_id=args.host_id, receipt=args.receipt) + if args.observation is None: + raise ValueError("record requires --observation") + return agent_sessions.record(job, observation=args.observation) diff --git a/tests/check_installed.py b/tests/check_installed.py index 0054365a9..7407bbd28 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -624,7 +624,7 @@ def _inventory(root, prefix): ("candidate_provider", "handoff"), ("campaign", "workflows"), ("benchmark_operations", "workflows"), ("improvement", "workflows"), ("skill_evaluation", "workflows"), - ("skill_eval_engineering", "workflows"), + ("skill_eval_engineering", "workflows"), ("benchmark_compare", "workflows"), ): old_name = f"inferrsi.apps.{name}" canonical_name = f"inferrsi.apps.{group}.{name}" diff --git a/tests/test_benchmark_compare.py b/tests/test_benchmark_compare.py index 4f0038d00..5b0dfe8d8 100644 --- a/tests/test_benchmark_compare.py +++ b/tests/test_benchmark_compare.py @@ -10,8 +10,8 @@ import pytest -from inferrsi.apps import benchmark_compare as compare from inferrsi.apps.cli import main +from inferrsi.apps.workflows import benchmark_compare as compare from inferrsi.benchmark import registry from inferrsi.harness import agent_sessions from inferrsi.harness.optimization_session import _runtime_identity From 8d19587d3d18c97e8f6ef79a4cdc641eb9bf1649 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Thu, 19 Nov 2026 15:48:53 +0800 Subject: [PATCH 3/3] Document workflow relocation and pinned runtime recovery Signed-off-by: zengchuang Agent-assisted-by: Codex (author) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d50cfec3..1bfcc7fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ### Changed +- Apps workflow implementations now live in `apps/workflows/`: campaign capture, + task launching, paired benchmark comparison, improvement and skill evaluation. + The previous Apps imports retain explicit compatibility exports. New optimization + plans fingerprint the canonical implementation; existing frozen sessions must use + their original pinned runtime for `status` and `recover`. See + [release-boundary recovery](docs/optimization-workflow-unification.md#release-boundary-recovery). - `docs/contracts/schemas/contract-v1.schema.json` validates more strictly (Refs #497). Eleven `pattern` values ended with `$`, which in Python `re`, and so in `jsonschema`, also matches before one trailing LF: `"att-1\n"` passed `Id`. They now