Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@

### Changed

- Apps workflow implementations now live in `apps/workflows/`: campaign capture,
task launching, paired benchmark comparison, improvement and skill evaluation.
The previous Apps imports retain explicit compatibility exports. New optimization
plans fingerprint the canonical implementation; existing frozen sessions must use
their original pinned runtime for `status` and `recover`. See
[release-boundary recovery](docs/optimization-workflow-unification.md#release-boundary-recovery).
- The host bindings move from `inferrsi.infra` to the new `inferrsi.apps.host`
package (Refs #498, slice 3a). There is no forwarding module or alias, and the
import gate fails any import of an old path.
Expand Down
155 changes: 154 additions & 1 deletion docs/apps-module-migrations.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/api.py",
"src/inferrsi/apps/benchmark_compare.py",
"src/inferrsi/apps/archive.py",
Comment thread
zengchuang-hw marked this conversation as resolved.
Resolved
"src/inferrsi/apps/workflows/benchmark_compare.py",
"src/inferrsi/apps/delivery/__init__.py",
"src/inferrsi/apps/handoff/bridge.py",
"src/inferrsi/apps/handoff/candidate_provider.py",
Expand Down Expand Up @@ -51,6 +52,7 @@
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/api.py",
"src/inferrsi/apps/artifact_reader.py",
"src/inferrsi/apps/handoff/candidate_provider.py",
"src/inferrsi/apps/views/attempt.py",
"tests/check_installed.py",
Expand Down Expand Up @@ -81,6 +83,7 @@
"src/inferrsi/apps/delivery/adapters.py",
"src/inferrsi/apps/handoff/bridge.py",
"src/inferrsi/apps/handoff/candidate_provider.py",
"src/inferrsi/apps/result.py",
"src/inferrsi/apps/storage/archive.py",
"src/inferrsi/apps/storage/artifact_reader.py",
"src/inferrsi/apps/views/attempt.py",
Expand All @@ -101,6 +104,7 @@
"docs/contracts/current-design.md",
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/evidence_layout.py",
"src/inferrsi/contracts/evidence.py",
"tests/_gate_policy_support.py",
"tests/check_installed.py",
Expand All @@ -126,6 +130,7 @@
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/api.py",
"src/inferrsi/apps/bridge.py",
"src/inferrsi/apps/handoff/candidate_provider.py",
"src/inferrsi/apps/views/attempt.py",
"tests/check_installed.py",
Expand All @@ -147,6 +152,7 @@
"docs/architecture.md",
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/context_provider.py",
"tests/check_installed.py",
"tests/test_apps_context_provider.py",
"tests/test_apps_module_migrations.py",
Expand All @@ -165,8 +171,155 @@
"docs/module-layout.md",
"docs/operations.md",
"src/inferrsi/apps/api.py",
"src/inferrsi/apps/candidate_provider.py",
"tests/check_installed.py",
"tests/test_apps_module_migrations.py"
]
},
{
"old": "inferrsi.apps.campaign",
"canonical": "inferrsi.apps.workflows.campaign",
"exports": [
"create_campaign",
"capture_context",
"save_retrieval",
"verify_capture"
],
"consumers": [
"docs/architecture.md",
"docs/knowledge-retrieval-trace.md",
"docs/module-layout.md",
"docs/operations.md",
"docs/optimization-workflow-unification.md",
"src/inferrsi/apps/cli/__init__.py",
"src/inferrsi/apps/views/capture_delivery.py",
"src/inferrsi/apps/views/optimization_story.py",
"src/inferrsi/harness/optimization_session.py",
"tests/check_installed.py",
"tests/fixtures/dashboard_profile_review.py",
"tests/fixtures/knowledge_retrieval.py",
"tests/test_apps_module_migrations.py",
"tests/test_campaign_record_injection.py",
"tests/test_campaign_task_selection.py",
"tests/test_knowledge_navigation_browser.py",
"tests/test_optimization_session.py",
"tests/test_optimization_story.py",
"tests/test_round_citations.py",
"tests/test_search_archive.py",
"tests/test_search_attribution.py",
"tests/test_search_capture_delivery.py"
]
},
{
"old": "inferrsi.apps.benchmark_operations",
"canonical": "inferrsi.apps.workflows.benchmark_operations",
"exports": [
"run_task"
],
"consumers": [
"docs/architecture.md",
"docs/module-layout.md",
"src/inferrsi/apps/cli/__init__.py",
"tests/_gate_policy_support.py",
"tests/check_installed.py",
"tests/test_apps_module_migrations.py",
"tests/test_benchmark_operations.py",
"tests/test_optimization_session.py",
"tests/test_package_root_inventory.py",
"tests/test_project_optimization.py",
"tests/test_target_module_import_gate.py"
]
},
{
"old": "inferrsi.apps.improvement",
"canonical": "inferrsi.apps.workflows.improvement",
"exports": [
"assessment",
"choose",
"holdout_accepts",
"rerun_assessment"
],
"consumers": [
"docs/contracts/function-owners.md",
"docs/harness-improvement.md",
"docs/module-layout.md",
"tests/check_installed.py",
"tests/test_apps_module_migrations.py",
"tests/test_harness_improvement.py",
"tools/skill_eval_improve.py"
]
},
{
"old": "inferrsi.apps.skill_evaluation",
"canonical": "inferrsi.apps.workflows.skill_evaluation",
"exports": [
"export_bundle",
"summarize",
"run_trial"
],
"consumers": [
"docs/contracts/function-owners.md",
"docs/module-layout.md",
"docs/skill-evaluation.md",
"tests/check_installed.py",
"tests/test_anthropic_agent.py",
"tests/test_apps_module_migrations.py",
"tests/test_skill_eval_compare.py",
"tests/test_skill_eval_generation.py",
"tests/test_skill_eval_pipeline.py",
"tests/test_skill_evaluation.py",
"tools/skill_eval_compare.py",
"tools/skill_eval_pilot.py"
]
},
{
"old": "inferrsi.apps.skill_eval_engineering",
"canonical": "inferrsi.apps.workflows.skill_eval_engineering",
"exports": [
"verify",
"main"
],
"consumers": [
"docs/contracts/function-owners.md",
"docs/local-skill-evaluation.md",
"docs/module-layout.md",
"src/inferrsi/apps/workflows/skill_evaluation.py",
"tests/check_installed.py",
"tests/test_apps_module_migrations.py",
"tests/test_skill_eval_compare.py",
"tests/test_skill_eval_engineering.py",
"tests/test_skill_eval_generation.py",
"tests/test_skill_eval_local.py",
"tests/test_skill_eval_local_cli.py",
"tools/skill_eval_cloud_replay.py",
"tools/skill_eval_compare.py",
"tools/skill_eval_local.py",
"tools/skill_eval_qualify.py"
],
"script_entry": "main"
},
{
"old": "inferrsi.apps.benchmark_compare",
"canonical": "inferrsi.apps.workflows.benchmark_compare",
"exports": [
"DEFAULT_CONFIG",
"CONFIG_NAME",
"OBSERVED_FIELDS",
"resolve_config",
"prepare",
"status",
"dispatch",
"collect",
"report",
"compare_command"
],
"consumers": [
"docs/architecture.md",
"docs/module-layout.md",
"src/inferrsi/apps/cli/__init__.py",
"tests/check_installed.py",
"tests/test_apps_module_migrations.py",
"tests/test_benchmark_compare.py"
]
}
]
39 changes: 19 additions & 20 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ packages. `inferrsi benchmarks run <package> ...` launches the package's
`benchmark.py`. Task-specific orchestration, sampling and assessment live inside
each package. The task library owns its `index.json` registration. The existing
`benchmark/registry.py` discovers the resources. The CLI package `apps/cli/`
and `apps/benchmark_operations.py` provide application assembly and launch.
and `apps/workflows/benchmark_operations.py` provide application assembly and launch.
Each task retains its own protocol and task-specific code.

The [current-Agent optimization session](optimization-benchmarks.md) assembles
Expand All @@ -16,7 +16,7 @@ budget, attempts, feedback records, selection and independent final execution.
`harness/task_protocol.py` freezes a supplied resource directory and loads its
task-owned assessment protocol. `harness/optimization_session.py` assembles
execution and assessment; the CLI resolves task resources and injects knowledge
access and the capture callback from `apps/campaign.py`.
access and the capture callback from `apps/workflows/campaign.py`.
The external Agent chooses and edits candidates between round submissions.
The same controller serves ordinary projects through `campaign start/round/final`
and packaged tasks through `benchmarks session`. The
Expand Down Expand Up @@ -163,7 +163,6 @@ the comparator; the engine AB/BA path retains its independent acceptance contrac

```text
src/inferrsi/
apps/cli/ CLI entry `main` and its argument parser
benchmark/
registry.py Generic task resource discovery
tasks/ Task-specific definitions, workloads and assessment
Expand All @@ -178,14 +177,17 @@ src/inferrsi/
skill_eval_diagnostic.py Synthetic KV admission diagnosis task and grader
skill_eval_engineering.py Serving-parser task, public vectors and vector judgment
improvement.py Improvement-step thresholds and acceptance rules
contracts/
runspec.py Immutable experiment contract; the initializer re-exports only RunSpec
json_v1.py Canonical inferrsi-json-v1 encoder and decoder
claim.py Campaign verdict labels PASS/FAIL/INCONCLUSIVE; rules only
evidence.py Evidence segments and per-campaign path rules; no filesystem I/O
resources/
__init__.py Import-free initializer
repository_packs.py Neutral REPO_TYPES and pack.json reader; imports no inferrsi module
apps/
api.py Python entry points for assembling and publishing Apps operations
benchmark_operations.py Resolve a task entry and launch it through Harness
archive.py Publish and read the standardized Apps output in a workspace
result.py Assemble and check the AppsResultV1 output envelope
skill_evaluation.py Composed bundle export, trial run and summary
skill_eval_engineering.py Composed artifact check: task rules around sandbox replay
improvement.py Composed assessment, selection, rerun and holdout acceptance
cli/ CLI entry `main` and its argument parser
inspection.py Environment inspection bound to the host probe and Git identity
host/
inspection.py Environment inspection facts API; command and snapshot ports
Expand All @@ -197,15 +199,6 @@ src/inferrsi/
run.py Old run composed from Harness, Benchmark and the report
evaluation.py Old evaluate entry: RunSpec to the Benchmark gates
reporting.py HTML evidence report
contracts/
runspec.py Immutable experiment contract; the initializer re-exports only RunSpec
json_v1.py Canonical inferrsi-json-v1 encoder and decoder
claim.py Campaign verdict labels PASS/FAIL/INCONCLUSIVE; rules only
evidence.py Evidence segments and per-campaign path rules; no filesystem I/O
resources/
__init__.py Import-free initializer
repository_packs.py Neutral REPO_TYPES and pack.json reader; imports no inferrsi module
apps/
handoff/
bridge.py Bind preparations to specs and fixed records to evidence
context_provider.py Deliver Knowledge bytes and pins for Harness sealing
Expand All @@ -215,7 +208,13 @@ src/inferrsi/
artifact_reader.py Read verified artifacts through authorized mounts
result.py AppsResultV1 construction and validation
evidence_layout.py Create plan/results/patch/validator evidence directories
campaign.py Knowledge values connected to Harness campaign records
workflows/
campaign.py Knowledge values connected to Harness campaign records
benchmark_operations.py Resolve a task entry and launch it through Harness
benchmark_compare.py Paired benchmark plans, native handoff, collection and reports
skill_evaluation.py Composed bundle export, trial run and summary
skill_eval_engineering.py Composed artifact check: task rules around sandbox replay
improvement.py Composed assessment, selection, rerun and holdout acceptance
mindie/
project.py Opt-in per-checkout grant, the worker launch and the recorded read
context.py Export supplied MindIE reads into a caller-reviewed campaign trace
Expand Down Expand Up @@ -295,7 +294,7 @@ The package root holds exactly two tracked files: `__init__.py` and `__main__.py
`mcp.py` and the standalone `read_worker.py`) and deleted root `mindie_project.py`,
`mindie_context.py`, `mindie_mcp.py` and `_mindie_read_worker.py` with no forwarding
shell or alias. #408 A2 moved the chart renderer to
`apps/views/summary_chart.py` and the task launcher to `apps/benchmark_operations.py`
`apps/views/summary_chart.py` and the task launcher to `apps/workflows/benchmark_operations.py`
and deleted root `summary_chart.py` and `benchmark_operations.py` the same way. #408 A1 moved the CLI and its parser to `apps/cli/`
and deleted root `cli.py` and `arguments.py`;
`__main__.py` and the console script call `inferrsi.apps.cli:main`. #412 R1 deleted eleven root modules with no
Expand Down
2 changes: 1 addition & 1 deletion docs/contracts/function-owners.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Any later split of a mixed function is accepted by real pytest/host/wheel/record

## Implemented rows

The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py` and `apps/improvement.py`. Two points follow the tree as it stands after #338, not the row text:
The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/workflows/skill_evaluation.py`, `apps/workflows/skill_eval_engineering.py` and `apps/workflows/improvement.py`. Two points follow the tree as it stands after #338, not the row text:

- `usage_totals` stays in `harness/usage.py`, because `harness/optimization.py` consumes it for budget accounting. B `assessment` receives the baseline and candidate totals from Apps and does not import H.
- `SelectionPolicy` holds only the two thresholds, so it lives whole in B. H `choose` receives the assessments, the validity block and the composition flag; it holds no threshold and no policy class.
2 changes: 1 addition & 1 deletion docs/harness-improvement.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ flowchart TB

`src/inferrsi/benchmark/improvement.py` owns the thresholds and acceptance rules,
`src/inferrsi/harness/improvement.py` the search progression and
`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/improvement.py`
`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/workflows/improvement.py`
composes them.
`tools/skill_eval_improve.py` owns freezing, declarative snapshot materialization
and orchestration through `tools/skill_eval_compare.py`. There is no second
Expand Down
2 changes: 1 addition & 1 deletion docs/knowledge-retrieval-trace.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ same immutable capture envelope used by `capture-context`:
```python
from pathlib import Path
from inferrsi.knowledge.catalog import search_knowledge
from inferrsi.apps.campaign import save_retrieval
from inferrsi.apps.workflows.campaign import save_retrieval

result = search_knowledge("decode launch", explain=True)
path, sha256 = save_retrieval(
Expand Down
2 changes: 1 addition & 1 deletion docs/local-skill-evaluation.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ snapshots. `tools/skill_eval_local.py` adds manual stage/capture, without launch
an agent. `src/inferrsi/benchmark/skill_eval_engineering.py` owns the one task, its
19 public behavior probes and their judgment. `src/inferrsi/harness/artifact_sandbox.py`
replays the retained artifact under bubblewrap and separates the single writable
artifact from frozen inputs; `src/inferrsi/apps/skill_eval_engineering.py` composes the two.
artifact from frozen inputs; `src/inferrsi/apps/workflows/skill_eval_engineering.py` composes the two.
The existing cloud read/submit route is unchanged;
it cannot perform this artifact-editing task and is rejected for this adapter.

Expand Down
Loading
Loading