Remove cm

.gitignore

@@ -55,6 +55,11 @@ outputs/
 # Large generated files — not tracked in git
 *.bak
 *.pptx
-workflows/run_campaign/dreamer_campaign/*.json
-workflows/run_campaign/dreamer_campaign/log
-workflows/run_campaign/dreamer_campaign/plots/
+
+# Runtime output directories (campaign artifacts)
+workflows/esm2_inference/ESM2
+workflows/esm2_inference/cache/
+workflows/*/telemetry*/
+workflows/*/*.npy
+workflows/*/ddict_*
+workflows/sgdes/mayv_output/

README.md

@@ -5,6 +5,7 @@ HPC workflow orchestration framework for multi-GPU protein inference and enginee
 ## Features
 
 - **AsyncCampaignManager** — async-native orchestrator for concurrent multi-workflow campaigns with priority scheduling, resource pools, and dependency signalling
+- **Adaptive Optimization Layers** — opt-in, config-driven: quality routing (Sharder), flow control (Backpressure), surrogate-gated Triage (RUN/DISCARD/ADVANCE), and a BudgetController that keeps spend on plan; drift-driven Replanning. Cross-stage scheduling priority is driven by the **ADR agent layer** (rule / bandit / LLM policies), not an in-CM bandit
 - **Multi-GPU Inference** — worker pool per GPU with automatic load balancing; aiohttp HTTP server/client
 - **ESM2 Inference Workflow** — standalone or campaign-embedded ESM2-650M embedding service
 - **SGDES Workflow** — Structure-Guided Deep Evolution Solver for iterative protein sequence optimisation
@@ -20,8 +21,6 @@ HPC workflow orchestration framework for multi-GPU protein inference and enginee
 ```
 spherical/
 ├── src/
-│   ├── campaign/                    # AsyncCampaignManager + BaseWorkflow + ResourcePool
-│   │   └── campaign_manager.py
 │   ├── inference/                   # InferenceService base, orchestrator, server
 │   │   ├── esm2_service/            # ESM2InferenceService + ESM2Client
 │   │   ├── inference_client.py
@@ -36,18 +35,20 @@ spherical/
 │   ├── esm2_inference/              # Standalone ESM2 inference runner
 │   │   ├── run_esm2_infern.py
 │   │   └── config.yaml
-│   ├── run_campaign/                # Multi-workflow campaign (DDSim + Inference)
-│   │   ├── run_campaing.py
-│   │   ├── inference_workflow.py
-│   │   ├── ddmd_workflow.py
-│   │   ├── plot_cm_timeline.py.py         # Gantt timeline + resource chart from SLURM log
-│   │   └── config.yaml
+│   ├── run_campaign/                # Multi-workflow campaigns
+│   │   ├── plot_cm_timeline.py         # Gantt timeline + resource chart from SLURM log
+│   │   ├── esm2_ddsim_campaign/        # real HPC campaign: ESM2 inference + DeepDriveSim (Dragon/GPU)
+│   │   │   ├── run_campaing.py · config.yaml · gpu_sbatch.sh
+│   │   │   └── inference_workflow.py · ddmd_workflow.py · miniapps_workflow.py · dummy_workflow.py
+│   │   └── dreamer_campaign/           # in-process emulation (radical.dreamer) for benchmarking
+│   │       ├── run_campaign.py · config*.yaml
+│   │       ├── benchmark.py · benchmark_adr.py    # feature-flag + ADR-policy benchmarks
+│   │       └── plot_optimizations.py · plot_policy_comparison.py · plot_deadline_yield.py
 │   └── sgdes/                       # SGDES protein engineering
 │       ├── run_workflow.py
 │       ├── sgdes_workflow.py
 │       └── config.yaml
 └── tests/
-    ├── test_campaign_manager.py
     ├── test_inference_service.py
     ├── test_client.py
     ├── test_server.py
@@ -107,8 +108,17 @@ service_python: "${VE_HOME}/esm2/bin/python"   # resolved at load time
 
 ### Multi-workflow Campaign
 
+Two campaigns ship under `workflows/run_campaign/`:
+
 ```bash
-python workflows/run_campaign/run_campaing.py --config workflows/run_campaign/config.yaml
+# Real HPC campaign (ESM2 inference + DeepDriveSim) — Dragon backend, real GPUs:
+cd workflows/run_campaign/esm2_ddsim_campaign
+dragon run_campaing.py --config config.yaml
+# local smoke test (no Dragon): python run_campaing.py --config config.yaml --engine concurrent
+
+# Emulated campaign (radical.dreamer, in-process) — for benchmarking scheduling policies:
+cd workflows/run_campaign/dreamer_campaign
+python run_campaign.py --config config.yaml --policy rule    # none | rule | bandit | llm
 ```
 
 Config structure:
@@ -118,19 +128,25 @@ resources:
   total_cpus: 128
   total_gpus: 4
 
+# Optional ADR agent layer — drives cross-stage scheduling priority each tick.
+cm:
+  adr:
+    policy: rule        # none | rule | bandit | llm  (override with --policy)
+    tick_s: 2.0
+
 workflows:
   ddsim:
     replicas:      8
-    min_replicas:  2
-    max_replicas:  4
+    concurrency_floor:  2
+    concurrency_cap:  4
     priority:      5
     required_cpus: 20
     dependencies:  []
 
   inference:
     replicas:      16
-    min_replicas:  1
-    max_replicas:  4
+    concurrency_floor:  1
+    concurrency_cap:  4
     priority:      10
     required_cpus: 32
     required_gpus: 1
@@ -147,40 +163,6 @@ See [workflows/sgdes/README.md](workflows/sgdes/README.md) for full setup, confi
 
 ---
 
-## Campaign Manager
-
-`AsyncCampaignManager` orchestrates heterogeneous workflow groups inside a single `asyncio` event loop.
-
-### Authoring a workflow
-
-```python
-from src.campaign import BaseWorkflow
-
-class MyWorkflow(BaseWorkflow):
-    workflow_id = "my_wf"
-
-    async def run(self, replica_id: str) -> None:
-        await do_work(self.asyncflow, self.config)
-        await self._signal_ready()          # unblock dependent groups immediately
-
-    async def on_replica_done(self, replica_id, cm, final_state):
-        if final_state == "done":
-            await cm.add_replicas("downstream", n=1)
-```
-
-### Runner pattern
-
-```python
-cm = AsyncCampaignManager.from_config(config, WORKFLOW_REGISTRY)
-await cm.start()
-await cm.wait()
-await cm.close()
-```
-
-See [src/campaign/README.md](src/campaign/README.md) for full API reference, scheduler details, and a live run trace.
-
----
-
 ## Extending for New Model Types
 
 Subclass `InferenceService` from `src.inference.inference_service`:
@@ -250,13 +232,13 @@ bash workflows/plot_telemetry.sh \
 
 ### Campaign Manager replica timeline
 
-`workflows/run_campaign/plot_cm_timeline.py.py` parses a SLURM output log and
+`workflows/run_campaign/plot_cm_timeline.py` parses a SLURM output log and
 produces a Gantt chart of replica execution spans with a resource utilization
 panel (GPU/CPU in use over time) and a campaign config summary table.
 
 ```bash
-python workflows/run_campaign/plot_cm_timeline.py.py slurm-<jobid>.out \
-    [--config workflows/run_campaign/config.yaml] \
+python workflows/run_campaign/plot_cm_timeline.py slurm-<jobid>.out \
+    [--config workflows/run_campaign/esm2_ddsim_campaign/config.yaml] \
     [--out timeline.png]
 ```
 
@@ -272,11 +254,110 @@ from the log lines.
 
 **Example**:
 ```bash
-python workflows/run_campaign/plot_cm_timeline.py.py \
+python workflows/run_campaign/plot_cm_timeline.py \
     workflows/run_campaign/slurm-17715157.out \
     --out replica_timeline.png
 ```
 
+### Dreamer campaign timeline (with simulation stats)
+
+`workflows/run_campaign/dreamer_campaign/plot_dreamer_timeline.py` is a
+Dreamer-specific superset of the timeline above: it produces the same Gantt +
+resource-utilization rows **plus** a third row of emulation metrics (simulated
+makespan per replica, task-ops box plots from the `dreamer-profiles/*.json`,
+and a per-workflow stats table).
+
+```bash
+python workflows/run_campaign/dreamer_campaign/plot_dreamer_timeline.py <log> \
+    [--profiles-dir dreamer-profiles/] \
+    [--config workflows/run_campaign/dreamer_campaign/config.yaml] \
+    [--out dreamer_timeline.png]
+```
+
+The profiles directory is auto-detected next to the log when `--profiles-dir`
+is omitted. Use `plot_cm_timeline.py` for non-Dreamer campaigns.
+
+### Benchmark optimization plots
+
+`workflows/run_campaign/dreamer_campaign/plot_optimizations.py` reads the
+`benchmark_results.json` produced by `benchmark.py` and writes 7 comparison
+plots (wall time, pipeline Gantt, cascade funnel, GPU utilization, shard
+dispatch, bandit convergence, time-to-target) — one per optimization axis.
+
+```bash
+# 1. produce the results (N runs per configuration)
+python workflows/run_campaign/dreamer_campaign/benchmark.py \
+    --config workflows/run_campaign/dreamer_campaign/config.yaml \
+    --runs 5 --out benchmark_results.json
+
+# 2. render the plots
+python workflows/run_campaign/dreamer_campaign/plot_optimizations.py \
+    [--results benchmark_results.json] \
+    [--out-dir plots/optimizations]
+```
+
+Config display names are mapped via `CFG_DISPLAY` and workflow stage labels via
+`DISPLAY` at the top of the script; both default to the antigen-cascade names.
+
+### Budget-control illustration
+
+`workflows/run_campaign/dreamer_campaign/plot_budget_control.py` renders the
+score-cutoff adaptation and burn-ratio convergence for the `budget_control`
+benchmark case (a 2-panel figure) from the same `benchmark_results.json`.
+
+```bash
+python workflows/run_campaign/dreamer_campaign/plot_budget_control.py \
+    [--results benchmark_results.json] \
+    [--out plots/diagrams/budget_control_illustration.png]
+```
+
+### ADR scheduling-policy comparison
+
+The dreamer runner can drive scheduling from a swappable `radical.adr` policy
+(`--policy {none|rule|bandit|llm}`) and record each decision cycle to JSONL with
+`--record`. `plot_policy_comparison.py` then plots the policies side by side —
+assigned priority per workflow over cycles — so the rule/llm stable downstream-first
+ladder contrasts visually with the bandit's still-exploring (reshuffling) priorities.
+
+```bash
+cd workflows/run_campaign/dreamer_campaign
+
+# run the same campaign under each policy, recording decisions
+python run_campaign.py --policy rule   --record
+python run_campaign.py --policy bandit --record
+python run_campaign.py --policy llm    --record    # needs OPENROUTER_API_KEY
+
+# plot them together
+python plot_policy_comparison.py \
+    adr-decisions-rule.jsonl adr-decisions-bandit.jsonl adr-decisions-llm.jsonl \
+    --out plots/policy_comparison.png
+```
+
+Requires `pip install -e ".[adr]"` (the LLM policy also needs `".[llm]"`). The
+policy and recording can also be set in `config.yaml` under `cm.adr`.
+
+**Batch benchmark (all policies in one job).** `benchmark_adr.py` runs every
+policy N times (same metrics shape as `benchmark.py`), writing one results JSON
+plus per-cycle decision logs under `adr-logs/`:
+
+```bash
+python workflows/run_campaign/dreamer_campaign/benchmark_adr.py \
+    --runs 5 --out benchmark_adr_results.json
+    # or restrict: --policies none rule bandit
+```
+
+Cross-stage scheduling priority is owned entirely by the ADR policy (the CM has
+no in-loop scheduling bandit); `--policy bandit` runs the same Thompson-sampling
+bandit wrapped as an ADR agent.
+
+`benchmark_adr.py` also supports a **deadline-yield** objective (`--mode
+deadline-yield --deadline 60`): instead of time-to-N-leads, it measures how many
+terminal leads each policy produces within a fixed wall-clock window (higher is
+better — the realistic HPC framing). `plot_deadline_yield.py` renders the
+leads-per-policy figure with per-run spread. For the full analysis of when each
+policy wins and why downstream-first is hard to beat, see
+[docs/scheduling_policy_comparison.md](docs/scheduling_policy_comparison.md).
+
 ---
 
 ## Development

pyproject.toml

@@ -5,13 +5,13 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "spherical"
 version = "0.1.0"
-description = "Multi-GPU Inference Service Framework with Worker Pool Management"
+description = "Multi-GPU Inference Service Framework — ESM2 inference service and SGDES workflow"
 
 authors = [
-    {name = "Masha", email = "masha@example.com"},
+    {name = "Masha", email = "mg2347@soe.rutgers.edu"},
 ]
 maintainers = [
-    {name = "Masha", email = "masha@example.com"},
+    {name = "Masha", email = "mg2347@soe.rutgers.edu"},
 ]
 readme = "README.md"
 requires-python = ">=3.10"
@@ -29,23 +29,23 @@ Homepage = "https://github.com/masha/spherical"
 Issues = "https://github.com/masha/spherical/issues"
 
 [project.optional-dependencies]
-# ESM2 model support
+# ESM2 model support (torch + transformers)
 esm2 = [
     "torch>=2.0.0,<2.4.0",
     "transformers>=4.30.0",
     "numpy>=1.24.0,<2.0.0",
 ]
 
-# Dragon/RADICAL support
+# Dragon/RADICAL HPC backend
 dragon = [
     "dragonhpc>=0.13.2",
     "rhapsody-py>=0.2.0",
     "nvidia-ml-py"
 ]
 
-# SGDES workflow (examples/sgdes) — PyPI-available deps only.
+# SGDES workflow — PyPI-available deps only.
 # PyTorch+CUDA and foldseek/seqkit binaries must be installed separately;
-# see examples/sgdes/requirements.txt and delta_env_setup.sh / bridges2_env_setup.sh.
+# see workflows/sgdes/requirements.txt and delta_env_setup.sh / bridges2_env_setup.sh.
 sgdes = [
     "biopython>=1.81",
     "scikit-learn>=1.4.0",
@@ -89,15 +89,9 @@ doc = [
     "mkdocstrings[python]>=0.24.0",
 ]
 
-# Plotting/metrics visualization
-plotting = [
-    "matplotlib>=3.7.0",
-    "numpy>=1.24.0,<2.0.0",
-]
-
 [tool.setuptools.packages.find]
 where = ["."]
-include = ["src*", "examples*"]
+include = ["src*"]
 
 [tool.pyright]
 pythonVersion = "3.10"
@@ -120,7 +114,7 @@ indent-style = "space"
 [tool.pytest.ini_options]
 minversion = "7.0"
 testpaths = ["tests"]
-asyncio_mode = "auto"
+asyncio_mode = "strict"
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "integration: marks tests as integration tests",
@@ -129,7 +123,7 @@ markers = [
 
 [tool.coverage.run]
 source = ["src"]
-omit = ["tests/*", "example/*"]
+omit = ["tests/*"]
 
 [tool.coverage.report]
 exclude_lines = [