tmp: verify #229 with Tests workflow from #228 (do not merge)

.coderabbit.yaml

@@ -0,0 +1,14 @@
+# CodeRabbit config. Install the app: https://github.com/apps/coderabbitai
+language: en-US
+reviews:
+  profile: chill
+  request_changes_workflow: false
+  high_level_summary: true
+  poem: false
+  review_status: true
+  collapse_walkthrough: true
+  auto_review:
+    enabled: true
+    drafts: false
+chat:
+  auto_reply: true

.gemini/config.yaml

@@ -0,0 +1,10 @@
+# Gemini Code Assist config. Install the app: https://github.com/apps/gemini-code-assist
+have_fun: false
+code_review:
+  disable: false
+  comment_severity_threshold: MEDIUM
+  max_review_comments: -1
+  pull_request_opened:
+    help: false
+    summary: true
+    code_review: true

.github/workflows/claude-review.yml

@@ -0,0 +1,52 @@
+name: Claude PR Review
+
+# Runs on the repo owner's Claude subscription (CLAUDE_CODE_OAUTH_TOKEN from
+# `claude setup-token`), not paid API credits. Triggers are kept narrow to save
+# plan quota: new / ready / reopened PRs, plus the `claude-review` label to re-run.
+on:
+  pull_request:
+    types: [opened, ready_for_review, reopened, labeled]
+
+concurrency:
+  group: claude-review-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  review:
+    # Skip drafts, PRs from forks (no secret access), and unrelated label events.
+    if: >-
+      github.event.pull_request.draft == false &&
+      github.event.pull_request.head.repo.full_name == github.repository &&
+      (github.event.action != 'labeled' || github.event.label.name == 'claude-review')
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          prompt: |
+            REPO: ${{ github.repository }}
+            PR NUMBER: ${{ github.event.pull_request.number }}
+
+            Review this pull request. Read AGENTS.md / CONTRIBUTING.md first if present
+            and hold the change to the repo's own conventions.
+
+            Focus on, in order: correctness bugs, security problems, broken invariants
+            or contracts, missing tests for changed behavior, and performance regressions.
+            Skip formatting and style nitpicks that linters already catch.
+
+            Leave an inline comment for each concrete issue on the exact line, saying what
+            breaks and how to fix it. Then post one short summary comment with
+            `gh pr comment`. If you find nothing worth flagging, say so in one line.
+          claude_args: |
+            --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Read,Glob,Grep"

.github/workflows/tests.yml

@@ -0,0 +1,28 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v5
+      - name: Install dependencies
+        run: uv sync --group dev
+      - name: Fast test suite
+        run: make test-fast
+        env:
+          JAX_PLATFORMS: cpu

AGENTS.md

@@ -89,7 +89,7 @@ uv run ow train ... artifacts=hybrid_promotion   # strict promotion: docker + to
 
 - Canonical feature path: Kaggle/JAX obs → `encode_turn` (planet-edge `TurnBatch`); golden tests live in `tests/test_feature_encoding_golden.py`.
 - JAX layout: rollout `src/jax/rollout/collect.py`, PPO `src/jax/ppo_update.py`, learner sampling `src/jax/action_sampling.py`, shield `src/jax/shield/*` (Python refs in `src/game/shield.py`, `shield_config.py`), train loop `src/jax/train/`, opponents `src/opponents/jax_actions/sampling.py` (mixed-family dispatch, family-batched reorder/merge, historical pool gather with `pool_row_indices` after batched path — use `jax.lax.cond` for trace-safe branches); baseline scripted (nearest_sniper/turtle/opportunistic) and random JAX families skip trajectory shield (`_scripted_edge_action`/`_random_edge_action`); learner/latest/historical self-play stay shielded; Python tournament baselines in `src/opponents/runtime.py` use `BASELINE_RUNTIME_ENV` (shield off) — Kaggle `random_agent` must not pass through `filter_moves_with_trajectory_shield` (collapses to noop); `OPPONENT_FAMILY_*` in `src/opponents/constants.py`; import `src/opponents.pool` only for pool logic. JAX training `nearest_sniper` is not proven action-for-action vs tournament `SniperOpponent`; `latest` uses `encode_turn` without learner `feature_history` — expect learner-skewed win rates, not sniper-beating proof. Learner K-scan now uses O(K) incremental decode (`factorized_decode_step` + edge-gather carry advance) via `src/jax/factored_decode_scan.py`, wired into both rollout sampling (`action_sampling.py`) and PPO replay (`factored_sequence_scan.py`); decoder hidden is exported from the final scan carry `.state` (no trailing full `factorized_decode`). Stacked latest self-play sampler gated by static config (`_static_latest_only_self_play_sample_enabled` in collect/collect_kernel) — never traced `stage_view` on jitted collect (`float(tracer)` on noop/preflight); off for `curriculum=noop_only`, on for `curriculum=latest_only`, off for mixed/historical. Per-format timing (`*_2p`/`*_4p`) only when `metric_groups.debug`; `average_placement_4p` stays default. Profile opponent collect offline via `ow benchmark rollout-phase-profile` / `rollout-phase-breakdown` — default admission preset is `noop_only`; production_mix needs `--train-overrides curriculum=production_mix` (+ `--full-geometry` for 32×256); never `telemetry=rollout_phase_timing` on production `ow train` (host sync stalls). **`ow debug`:** Tier C driver in `src/jax/debug/driver.py`; VS Code launch uses `.venv/bin/ow` (not `python -m src.cli`) plus `ORBIT_WARS_ALLOW_CPU_JAX_ON_NVIDIA=1` for `--cpu` on NVIDIA.
-- Hydra dataclass defaults in `src/config/schema.py` can differ from `conf/` YAML; verify with `uv run ow train print_resolved_config=true`. Doc-type map: `docs/README.md` (`make agent-context` → `docs.readme`); calibration index `docs/benchmarks/README.md`; operator tools `docs/tools/`. `docs/archive/` and `docs/audits/` have been removed — canonical specs live under `docs/brainstorms/` + root docs; `tests/test_docs_navigation.py` enforces no `docs/archive|plans|ideation` references and validates markdown + backtick `docs/*.md` / `scripts/*.py` links across docs, `AGENTS.md`, and `CONCEPTS.md` (do not reintroduce archive-path links).
+- Hydra dataclass defaults in `src/config/schema.py` can differ from `conf/` YAML; verify with `uv run ow train print_resolved_config=true`. Doc-type map: `docs/README.md` (`make agent-context` → `docs.readme`); calibration index `docs/benchmarks/README.md`; operator tools `docs/tools/`. The former archive and audits doc folders have been removed — canonical specs live under `docs/brainstorms/` + root docs; `tests/test_docs_navigation.py` enforces no archive/plans/ideation doc references and validates markdown + backtick doc / `scripts/*.py` links across docs, `AGENTS.md`, and `CONCEPTS.md` (do not reintroduce archive-path links).
 - **JAX/Kaggle env parity:** Kaggle reference `kaggle_environments/.../orbit_wars/orbit_wars.py`; JAX `src/jax/env.py` (single JIT-safe path — **no** `pure_callback`, **no** `_reference_*` helpers, **no** `src/game/*` imports in the env hot path; **no** parallel Python/callback fallbacks in train/rollout/env hot path). Parity via mechanical validity invariants, not bit-exact seed replay; see `docs/solutions/conventions/jax-no-kaggle-callbacks.md`. Verify with `make test-kaggle-parity` (includes `tests/test_map_pool_reset.py` when map pool is landed). Live tournament/eval/submit paths use `jax_game_from_observation_fast` obs replay — **not** map-pool indices. Planet rotation index matches Kaggle `obs.step`. Rollout opponents call `encode_turn(game, cfg.task)` without `feature_history` (learner history is updated in env `_finish_step`); Phase B `opp_batch_cache` must not reuse learner `feature_history`; opponent batch-cache skip must use `jax.lax.cond`, not Python `if` on traced values. **Map pool:** offline bake via `ow benchmark map-pool {profile,bake,validate}` → `data/jax_map_pool/default_v1.npz`; training `batched_reset_with_pool` gather only — no inline `while_loop` rejection sampling in vmapped `reset()`; comets activate from pre-baked schedules in `step()`. Map pool is the **default** training path — `conf/task/base.yaml` sets `map_pool_path: data/jax_map_pool/default_v1.npz`, inherited by every task profile (`shield_cheap`/`default`/`map_pool`); rollout init uses gather reset whenever `map_pool_path` is set, and only configs **without** a path fall to `batched_reset` → `_reset_legacy` stub board (tests only). `task=map_pool` (`conf/task/map_pool.yaml`) is just an alias to the same default pool — not "turn on map pool"; the real knob for a *different* pool is `task.map_pool_path=` (+ `map_pool_sha256`), and throughput admission uses `task=map_pool` only as a benchmarking preset label. Failed June Colab pilots collapsed on `production_mix`/early self-play, not on map pool.
 - **Hybrid promotion / submit-valid:** Decision tree in `docs/AGENT_CAPABILITIES.md` (hybrid poll → `ow eval results show` for `validation_ok`, or `ow eval package --validate-docker` with JSON `"ok": true`); local replay HTML and packaging-only are never submit-valid proof. `artifacts=hybrid_promotion` sets `promotion.strategy=hybrid`, `tournament.enabled=true`, `checkpoint_eval_async=true`, `replay.enabled=false`. Scalar metric improvements queue composite `checkpoint_eval` (Docker validation → tournament → promote). Artifacts: `evaluations/checkpoint_eval_u<update>_<id>/{docker_validation,tournament}/`. Poll `ow eval status --run <path> --watch`; dry-run `ow eval jobs cancel` before cancelling queue jobs. Bare `ow train` uses `artifacts=default`—not the submit-valid funnel.
 - **Training geometry & run layout:** `rollout_microbatch_envs` must be ≤ each rollout group's env count (`src/config/rollout_allocation.py`); canonical `training=default` inherits `2p4p_32_split` (16+16 envs, microbatch 16). Mixed 2p/4p uses Option A: separate jitted collectors per format (2p vs 4p differ in env rules/home assignment, not just seat count), sequential Python-loop collect then concat before PPO — `2p4p_*_rotate` collects one format per update. Compose tests parametrize every `conf/training/*` profile. `ow train` with `output.campaign=<name>` writes under `outputs/campaigns/<campaign>/runs/<run_id>/` — metrics in `logs/*_jax.jsonl` (not run-root `metrics.jsonl`), checkpoints in `checkpoints/jax_ckpt_*.pkl`, artifact jobs in `queue/optional_jobs/`, evaluations in `evaluations/`.

docs/README.md

@@ -64,7 +64,7 @@ Long-lived references at `docs/` root (not folder indexes):
 - [agent-native-phase3-status.md](agent-native-phase3-status.md) — shipped benchmark/gate/sweep CLI primitives
 - [nomenclature-rfc.md](nomenclature-rfc.md) — user-facing term mappings
 
-Repo root: [COLAB_LAUNCH_AND_INTEGRATION_PROMOTION.md](../COLAB_LAUNCH_AND_INTEGRATION_PROMOTION.md) — live Colab pilot tracker; [CONCEPTS.md](../CONCEPTS.md) — glossary.
+Repo root: [CONCEPTS.md](../CONCEPTS.md) — glossary. (The Colab pilot tracker was retired with the project; see [RETROSPECTIVE.md](../RETROSPECTIVE.md).)
 
 ## Maintenance
 

docs/ROADMAP.md

@@ -27,7 +27,7 @@
 | Item | Link |
 | ---- | ---- |
 | Launch hygiene learner ablation + tier-2 assessment | Tier-2 failed (~4× sec/update); hot path exhausted; B (hygiene) wins learn-proof vs A (79162a); artifact [ablation.json](benchmarks/launch-hygiene-ablation.json) | [plan](solutions/tooling-decisions/launch-hygiene-learner-ablation-gate.md) · [runbook](operator-runbook.md) |
-| Preflight learn-proof refresh (post-hygiene) | VERIFIED through `beat_random` vs `preflight-calibration.json` | [report](../outputs/preflight/factorized_post_hygiene_learn_proof.json) · [runbook](operator-runbook.md) |
+| Preflight learn-proof refresh (post-hygiene) | VERIFIED through `beat_random` vs `preflight-calibration.json` | report: `outputs/preflight/factorized_post_hygiene_learn_proof.json` (generated locally, not committed) · [runbook](operator-runbook.md) |
 | Cursor session-start hook: `docs/CURSOR.md` + project `.cursor/hooks.json` example | [plan](CURSOR.md) · [Phase 3 §4](agent-native-phase3-status.md) |
 | Observability debug bundle: `mean_ships_per_launch`, PPO `_2p`/`_4p` + update-time fractions gated behind `metric_groups.debug` | [plan](solutions/developer-experience/benchmark-subprocess-training-observability.md) |
 | Split decoder replay batch contracts (#167) | [plan](architecture/jax-policy-encoder.md) · Closes #167 |

docs/solutions/workflow-issues/colab-long-run-monitor-sync-recovery.md

@@ -283,7 +283,7 @@ Eval artifacts: `outputs/colab_runner/monitor/evals/<run_id>/jax_ckpt_<n>/`. Mon
 ## Related
 
 - Operator reference: [`docs/colab_runner.md`](../../colab_runner.md) — monitor section
-- Launch tracker: [`COLAB_LAUNCH_AND_INTEGRATION_PROMOTION.md`](../../../COLAB_LAUNCH_AND_INTEGRATION_PROMOTION.md) — pilot gates, sweep invalidation, June 16 long-run state
+- Launch tracker: `COLAB_LAUNCH_AND_INTEGRATION_PROMOTION.md` (removed at project retirement) — pilot gates, sweep invalidation, June 16 long-run state
 - Train host architecture: [`colab-train-host-preflight-long-run.md`](../architecture-patterns/colab-train-host-preflight-long-run.md)
 - CLI observability (no tail pipe): [`ow-long-cli-stderr-progress-no-tail-pipe.md`](../developer-experience/ow-long-cli-stderr-progress-no-tail-pipe.md) — prefer `ow train colab monitor` over piping `launch`
 - Sweep selection anti-patterns (generalized): [`planet-flow-sweep-gameable-objective.md`](../logic-errors/planet-flow-sweep-gameable-objective.md)

scripts/kaggle_runtime_env.py

@@ -0,0 +1,147 @@
+"""Kaggle/runtime environment helpers for Orbit Wars workers.
+
+This module must be safe to import before JAX.  The critical rule for NVIDIA
+Kaggle workers is: never set ``JAX_PLATFORMS=gpu``.  Newer JAX treats the
+``gpu`` alias as a request to consider GPU backends broadly and may attempt
+ROCm before/alongside CUDA.  Kaggle NVIDIA kernels should be pinned to
+``cuda,cpu``.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+KAGGLE_RUNTIME_ENV_PATCH_VERSION = "cuda-platform-pin-v10"
+_WORKER_VENV_ENV = "ORBIT_WARS_WORKER_VENV"
+_DRIVER_DIR_CANDIDATES: tuple[str, ...] = (
+    "/usr/local/nvidia/lib64",
+    "/usr/local/nvidia/lib",
+    "/usr/local/cuda/compat",
+    "/usr/local/cuda-12.8/compat",
+    "/usr/local/cuda/lib64",
+    "/usr/lib/x86_64-linux-gnu",
+)
+
+
+def pin_jax_platform_from_kaggle(*, env: dict[str, str] | None = None) -> None:
+    """Pin JAX's platform selection for Kaggle before importing JAX.
+
+    ``JAX_PLATFORMS=gpu`` is intentionally avoided for NVIDIA workers because it
+    can trigger ROCm initialization attempts.  Use ``cuda,cpu`` instead.
+    """
+
+    target = os.environ if env is None else env
+    accelerator_id = target.get("KAGGLE_ACCELERATOR_ID", os.environ.get("KAGGLE_ACCELERATOR_ID", ""))
+    backend = target.get("ORBIT_WARS_JAX_BACKEND", os.environ.get("ORBIT_WARS_JAX_BACKEND", "")).strip().lower()
+    target.pop("JAX_PLATFORM_NAME", None)
+
+    if backend == "tpu" or _accelerator_requests_tpu(accelerator_id):
+        target["JAX_PLATFORMS"] = "tpu,cpu"
+    elif backend == "cpu" or _truthy(target.get("ORBIT_WARS_FORCE_JAX_CPU", os.environ.get("ORBIT_WARS_FORCE_JAX_CPU", ""))):
+        target["JAX_PLATFORMS"] = "cpu"
+    elif _accelerator_requests_nvidia(accelerator_id) or _nvidia_runtime_visible():
+        target["JAX_PLATFORMS"] = "cuda,cpu"
+    else:
+        # Avoid GPU plugin probing on plain CPU hosts.  Preserve an explicit user
+        # setting if one was already provided.
+        target.setdefault("JAX_PLATFORMS", "cpu")
+
+
+def isolate_worker_python_env(*, env: dict[str, str] | None = None) -> None:
+    """Make subprocesses prefer the managed worker venv and avoid user site dirs."""
+
+    target = os.environ if env is None else env
+    target["PYTHONNOUSERSITE"] = "1"
+    target.pop("PYTHONHOME", None)
+    venv = _worker_venv(target)
+    if venv is None:
+        return
+    target["VIRTUAL_ENV"] = str(venv)
+    target["UV_PROJECT_ENVIRONMENT"] = str(venv)
+    bin_dir = venv / "bin"
+    if bin_dir.exists():
+        target["PATH"] = _prepend_path(str(bin_dir), target.get("PATH", ""))
+
+
+def add_worker_cuda_library_path(*, env: dict[str, str] | None = None) -> None:
+    """Expose venv CUDA wheel libraries plus Kaggle's NVIDIA driver libraries."""
+
+    target = os.environ if env is None else env
+    accelerator_id = target.get("KAGGLE_ACCELERATOR_ID", os.environ.get("KAGGLE_ACCELERATOR_ID", ""))
+    if not (_accelerator_requests_nvidia(accelerator_id) or _nvidia_runtime_visible()):
+        return
+    venv = _worker_venv(target)
+    dirs: list[str] = []
+    if venv is not None:
+        dirs.extend(_cuda_wheel_library_dirs(venv))
+    dirs.extend(_cuda_driver_library_dirs(target.get("LD_LIBRARY_PATH", "")))
+    if dirs:
+        target["LD_LIBRARY_PATH"] = os.pathsep.join(_dedupe(dirs))
+    target.setdefault("TF_GPU_ALLOCATOR", "cuda_malloc_async")
+
+
+def _worker_venv(env: dict[str, str]) -> Path | None:
+    value = env.get(_WORKER_VENV_ENV, os.environ.get(_WORKER_VENV_ENV, "")).strip()
+    if not value:
+        return None
+    return Path(value).resolve()
+
+
+def _cuda_wheel_library_dirs(venv: Path) -> list[str]:
+    dirs: list[str] = []
+    for site_packages in sorted(venv.glob("lib/python*/site-packages")):
+        nvidia_root = site_packages / "nvidia"
+        if not nvidia_root.exists():
+            continue
+        for lib_dir in sorted(nvidia_root.glob("*/lib")):
+            if lib_dir.is_dir():
+                dirs.append(str(lib_dir.resolve()))
+    return dirs
+
+
+def _cuda_driver_library_dirs(existing_ld_library_path: str) -> list[str]:
+    candidates = [
+        *(item for item in existing_ld_library_path.split(os.pathsep) if item),
+        *_DRIVER_DIR_CANDIDATES,
+    ]
+    result: list[str] = []
+    for item in candidates:
+        path = Path(item)
+        if not path.is_dir():
+            continue
+        if any(path.glob("libcuda.so*")) or any(path.glob("libnvidia-ml.so*")):
+            result.append(str(path.resolve()))
+    return _dedupe(result)
+
+
+def _nvidia_runtime_visible() -> bool:
+    return Path("/usr/local/nvidia/lib64").is_dir() or any(Path("/dev").glob("nvidia[0-9]*"))
+
+
+def _accelerator_requests_nvidia(accelerator_id: str) -> bool:
+    return accelerator_id.strip().lower().startswith("nvidia")
+
+
+def _accelerator_requests_tpu(accelerator_id: str) -> bool:
+    return accelerator_id.strip().lower().startswith("tpu")
+
+
+def _truthy(value: str) -> bool:
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _prepend_path(item: str, existing: str) -> str:
+    parts = [part for part in existing.split(os.pathsep) if part]
+    return os.pathsep.join([item, *(part for part in parts if part != item)])
+
+
+def _dedupe(items: list[str]) -> list[str]:
+    seen: set[str] = set()
+    result: list[str] = []
+    for item in items:
+        if item in seen:
+            continue
+        seen.add(item)
+        result.append(item)
+    return result