feat(tier3): upgrade Harbor to 0.22.0

.github/workflows/ci.yml

@@ -183,23 +183,25 @@ jobs:
         run: python scripts/check_oss_boundary.py --root . --allowlist config/oss_boundary_allowlist.json --archive dist/*.whl --archive dist/*.tar.gz
       - name: Validate distribution metadata
         run: uvx --from twine==6.2.0 twine check --strict dist/*
-      - name: Install and smoke-test the wheel
+      - name: Install and smoke-test the wheel on Python 3.12 and 3.13
         shell: bash
         run: |
           wheel="$(find "$PWD/dist" -maxdepth 1 -type f -name 'skillevaluator-*.whl' -print -quit)"
           test -n "$wheel"
-          venv="$RUNNER_TEMP/skillevaluator-wheel"
-          uv venv --python 3.13 "$venv"
-          uv pip install --python "$venv/bin/python" "${wheel}[tier3,llm]"
-          cd "$RUNNER_TEMP"
-          "$venv/bin/python" -c 'import anthropic, harbor, openai, skillevaluator, skillevaluator.model_catalog, skillevaluator.model_commands'
-          "$venv/bin/python" -c 'from importlib.resources import files; assert files("skillevaluator.tier3.harbor").joinpath("templates/eval.py").is_file()'
-          "$venv/bin/skillevaluator" --version
-          "$venv/bin/skillevaluator" --help >/dev/null
-          "$venv/bin/skillevaluator" models --help >/dev/null
-          SKILL_EVAL_LLM_PROVIDER=nv_build NVIDIA_API_KEY=nvapi-ci-placeholder \
-            "$venv/bin/skillevaluator" doctor --agents opencode --env-mode docker >/dev/null
-          "$venv/bin/python" -c 'import io; from skillevaluator.tier3.harbor.progress import PlainProgressReporter,ProgressEvent,Tier3RunPlan; s=io.StringIO(); r=PlainProgressReporter(stream=s,refresh_interval=60); r.start(Tier3RunPlan(skill_name="wheel-smoke",environment="docker",agents=("opencode",))); r.emit(ProgressEvent(stage="configuration",state="complete")); r.close(); assert "configuration: complete" in s.getvalue()'
+          for python_version in 3.12 3.13; do
+            venv="$RUNNER_TEMP/skillevaluator-wheel-${python_version}"
+            uv venv --python "$python_version" "$venv"
+            uv pip install --python "$venv/bin/python" "${wheel}[tier3]"
+            cd "$RUNNER_TEMP"
+            "$venv/bin/python" -c 'import anthropic, harbor, openai, skillevaluator, skillevaluator.model_catalog, skillevaluator.model_commands'
+            "$venv/bin/python" -c 'from importlib.resources import files; assert files("skillevaluator.tier3.harbor").joinpath("templates/eval.py").is_file()'
+            "$venv/bin/skillevaluator" --version
+            "$venv/bin/skillevaluator" --help >/dev/null
+            "$venv/bin/skillevaluator" models --help >/dev/null
+            SKILL_EVAL_LLM_PROVIDER=nv_build NVIDIA_API_KEY=nvapi-ci-placeholder \
+              "$venv/bin/skillevaluator" doctor --agents opencode --env-mode docker >/dev/null
+            "$venv/bin/python" -c 'import io; from skillevaluator.tier3.harbor.progress import PlainProgressReporter,ProgressEvent,Tier3RunPlan; s=io.StringIO(); r=PlainProgressReporter(stream=s,refresh_interval=60); r.start(Tier3RunPlan(skill_name="wheel-smoke",environment="docker",agents=("opencode",))); r.emit(ProgressEvent(stage="configuration",state="complete")); r.close(); assert "configuration: complete" in s.getvalue()'
+          done
 
   tier3-macos:
     name: Tier 3 macOS contract and progress
@@ -275,6 +277,7 @@ jobs:
           tests/reporting
           tests/test_cli.py
           tests/test_harbor_output_provenance.py
+          tests/test_harbor_runner_status.py
           tests/test_harbor_runtime_skill_isolation.py
           tests/test_harbor_secure_copy.py::test_checked_windows_fallback_accepts_crt_descriptor_identity
           tests/test_harbor_secure_copy.py::test_checked_windows_fallback_verifies_portable_chmod_semantics

.gitleaks.toml

@@ -9,3 +9,17 @@ condition = "AND"
 targetRules = ["generic-api-key"]
 regexes = ['''^sk-AbCdEf1234567890$''']
 paths = ['''^tests/test_tier3_(progress|result_display)\.py$''']
+
+[[allowlists]]
+description = "Synthetic NVIDIA Build token used by streaming redaction tests"
+condition = "AND"
+targetRules = ["generic-api-key"]
+regexes = ['''^Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8$''']
+paths = ['''^tests/test_harbor_local_mode\.py$''']
+
+[[allowlists]]
+description = "Synthetic main-container values used by Docker isolation tests"
+condition = "AND"
+targetRules = ["generic-api-key"]
+regexes = ['''^main-(persistent|task|scoped)-secret-[0-9]{5}$''']
+paths = ['''^tests/test_harbor_secure_docker_environment\.py$''']

CHANGELOG.md

@@ -4,12 +4,67 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ## Unreleased
 
+### Changed
+
+- Upgraded the optional Tier 3 backend to Harbor 0.22.0 and its compatible
+  LiteLLM 1.92-1.93 window. Existing SkillEvaluator agent and environment
+  options now use Harbor's unified selectors; installed Codex adapters preserve
+  merged user and MCP configuration; Docker and local execution stream redacted
+  callbacks under a shared 16 MiB per-command output limit, while the parent
+  Harbor orchestration process has a separate 16 MiB combined stdout/stderr
+  limit; and Docker supports stdin plus isolated sidecar operations without
+  exposing environment values on Compose argv. Generated schema 1.3 and
+  unmodified native task schemas remain compatible, while collection accepts
+  Harbor 0.22 job, trial, reward, and ATIF v1.7 artifacts.
+- Exposed Harbor 0.22's complete 26-backend environment set alongside local
+  mode. Non-secret backend constructor options can be supplied with repeatable,
+  operator-only `--environment-kwarg` / `--ek` flags; skill-owned configuration,
+  credentials, and sandbox-policy overrides remain outside that surface.
+
 ### Fixed
 
+- Tier 3 native-task collection now keeps Harbor's staged directory selector,
+  logical dataset ID, and display name separate; ambiguous or unresolved
+  persisted identities fail closed instead of trusting grader-authored IDs.
+  Runner-owned attempt ordinals are carried structurally, so `attempt`-like
+  text in authored selectors, logical IDs, or display names cannot corrupt
+  pass@k or `stop_on_pass` accounting, including truncated aggregate names.
+- Tier 3 Harbor subprocess, Docker, and local diagnostics now redact raw and
+  percent-decoded URI/proxy userinfo components across streamed callbacks,
+  nonzero exits, timeouts, output limits, and persisted launch errors.
+- Tier 3 local OpenCode runs routed through NVIDIA Build now retain the rendered
+  user instruction for ATIF conversion and fail on OpenCode error events, in
+  parity with Harbor 0.22's upstream agent lifecycle.
+- Tier 3 reports now use the collector's logical attempt overall whenever a
+  condition mixes standard and custom rewards, including across separate
+  trials, and aggregate execution summaries preserve child-declared hidden
+  error counts and truncation through launch-error overlays.
+- Tier 3 now rejects non-finite, overflowing, and finite-but-unscalable timeout
+  multipliers at YAML, programmatic, and Harbor command boundaries.
+- Tier 3 now preserves paired baseline isolation by rejecting skill-owned
+  pre-agent setup, native task/step healthchecks, native step workdir overlays,
+  and Harbor task-shipped prior trajectories whenever the baseline arm is
+  enabled. Native standard grading also fails closed on step test overlays,
+  separate verifier contexts, post-agent collect hooks, and task-controlled
+  verifier executable-path, shell, loader, proxy/TLS, provider, and judge
+  environment controls; exact operator-staged provider/judge placeholders and
+  unrelated verifier variables remain compatible. Operator-configured judge
+  fallback models are forwarded only to standard verifier jobs, not agent or
+  `custom_only` environments. Its Python payload now runs from a replaced
+  evaluator-owned directory in isolated mode. `custom_only`
+  retains Harbor-native collect hooks and Harbor 0.22's shared/separate
+  step-test resolution, and fully authored native test paths are left untouched
+  instead of replacing their unused `tests/skill_evaluator/` package.
+  All native grading modes reject Windows agent or effective verifier
+  environments until evaluator projection and verifier scripts are OS-aware.
 - Tier 3 paired pass@k evidence now respects Python's active integer-string
   conversion limit, preserves nonzero Wilson interval widths and paired-effect
   directions at large case counts, and documents exact-rational omission
   markers.
+- Tier 3 collection now fails closed on unsafe reward identities and malformed
+  custom-metric contracts, publishes exact truncation metadata for bounded
+  case and failure-detail samples, and keeps findings, attribution, and
+  per-trial JSON inside the report loader's artifact envelope.
 
 ## 0.2.1 - 2026-08-24
 

THIRD_PARTY_NOTICES.md

@@ -8,5 +8,5 @@ records the exact resolved dependency set used for this release.
 | --- | --- |
 | Base | Click (BSD-3-Clause), IDNA (BSD-3-Clause), Jinja2 (BSD-3-Clause), Markdown-It-Py (MIT), Pydantic (MIT), PyYAML (MIT), Rich (MIT) |
 | LLM | Anthropic (MIT), Boto3 (Apache-2.0), LiteLLM (MIT), OpenAI (Apache-2.0) |
-| Tier 3 | Harbor (Apache-2.0) |
+| Tier 3 | Harbor (Apache-2.0), MCP (MIT), PyJWT (MIT) |
 | Security | Bandit (Apache-2.0), pip-audit (Apache-2.0) |